/******************************************************************************* ;******************************************************************************* ;** ** ;** COPYRIGHT 2001-2012 NUANCE COMMUNICATIONS ** ;** ** ;** NUANCE COMMUNICATIONS PROPRIETARY INFORMATION ** ;** ** ;** This software is supplied under the terms of a license agreement ** ;** or non-disclosure agreement with Nuance Communications and may not ** ;** be copied or disclosed except in accordance with the terms of that ** ;** agreement. ** ;** ** ;******************************************************************************* ;** ** ;** FileName: et9aslst.c ** ;** ** ;** Description: ET9 Alphabetic Selection List Module ** ;** ** ;******************************************************************************* ;******************************************************************************/ /*! \addtogroup et9slst Selection list for alphabetic * Selection list for alphabetic XT9. * @{ */ #include "et9api.h" #ifdef ET9_ALPHABETIC_MODULE #include "et9asys.h" #include "et9asym.h" #include "et9sym.h" #include "et9amisc.h" #include "et9adb.h" #include "et9adlm.h" #include "et9arudb.h" #include "et9acdb.h" #include "et9imu.h" #include "et9aspc.h" #include "et9aldb.h" #include "et9aslst.h" #include "et9sys.h" #include "et9alsasdb.h" #ifdef ET9_SPM_MODULE #include "et9aspath.h" #endif #ifdef EVAL_BUILD #include "__et9eval.h" #endif #ifdef ET9_ASLST_ACTIVATE_LOG #define ET9_DEBUGLOG2 #undef WLOG2 #undef __LogCDB #undef WLOG2Word #undef WLOG2String #endif #ifdef ET9_DEBUGLOG2 #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG2 ACTIVATED ***") #endif #include #include #ifdef XT9_ANDROID_BUILD static const char pczzzET9ASlStB[] = "/sdcard/data/zzzET9ASlStB.txt"; #else static const char pczzzET9ASlStB[] = "zzzET9ASlStB.txt"; #endif #define WLOG2(q) { if (pLogFile2 == NULL) { pLogFile2 = fopen(pczzzET9ASlStB, "w+"); } {q} fflush(pLogFile2); } static FILE *pLogFile2 = NULL; #else #define WLOG2(q) #endif #ifdef ET9_DEBUGLOG2B #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG2B ACTIVATED ***") #endif #define WLOG2B(q) WLOG2(q) #define WLOG2BWord(p1,p2,p3) WLOG2Word(p1,p2,p3) #else #define WLOG2B(q) #define WLOG2BWord(p1,p2,p3) #endif #if 1 /* log8 */ #ifdef ET9_DEBUGLOG8 #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG8 ACTIVATED ***") #endif #include #include #define WLOG8(q) { if (pLogFile8 == NULL) { pLogFile8 = fopen("zzzET9LDBTRACE.txt", "w"); } { q fflush(pLogFile8); } } static FILE *pLogFile8 = NULL; #else #define WLOG8(q) #endif #ifdef ET9_DEBUGLOG8B #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG8 ACTIVATED ***") #endif #define WLOG8B(q) WLOG8(q) #else #define WLOG8B(q) #endif #else /* log8 */ #define pLogFile8 pLogFile2 #ifndef WLOG8 #define WLOG8(q) WLOG2(q) #endif #ifndef WLOG8B #define WLOG8B(q) WLOG2B(q) #endif #endif /* log8 */ #ifdef ET9_DEBUG #include #endif /******************************************************************************* ** ** G L O B A L S A N D L O C A L S T A T I C S ** ** ET9 does not make use of any dynamic global or local static variables!! ** It is acceptable to make use of constant globals or local statics. ** If you need persistent dynamic memory in the ET9 core, it should be ** allocated in the ET9AWLingPrivate data structure and fogged through the definitions ** found in the et9asystm.h file. ** ******************************************************************************/ /*---------------------------------------------------------------------------- * Defines *----------------------------------------------------------------------------*/ #undef __ValueSwap #define __MaxStemDistanceFromInputLength(nInputLength) (((nInputLength) - 1) / 4 + 1) #define __SINGLE_KEYPRESS_MAX_WORD_INDEX 100 /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef enum ET9ASPECIAL_e { NOSPECIAL, /* < -IDR- */ PUNCT, /* < -IDR- */ SMARTPUNCT, /* < -IDR- */ EXPLICIT, /* < -IDR- */ LOCKPOINT, /* < -IDR- */ LAST_SPECIAL /* < -IDR- */ } ET9ASPECIAL; /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef enum ET9ABUILAROUND_e { BA_TRAILING, /* < -IDR- */ BA_LEADING, /* < -IDR- */ BA_EMBEDDED, /* < -IDR- */ BA_COMPOUND, /* < -IDR- */ BA_LAST /* < -IDR- */ } ET9ABUILAROUND; /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef enum __SortStage_e { SS_Final = 0, /* < -IDR- */ SS_Done, /* < -IDR- */ SS_Collecting, /* < -IDR- */ SS_Last /* < -IDR- */ } __SortStage; /*---------------------------------------------------------------------------*/ /* -IDR- * * */ enum ET9AWORDQUALITY_e { UNDEF_QUALITY, /* < -IDR- */ DISPOSABLE_QUALITY, /* < -IDR- */ STEM_QUALITY, /* < -IDR- */ DUPE_QUALITY, /* < -IDR- */ LIMITED_QUALITY, /* < -IDR- */ SHAPED_QUALITY, /* < -IDR- */ GENUINE_QUALITY, /* < -IDR- */ LAST_QUALITY /* < -IDR- */ }; #define UNDEFINED_STRING_INDEX 0xFFFF /* < -IDR- */ #define MAX_AUTO_APPEND_SYMBS 16 /* < -IDR- */ #define WORD_SOURCE_DEMOTE_FENCE 2 /* < -IDR- */ #define EMOTICON_LENGTH_THRESHOLD 5 /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __IsShiftNeutralSymb(ET9SymbInfo const * const pSymbInfo) { return (ET9BOOL)(pSymbInfo->bSymbType == ET9KTNUMBER || pSymbInfo->bSymbType == ET9KTSMARTPUNCT || pSymbInfo->bSymbType == ET9KTPUNCTUATION); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __IsExactSymb(ET9SymbInfo const * const pSymbInfo) { if (pSymbInfo->bTraceIndex) { return 0; } if (pSymbInfo->bAmbigType == ET9EXACT) { return 1; } { ET9U8 bNumBaseSyms; for (bNumBaseSyms = pSymbInfo->bNumBaseSyms; bNumBaseSyms > 1; --bNumBaseSyms) { if (!pSymbInfo->DataPerBaseSym[bNumBaseSyms - 1].bLimited) { break; } } if (bNumBaseSyms == 1 && pSymbInfo->DataPerBaseSym[0].bNumSymsToMatch == 1 && _ET9_IsPunctOrNumeric(pSymbInfo->DataPerBaseSym[0].sChar[0])) { return 1; } if (bNumBaseSyms == 1 && pSymbInfo->DataPerBaseSym[0].bNumSymsToMatch == 2 && pSymbInfo->DataPerBaseSym[0].sChar[0] == pSymbInfo->DataPerBaseSym[0].sChar[1] && _ET9_IsPunctOrNumeric((pSymbInfo)->DataPerBaseSym[0].sChar[0])) { return 1; } } return 0; } #define QUALITYTOSTRING(x) (x == GENUINE_QUALITY ? "GEN" : x == SHAPED_QUALITY ? "SHP" : x == LIMITED_QUALITY ? "LTD" : x == DUPE_QUALITY ? "DUP" : x == STEM_QUALITY ? "STM" : x == DISPOSABLE_QUALITY ? "DSP" : "\?\?\?") #define LINDEXTOSTRING(x) (x == ET9AWFIRST_LANGUAGE ? "1ST" : x == ET9AWSECOND_LANGUAGE ? "2ND" : x == ET9AWBOTH_LANGUAGES ? "BTH" : x == ET9AWUNKNOWN_LANGUAGE ? "UKN" : "\?\?\?") #define SPECIALTOSTRING(x) (x == NOSPECIAL ? "NOSPECIAL" : x == PUNCT ? "PUNCT" : x == SMARTPUNCT ? "SMARTPUNCT" : x == EXPLICIT ? "EXPLICIT" : x == LOCKPOINT ? "LOCKPOINT" : "\?\?\?") #define BUILDAROUNDTOSTRING(x) (x == BA_TRAILING ? "TRAILING" : x == BA_LEADING ? "LEADING" : x == BA_EMBEDDED ? "EMBEDDED" : x == BA_COMPOUND ? "COMPOUND" : "\?\?\?") #define SELLSTMODETOSTRING(x) (x == ET9ASLMODE_AUTO ? "AUTO" : x == ET9ASLMODE_CLASSIC ? "CLASSIC" : x == ET9ASLMODE_COMPLETIONSPROMOTED ? "COMPLETIONSPROMOTED" : x == ET9ASLMODE_MIXED ? "MIXED" : x == ET9ASLMODE_SUFFIX ? "SUFFIX" : "\?\?\?") #define SELLSTCORRMODETOSTRING(x) (x == ET9ASLCORRECTIONMODE_LOW ? "LOW" : x == ET9ASLCORRECTIONMODE_MEDIUM ? "MEDIUM" : x == ET9ASLCORRECTIONMODE_HIGH ? "HIGH" : "\?\?\?") #define SPCMODETOSTRING(x) (x == ET9ASPCMODE_OFF ? "OFF" : x == ET9ASPCMODE_EXACT ? "EXACT" : x == ET9ASPCMODE_REGIONAL ? "REGIONAL" : "\?\?\?") #define SPCFILTERTOSTRING(x) (x == ET9ASPCSEARCHFILTER_UNFILTERED ? "UNFILTERED" : x == ET9ASPCSEARCHFILTER_ONE_EXACT ? "ONE_EXACT" : x == ET9ASPCSEARCHFILTER_ONE_REGIONAL ? "ONE_REGIONAL" : x == ET9ASPCSEARCHFILTER_TWO_EXACT ? "TWO_EXACT" : x == ET9ASPCSEARCHFILTER_TWO_REGIONAL ? "TWO_REGIONAL" : "\?\?\?") #define ALTMODETOSTRING(x) (x == ET9AW_AltMode_1 ? "AltMode_1" : x == ET9AW_AltMode_2 ? "AltMode_2" : "\?\?\?") static ET9STATUS ET9LOCALCALL __ET9AWDoSelLstBuild(ET9AWLingInfo * const pLingInfo, ET9U8 * const pbTotalWords, const ET9BOOL bSuppressBuild, ET9U16 * const pwGestureValue); static void ET9LOCALCALL __ResetAllCaptureInfo(ET9AWLingInfo * const pLingInfo); static void ET9LOCALCALL __CaptureWord(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wSymbolLen); static void ET9LOCALCALL __CaptureDefault(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wSymbolLen, const ET9U16 bLastBuildLen); static ET9U16 ET9LOCALCALL __CaptureGetFlushStringLengthAtPoint(ET9AWLingInfo * const pLingInfo, const ET9U16 wSymbolPoint); /* ************************************************************************************************************** */ /* * LOG SUPPORT ************************************************************************************************ */ /* ************************************************************************************************************** */ #ifdef ET9_DEBUGLOG2 #define ET9AssertLog(x) { (void)((!!(x)) || !pLogFile2 || fprintf(pLogFile2, "\n\nASSERT line %d\n%s\n\n", __LINE__, #x) && fflush(pLogFile2)); ET9Assert(x); } #else /* ET9_DEBUGLOG2 */ #define ET9AssertLog(x) ET9Assert(x) #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 #include /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static char* ET9LOCALCALL __sourceToString(ET9U8 bSrc, ET9BOOL bIsUDBWord) { char *pcwsrc; static char pcwTmp[8]; switch (GETBASESRC(bSrc)) { case ET9WORDSRC_NONE: pcwsrc = "NONE"; break; case ET9WORDSRC_EXACT: pcwsrc = "EXACT"; break; case ET9WORDSRC_EXACTISH: pcwsrc = "EXACTISH"; break; case ET9WORDSRC_CSP: pcwsrc = "CSP"; break; case ET9WORDSRC_DLM: pcwsrc = "DLM"; break; case ET9WORDSRC_CDB: pcwsrc = "CDB"; break; case ET9WORDSRC_QUDB: pcwsrc = "QUDB"; break; case ET9WORDSRC_RUDB: pcwsrc = bIsUDBWord ? "UDB" : "RDB"; break; case ET9WORDSRC_MDB: pcwsrc = "MDB"; break; case ET9WORDSRC_ASDB_SHORTCUT: pcwsrc = "ASDB_SHORTCUT"; break; case ET9WORDSRC_LAS_SHORTCUT: pcwsrc = "LAS_SHORTCUT"; break; case ET9WORDSRC_LDB: pcwsrc = "LDB"; break; case ET9WORDSRC_KDB: pcwsrc = "KDB"; break; case ET9WORDSRC_BUILDAROUND_CSP: pcwsrc = "BUILDAROUND_CSP"; break; case ET9WORDSRC_BUILDAROUND_DLM: pcwsrc = "BUILDAROUND_DLM"; break; case ET9WORDSRC_BUILDAROUND_CDB: pcwsrc = "BUILDAROUND_CDB"; break; case ET9WORDSRC_BUILDAROUND_QUDB: pcwsrc = "BUILDAROUND_QUDB"; break; case ET9WORDSRC_BUILDAROUND_RUDB: pcwsrc = bIsUDBWord ? "BUILDAROUND_UDB" : "BUILDAROUND_RDB"; break; case ET9WORDSRC_BUILDAROUND_MDB: pcwsrc = "BUILDAROUND_MDB"; break; case ET9WORDSRC_BUILDAROUND_ASDB: pcwsrc = "BUILDAROUND_ASDB"; break; case ET9WORDSRC_BUILDAROUND_LAS: pcwsrc = "BUILDAROUND_LAS"; break; case ET9WORDSRC_BUILDAROUND_LDB: pcwsrc = "BUILDAROUND_LDB"; break; case ET9WORDSRC_BUILDAROUND_KDB: pcwsrc = "BUILDAROUND_KDB"; break; case ET9WORDSRC_BUILDCOMPOUND_CSP: pcwsrc = "BUILDCOMPOUND_CSP"; break; case ET9WORDSRC_BUILDCOMPOUND_DLM: pcwsrc = "BUILDCOMPOUND_DLM"; break; case ET9WORDSRC_BUILDCOMPOUND_CDB: pcwsrc = "BUILDCOMPOUND_CDB"; break; case ET9WORDSRC_BUILDCOMPOUND_QUDB: pcwsrc = "BUILDCOMPOUND_QUDB"; break; case ET9WORDSRC_BUILDCOMPOUND_RUDB: pcwsrc = bIsUDBWord ? "BUILDCOMPOUND_UDB" : "BUILDCOMPOUND_RDB"; break; case ET9WORDSRC_BUILDCOMPOUND_MDB: pcwsrc = "BUILDCOMPOUND_MDB"; break; case ET9WORDSRC_BUILDCOMPOUND_ASDB: pcwsrc = "BUILDCOMPOUND_ASDB"; break; case ET9WORDSRC_BUILDCOMPOUND_LAS: pcwsrc = "BUILDCOMPOUND_LAS"; break; case ET9WORDSRC_BUILDCOMPOUND_LDB: pcwsrc = "BUILDCOMPOUND_LDB"; break; case ET9WORDSRC_BUILDCOMPOUND_KDB: pcwsrc = "BUILDCOMPOUND_KDB"; break; case ET9WORDSRC_TERMPUNCT: pcwsrc = "TERMPUNCT"; break; case ET9WORDSRC_COMPLETIONPUNCT: pcwsrc = "COMPLETIONPUNCT"; break; case ET9WORDSRC_REQUIRED: pcwsrc = "REQUIRED"; break; case ET9WORDSRC_GDB: pcwsrc = "GDB"; break; case ET9WORDSRC_STEMPOOL: pcwsrc = "STEMPOOL"; break; case ET9WORDSRC_MAGICSTRING: pcwsrc = "MAGICSTRING"; break; case ET9WORDSRC_TERM: pcwsrc = "TERM"; break; case ET9WORDSRC_STEM: pcwsrc = "STEM"; break; case ET9WORDSRC_AUTOAPPEND: pcwsrc = "AUTOAPPEND"; break; case ET9WORDSRC_BUILDAPPEND: pcwsrc = "BUILDAPPEND"; break; case ET9WORDSRC_BUILDAPPENDPUNCT: pcwsrc = "BUILDAPPENDPUNCT"; break; case ET9WORDSRC_AUTOSPACE: pcwsrc = "AUTOSPACE"; break; case ET9WORDSRC_EXACT_LAST: pcwsrc = "EXACT_LAST"; break; default : pcwsrc = pcwTmp; sprintf(pcwTmp, "%d", GETBASESRC(bSrc)); break; } return pcwsrc; } #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static ET9UINT ET9LOCALCALL WLOG2String(FILE *pF, char *pcComment, ET9SYMB *psString, ET9INT snLen, ET9BOOL bReverse, ET9BOOL bSilent) { ET9INT snIndex; ET9UINT nLenUsed = 0; ET9AssertLog(pF != NULL); ET9AssertLog(pcComment != NULL); ET9AssertLog(psString != NULL); if (pcComment && *pcComment) { fprintf(pF, "%s: ", pcComment); } for (snIndex = bReverse ? (snLen - 1): 0; bReverse ? snIndex >= 0: snIndex < snLen; bReverse ? --snIndex : ++snIndex) { ET9SYMB sSymb = psString[snIndex]; if (sSymb > 0x20 && sSymb <= 0x7F || sSymb >= 0xA1 && sSymb <= 0xFF) { nLenUsed += fprintf(pF, "%c", (unsigned char)sSymb); } else { nLenUsed += fprintf(pF, "<%x>", (int)sSymb); } } if (!bSilent) { fprintf(pF, " (%d)\n", snLen); } fflush(pF); #ifdef ET9_DEBUG { ET9U16 wCount = (ET9U16)snLen; ET9SYMB *pSymb = psString; while (wCount--) { ET9AssertLog(*pSymb && *pSymb != (ET9SYMB)0xcccc); ++pSymb; } } #endif return nLenUsed; } #else /* ET9_DEBUGLOG2 */ #define WLOG2String(pF,pcComment,psString,snLen,bReverse,bSilent) #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __LogInputSymbs(ET9AWLingInfo *pLingInfo, ET9U8 bCount, FILE *pfLogFile) { ET9U16 wIndex; ET9INT snLockPos; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; fprintf(pfLogFile, "\n\n__LogInputSymbs, wReverseIndex %u, wReverseLength %u\n\n", pWordSymbInfo->Private.wReverseIndex, pWordSymbInfo->Private.wReverseLength); { ET9INT snIndex; fprintf(pfLogFile, "Context:"); for (snIndex = ET9NLM_CONTEXT_COUNT - 1; snIndex >= 0; --snIndex) { fprintf(pfLogFile, " "); if (!pLingCmnInfo->Private.pContextWords[snIndex].wLen) { fprintf(pfLogFile, "[NONE]"); } else { WLOG2String(pfLogFile, "", pLingCmnInfo->Private.pContextWords[snIndex].sString, pLingCmnInfo->Private.pContextWords[snIndex].wLen, 0, 1); } } fprintf(pfLogFile, "\n\n"); } snLockPos = -1; for (wIndex = 0; wIndex < bCount; ++wIndex) { ET9SymbInfo * const pSymbInfo = &pWordSymbInfo->SymbsInfo[wIndex]; if (pSymbInfo->bLocked) { snLockPos = wIndex; } } fprintf(pfLogFile, "Shift-Gesture %c, Caps-Gesture %c\n\n", (pLingCmnInfo->Base.pWordSymbInfo->Private.bHasShiftGesture ? 'Y' : 'N'), (pLingCmnInfo->Base.pWordSymbInfo->Private.bHasAllCapsGesture ? 'Y' : 'N')); for (wIndex = 0; wIndex < bCount; ++wIndex) { ET9U16 wBaseIndex; ET9U16 wSymsIndex; ET9SymbInfo * const pSymbInfo = &pWordSymbInfo->SymbsInfo[wIndex]; fprintf(pfLogFile, "Pos %2d ", wIndex); switch (pSymbInfo->eInputType) { case ET9DISCRETEKEY: fprintf(pfLogFile, "ET9DISCRETEKEY"); break; case ET9REGIONALKEY: fprintf(pfLogFile, "ET9REGIONALKEY"); break; case ET9HANDWRITING: fprintf(pfLogFile, "ET9HANDWRITING"); break; case ET9MULTITAPKEY: fprintf(pfLogFile, "ET9MULTITAPKEY"); break; case ET9CUSTOMSET: fprintf(pfLogFile, "ET9CUSTOMSET"); break; case ET9EXPLICITSYM: fprintf(pfLogFile, "ET9EXPLICITSYM"); break; case ET9MULTISYMBEXPLICIT: fprintf(pfLogFile, "ET9MULTISYMBEXPLICIT"); break; default: fprintf(pfLogFile, "Input<%d>", pSymbInfo->eInputType); break; } fprintf(pfLogFile, " "); switch (pSymbInfo->bAmbigType) { case ET9AMBIG: fprintf(pfLogFile, "ET9AMBIG"); break; case ET9EXACT: fprintf(pfLogFile, "ET9EXACT"); break; default: fprintf(pfLogFile, "Ambig<%d>", pSymbInfo->bAmbigType); break; } if (pSymbInfo->bAmbigType == ET9AMBIG && __IsExactSymb(pSymbInfo)) { fprintf(pfLogFile, "+SSX"); } fprintf(pfLogFile, " "); switch (pSymbInfo->bSymbType) { case ET9KTINVALID: fprintf(pfLogFile, "ET9KTINVALID"); break; case ET9KTLETTER: fprintf(pfLogFile, "ET9KTLETTER"); break; case ET9KTPUNCTUATION: fprintf(pfLogFile, "ET9KTPUNCTUATION"); break; case ET9KTNUMBER: fprintf(pfLogFile, "ET9KTNUMBER"); break; case ET9KTSTRING: fprintf(pfLogFile, "ET9KTSTRING"); break; case ET9KTFUNCTION: fprintf(pfLogFile, "ET9KTFUNCTION"); break; case ET9KTSMARTPUNCT: fprintf(pfLogFile, "ET9KTSMARTPUNCT"); break; case ET9KTUNKNOWN: fprintf(pfLogFile, "ET9KTUNKNOWN"); break; default: fprintf(pfLogFile, "Symb<%d>", pSymbInfo->bSymbType); break; } if (pSymbInfo->eShiftState == ET9SHIFT) { fprintf(pfLogFile, " SHIFT"); } else if (pSymbInfo->eShiftState == ET9CAPSLOCK) { fprintf(pfLogFile, " CAPSLOCK"); } else if (pSymbInfo->bForcedLowercase) { fprintf(pfLogFile, " FORCED LOWERCASE"); } if (ET9_SPC_IS_LOCKED_POS(pSymbInfo)) { fprintf(pfLogFile, " SPC-LOCKED"); } if (pLingCmnInfo->Private.sBuildInfo.pbFlushPos[wIndex] && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[wIndex] <= pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { fprintf(pfLogFile, " FLUSHED"); } switch (pSymbInfo->bLocked) { case ET9NOLOCK: break; case ET9STEMLOCK: fprintf(pfLogFile, " STEMLOCK"); break; case ET9EXACTLOCK: fprintf(pfLogFile, " EXACTLOCK"); break; default: fprintf(pfLogFile, " LOCK<%d>", pSymbInfo->bLocked); break; } if (pSymbInfo->wKeyIndex != ET9UNDEFINEDKEYVALUE) { fprintf(pfLogFile, " (Key %d)", (int)pSymbInfo->wKeyIndex); } if (pSymbInfo->wTapX != ET9UNDEFINEDTAPVALUE && pSymbInfo->wTapY != ET9UNDEFINEDTAPVALUE) { fprintf(pfLogFile, " [%d,%d]", (int)pSymbInfo->wTapX, (int)pSymbInfo->wTapY); } if (pSymbInfo->eInputType == ET9MULTISYMBEXPLICIT) { fprintf(pfLogFile, " (built by %d)", (int)pSymbInfo->wInputIndex); } fprintf(pfLogFile, " (ext %d %d %3.1f)", (int)pSymbInfo->bTraceProbability, (int)pSymbInfo->bTraceIndex, (float)pSymbInfo->fTraceLen); fprintf(pfLogFile, "\n"); if (!pLingCmnInfo->Private.sWordC.pCurrC->bPureCollection && wIndex <= snLockPos) { ET9SYMB sChar = pSymbInfo->sLockedSymb; fprintf(pfLogFile, " Locked symb "); if (sChar > 0x20 && sChar <= 0x7F || sChar >= 0xA1 && sChar <= 0xFF) { fprintf(pfLogFile, "%c", sChar); } else { fprintf(pfLogFile, "<%x>", sChar); } fprintf(pfLogFile, "\n"); } for (wBaseIndex = 0; wBaseIndex < pSymbInfo->bNumBaseSyms; ++wBaseIndex) { fprintf(pfLogFile, " SymbsInfo %2u (%3d %3d)%c ", wBaseIndex, pSymbInfo->DataPerBaseSym[wBaseIndex].bSymFreq, pSymbInfo->DataPerBaseSym[wBaseIndex].bSymDist, (pSymbInfo->DataPerBaseSym[wBaseIndex].bLimited ? 'L' : ' ')); for (wSymsIndex = 0; wSymsIndex < pSymbInfo->DataPerBaseSym[wBaseIndex].bNumSymsToMatch; ++wSymsIndex) { ET9SYMB sChar = pSymbInfo->DataPerBaseSym[wBaseIndex].sChar[wSymsIndex]; if (sChar > 0x20 && sChar <= 0x7F || sChar >= 0xA1 && sChar <= 0xFF) { fprintf(pfLogFile, "%c", sChar); } else { fprintf(pfLogFile, "<%x>", sChar); } } fprintf(pfLogFile, " "); for (wSymsIndex = 0; wSymsIndex < pSymbInfo->DataPerBaseSym[wBaseIndex].bNumSymsToMatch; ++wSymsIndex) { ET9SYMB sChar = pSymbInfo->DataPerBaseSym[wBaseIndex].sUpperCaseChar[wSymsIndex]; if (sChar > 0x20 && sChar <= 0x7F || sChar >= 0xA1 && sChar <= 0xFF) { fprintf(pfLogFile, "%c", sChar); } else { fprintf(pfLogFile, "<%x>", sChar); } } fprintf(pfLogFile, "\n"); } } fprintf(pfLogFile, "\n"); fflush(pfLogFile); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static void ET9LOCALCALL __LogPartialSelList(ET9AWLingCmnInfo * const pLingCmnInfo, ET9UINT nFirstPos, ET9UINT nLastPos, char *pcNote, FILE *pfLogFile) { ET9UINT nIndex; ET9UINT nMaxWordLen = 0; FILE *pfLog = pfLogFile; if (nFirstPos >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || nLastPos >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { fprintf(pfLog, "__LogPartialSelList - invalid partition\n"); return; } for (nIndex = nFirstPos; nIndex <= nLastPos; ++nIndex) { ET9UINT nThisLen; ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; nThisLen = pWord->Base.wWordLen; if (pWord->Base.wWordCompLen) { ++nThisLen; } if (pWord->Body.wPrefixLen) { nThisLen += 2; } if (nMaxWordLen < nThisLen) { nMaxWordLen = nThisLen; } } nMaxWordLen += 10; fprintf(pfLog, "\n"); if (pcNote && strlen(pcNote)) { fprintf(pfLog, "Note: %s\n\n", pcNote); } fprintf(pfLog, " SD ED STID FD L AC O C attr tot-freq tap-freq word-freq index input-score spm-score s-tap c-c s-dist\n"); for (nIndex = nFirstPos; nIndex <= nLastPos; ++nIndex) { ET9SYMB *ps; ET9U16 wCount; ET9INT snBytesWritten = 0; ET9INT snBytesBeforeWord; char *pcwsrc; char *pcwsrcext; ET9UINT nWordIndex = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]; ET9AWPrivWordInfo *pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[nWordIndex]; ET9AssertLog(pWord->Base.wWordLen); if (ISBASESRC(pWord->Body.bWordSrc)) { pcwsrcext = " "; } else if (ISEXACTSRC(pWord->Body.bWordSrc)) { pcwsrcext = "X "; } else { pcwsrcext = "XS"; } pcwsrc = __sourceToString(pWord->Body.bWordSrc, pWord->Body.bIsUDBWord); snBytesWritten += fprintf(pfLog, "[%03u] %2X, %2X:%1u|%u%u%u%u %2X %2X, %u%u %u|%u, %c%c%c%c%c%c, %20.0f (%14.2f %18.2f %6u), (%15.2f %15.2f %8.2f %4d %8.1f), %s %s %s %c %s %-18s", (nFirstPos == 0 && nLastPos == pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1 ? nIndex : nWordIndex), pWord->Body.bEditDistStem, pWord->Body.bEditDistSpc, pWord->Body.bHasPrimEditDist, pWord->Body.bEditDistSpcSbt, pWord->Body.bEditDistSpcTrp, pWord->Body.bEditDistSpcIns, pWord->Body.bEditDistSpcDel, pWord->Body.bEditDistFree, pWord->Body.bLimitedCount, pWord->Body.bIsAcronym, pWord->Body.bIsCapitalized, pWord->Body.bNLMOrder, pWord->Body.bCollectionPrio, (char)(pWord->Body.bIsWeak ? 'W' : ' '), (char)(pWord->Body.bIsGray ? 'G' : ' '), (char)(pWord->Body.bIsBlack ? 'B' : ' '), (char)(pWord->Body.bIsHidden ? 'H' : ' '), (char)(pWord->Body.bIsQuarantine ? 'Q' : ' '), (char)(pWord->Body.bIsRetain ? 'R' : ' '), (double)pWord->Body.xTotFreq, (double)pWord->Body.xTapFreq, (double)pWord->Body.xWordFreq, pWord->Body.dwWordIndex, (double)pWord->Body.xInputScore, (double)pWord->Body.xInputScoreSpm, (double)pWord->Body.xScaledTapFreq, pWord->Body.snCorrectionCost, (double)pWord->Body.xSymbDist, LINDEXTOSTRING(pWord->Base.bLangIndex), LINDEXTOSTRING(pWord->Body.bLangIndexScoring), pcwsrcext, (char)(pWord->Base.bIsTerm ? 'T' : 'S'), QUALITYTOSTRING(pWord->Body.bWordQuality), pcwsrc); if (nIndex + nFirstPos == pLingCmnInfo->Private.nDefaultIndex) { snBytesWritten += fprintf(pfLog, " * "); } else { snBytesWritten += fprintf(pfLog, " : "); } snBytesWritten += fprintf(pfLog, "%02u ", pWord->Base.wWordLen); snBytesBeforeWord = snBytesWritten; if (pWord->Body.wPrefixLen) { snBytesWritten += fprintf(pfLog, "{"); } for (ps = pWord->Base.sWord, wCount = pWord->Base.wWordLen; wCount; ++ps, --wCount) { if (wCount == pWord->Base.wWordCompLen) { snBytesWritten += fprintf(pfLog, "|"); } if (wCount == (pWord->Base.wWordLen - pWord->Body.wPrefixLen) && pWord->Body.wPrefixLen) { snBytesWritten += fprintf(pfLog, "}"); } if (*ps >= 0x20 && *ps <= 0x7F || *ps >= 0xA1 && *ps <= 0xFF) { snBytesWritten += fprintf(pfLog, "%c", (char)*ps); } else { snBytesWritten += fprintf(pfLog, "<%x>", (int)*ps); } } #ifdef ET9_DEBUG { const ET9INT snTarget = snBytesBeforeWord + nMaxWordLen; ET9INT snContextIndex; while (snBytesWritten < snTarget) { snBytesWritten += fprintf(pfLog, " "); } fprintf(pfLog, " "); for (snContextIndex = ET9NLM_CONTEXT_COUNT - 1; snContextIndex >= 0; --snContextIndex) { ET9SimpleWord * const pContextWord = (GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM) ? &pWord->Body.sScoreContext.pDLMContextWords[snContextIndex] : &pWord->Body.sScoreContext.pReverseContextWords[snContextIndex]; ET9UINT nIndex; if (!pContextWord->wLen) { snBytesWritten += fprintf(pfLog, "\xB0"); } for (nIndex = 0; nIndex < pContextWord->wLen; ++nIndex) { const ET9SYMB sSymb = pContextWord->sString[nIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F || sSymb >= 0xA1 && sSymb <= 0xFF) { snBytesWritten += fprintf(pfLog, "%c", (char)sSymb); } else { snBytesWritten += fprintf(pfLog, "<%x>", (int)sSymb); } } if (snContextIndex > 0) { fprintf(pfLog, " "); } } fprintf(pfLog, " "); } #endif #if 0 fprintf(pfLog, " "); for (ps = pWord->Base.sWord, wCount = pWord->Base.wWordLen; wCount; ++ps, --wCount) { if (wCount == pWord->Base.wWordCompLen) { fprintf(pfLog, "|"); } if (_ET9SymIsLower(*ps, pLingCmnInfo->Base.pWordSymbInfo->Private.wLocale)) { fprintf(pfLog, "L"); } else if (_ET9SymIsUpper(*ps, pLingCmnInfo->Base.pWordSymbInfo->Private.wLocale)) { fprintf(pfLog, "U"); } else { fprintf(pfLog, "N"); } } #endif if (pWord->Base.wSubstitutionLen) { fprintf(pfLog, " -> "); for (ps = pWord->Base.sSubstitution, wCount = pWord->Base.wSubstitutionLen; wCount; ++ps, --wCount) { if (*ps >= 0x20 && *ps <= 0x7F || *ps >= 0xA1 && *ps <= 0xFF) { fprintf(pfLog, "%c", (char)*ps); } else { fprintf(pfLog, "<%x>", (int)*ps); } } } fprintf(pfLog, "\n"); } fprintf(pfLog, "\n"); fflush(pfLog); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __LogFullSelList(ET9AWLingInfo *pLingInfo, char *pcNote, FILE *pfLogFile) { FILE *pfLog; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (pfLogFile != NULL) { pfLog = pfLogFile; } else { pfLog = fopen("zSlstLog.txt", "w"); } if (!pfLog) { return; } if (pfLogFile != NULL) { fprintf(pfLog, "\n***** SELECTION LIST START *****\n\n"); } fprintf(pfLog, "Note: %s\n\n", pcNote); fprintf(pfLog, "SPC mode = %d\n", pLingCmnInfo->Private.ASpc.eMode); fprintf(pfLog, "Selection list mode = %d\n", pLingCmnInfo->Private.eSelectionListMode); fprintf(pfLog, "Post Sort = %d\n", ET9POSTSORTENABLED(pLingCmnInfo)); fprintf(pfLog, "\n"); #ifdef ET9_ACTIVATE_SLST_STATS _ET9AWSpcPrintStats(pfLog, pLingInfo); #endif __LogInputSymbs(pLingInfo, pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs, pfLog); if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { __LogPartialSelList(pLingInfo->pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "", pfLog); } else { fprintf(pfLog, "<>\n\n"); } fflush(pfLog); if (pfLogFile != NULL) { fprintf(pfLog, "\n..... selection list end .....\n\n"); } if (pfLogFile == NULL) { fclose(pfLog); } } #else /* ET9_DEBUGLOG2 */ #define __LogInputSymbs(pLingInfo,bCount,pfLogFile) #define __LogPartialSelList(pLingInfo,bFirstPos,bLastPos,pcNote,pfLogFile) #define __LogFullSelList(pLingInfo,pcNote,pfLogFile) #endif /* ET9_DEBUGLOG2 */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ void ET9FARCALL _ET9_LogSelList(ET9AWLingCmnInfo * const pLingCmnInfo) { if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { __LogPartialSelList(pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "_ET9_LogSelList", pLogFile2); } } #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL WLOG2Word(FILE *pF, char *pcComment, ET9AWPrivWordInfo const * const pWord) { ET9U16 wIndex; ET9AssertLog(pF != NULL); ET9AssertLog(pcComment != NULL); fprintf(pF, "%s", pcComment); if (pWord == NULL) { fprintf(pF, "NULL\n"); return; } fprintf(pF, " (%c %s %s %s : %02d,%02d : %d,%d:%d|%u%u%u%u : %20s, tot %24.2f, tap %10.2f, stap %10.2f, wrd %12.0f, cc %4d, s-dist %8.1f, idx %6u, o %u|%u, a %c%c%c%c%c%c) ", (char)(pWord->Base.bIsTerm ? 'T' : 'S'), QUALITYTOSTRING(pWord->Body.bWordQuality), LINDEXTOSTRING(pWord->Base.bLangIndex), LINDEXTOSTRING(pWord->Body.bLangIndexScoring), pWord->Base.wWordLen, pWord->Base.wWordCompLen, pWord->Body.bEditDistStem, pWord->Body.bEditDistSpc, pWord->Body.bHasPrimEditDist, pWord->Body.bEditDistSpcSbt, pWord->Body.bEditDistSpcTrp, pWord->Body.bEditDistSpcIns, pWord->Body.bEditDistSpcDel, __sourceToString(pWord->Body.bWordSrc, pWord->Body.bIsUDBWord), (double)pWord->Body.xTotFreq, (double)pWord->Body.xTapFreq, (double)pWord->Body.xScaledTapFreq, (double)pWord->Body.xWordFreq, pWord->Body.snCorrectionCost, (double)pWord->Body.xSymbDist, pWord->Body.dwWordIndex, pWord->Body.bNLMOrder, pWord->Body.bCollectionPrio, (char)(pWord->Body.bIsWeak ? 'W' : ' '), (char)(pWord->Body.bIsGray ? 'G' : ' '), (char)(pWord->Body.bIsBlack ? 'B' : ' '), (char)(pWord->Body.bIsHidden ? 'H' : ' '), (char)(pWord->Body.bIsQuarantine ? 'Q' : ' '), (char)(pWord->Body.bIsRetain ? 'R' : ' ')); for (wIndex = 0; wIndex < pWord->Base.wWordLen; ++wIndex) { ET9SYMB sSymb = pWord->Base.sWord[wIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F || sSymb >= 0xA1 && sSymb <= 0xFF) { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } if (pWord->Base.wSubstitutionLen) { fprintf(pF, " -> "); for (wIndex = 0; wIndex < pWord->Base.wSubstitutionLen; ++wIndex) { ET9SYMB sSymb = pWord->Base.sSubstitution[wIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F || sSymb >= 0xA1 && sSymb <= 0xFF) { fprintf(pF, "%c", (unsigned char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } } #ifdef ET9_DEBUG { ET9INT snContextIndex; fprintf(pF, " "); for (snContextIndex = ET9NLM_CONTEXT_COUNT - 1; snContextIndex >= 0; --snContextIndex) { ET9SimpleWord const * const pContextWord = (GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM) ? &pWord->Body.sScoreContext.pDLMContextWords[snContextIndex] : &pWord->Body.sScoreContext.pReverseContextWords[snContextIndex]; ET9UINT nIndex; if (!pContextWord->wLen) { fprintf(pF, "\xB0"); } for (nIndex = 0; nIndex < pContextWord->wLen; ++nIndex) { const ET9SYMB sSymb = pContextWord->sString[nIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F || sSymb >= 0xA1 && sSymb <= 0xFF) { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } if (snContextIndex > 0) { fprintf(pF, " "); } } fprintf(pF, " "); } #endif fprintf(pF, "\n"); fflush(pF); #ifdef ET9_DEBUG { ET9U16 wCount = pWord->Base.wWordLen; ET9SYMB const *pSymb = pWord->Base.sWord; ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); ET9AssertLog(pWord->Base.wWordCompLen <= pWord->Base.wWordLen || pWord->Base.wWordCompLen == 0xFFFF); ET9AssertLog(pWord->Base.wSubstitutionLen <= ET9MAXSUBSTITUTIONSIZE); while (wCount--) { ET9AssertLog(*pSymb && *pSymb != (ET9SYMB)0xcccc); ++pSymb; } } #endif } #else /* ET9_DEBUGLOG2 */ #define WLOG2Word(pF,pcComment,pWord) #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL WLOG2Captures(FILE *pF, char *pcComment, ET9AWLingInfo *pLingInfo) { ET9U16 wIndex; ET9U16 wSymbolIndex; ET9U16 wCapture; ET9U16 wMaxCaptureLen = 0; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9AssertLog(pF != NULL); ET9AssertLog(pcComment != NULL); fprintf(pF, "%s (captures log)\n", pcComment); for (wCapture = 0; wCapture < ET9MAXBUILDCAPTURES; ++wCapture) { if (pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].bIsValid) { if (pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wWordLen > wMaxCaptureLen) { wMaxCaptureLen = pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wWordLen; } } } for (wCapture = 0; wCapture < ET9MAXBUILDCAPTURES; ++wCapture) { fprintf(pF, " Capture %d, bIsValid = %d %s ", wCapture, pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].bIsValid, (char*)(wCapture == pLingCmnInfo->Private.sBuildInfo.bCurrCapture ? "<*>" : " ")); if (pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].bIsValid) { ET9U8 bCompChar = 0; for (wIndex = 0; wIndex < pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wWordLen; ++wIndex) { ET9SYMB sSymb = pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].sWord[wIndex]; if (wIndex == (pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wWordLen - pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wWordCompLen)) { fprintf(pF, "|"); bCompChar = 1; } if (sSymb >= 0x20 && sSymb <= 0x7F && sSymb != '|') { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } for (; wIndex < wMaxCaptureLen + 3 - bCompChar; ++wIndex) { fprintf(pF, " "); } fprintf(pF, " (len %2d, comp len %2d, symb len %2d)", pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wWordLen, pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wWordCompLen, pLingCmnInfo->Private.sBuildInfo.pCaptures[wCapture].wSymbolLen); } fprintf(pF, "\n"); } fprintf(pF, " Capture actions log (%d symbs):", pWordSymbInfo->bNumSymbs); for (wIndex = 0; wIndex <= pWordSymbInfo->bNumSymbs; ++wIndex) { if (pLingCmnInfo->Private.sBuildInfo.pCaptureActions[wIndex].bPop) { fprintf(pF, " POP"); } else if (!pLingCmnInfo->Private.sBuildInfo.pCaptureActions[wIndex].sbAddWordLen && !pLingCmnInfo->Private.sBuildInfo.pCaptureActions[wIndex].sbAddWordCompLen && !pLingCmnInfo->Private.sBuildInfo.pCaptureActions[wIndex].bAddSymbolLen) { fprintf(pF, " -"); } else { fprintf(pF, " (%d,%d,%d)", pLingCmnInfo->Private.sBuildInfo.pCaptureActions[wIndex].sbAddWordLen, pLingCmnInfo->Private.sBuildInfo.pCaptureActions[wIndex].sbAddWordCompLen, pLingCmnInfo->Private.sBuildInfo.pCaptureActions[wIndex].bAddSymbolLen); } if (wIndex % 5 == 4) { fprintf(pF, " ."); } } fprintf(pF, "\n"); fprintf(pF, " Capture locked symbs: "); for (wIndex = 0; wIndex < ET9MAXWORDSIZE; ++wIndex) { const ET9SYMB sSymb = pWordSymbInfo->SymbsInfo[wIndex].sLockedSymb; if (!sSymb) { break; } if (sSymb >= 0x20 && sSymb <= 0x7F && sSymb != '|') { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } fprintf(pF, "\n"); fprintf(pF, " Capture flushed symbs: "); for (wIndex = 0; wIndex < ET9MAXWORDSIZE; ++wIndex) { const ET9SYMB sSymb = pLingCmnInfo->Private.sBuildInfo.psFlushedSymbs[wIndex]; if (!sSymb) { break; } if (sSymb >= 0x20 && sSymb <= 0x7F && sSymb != '|') { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } fprintf(pF, "\n"); fprintf(pF, " Capture flush pos log (%d symbs): ", pWordSymbInfo->bNumSymbs); for (wIndex = 0; wIndex <= pWordSymbInfo->bNumSymbs; ++wIndex) { fprintf(pF, "%2d ", pLingCmnInfo->Private.sBuildInfo.pbFlushPos[wIndex]); if (wIndex % 5 == 4) { fprintf(pF, ". "); } } fprintf(pF, "\n"); fprintf(pF, " Capture flush len log (%d symbs): ", pWordSymbInfo->bNumSymbs); for (wIndex = 0; wIndex <= pWordSymbInfo->bNumSymbs; ++wIndex) { fprintf(pF, "%2d ", pLingCmnInfo->Private.sBuildInfo.pbFlushLen[wIndex]); if (wIndex % 5 == 4) { fprintf(pF, ". "); } } fprintf(pF, "\n"); for (wSymbolIndex = 0; wSymbolIndex < pWordSymbInfo->bNumSymbs; ++wSymbolIndex) { fprintf(pF, " Capture default %02d: %2d %2d (%4x) ", wSymbolIndex, pLingCmnInfo->Private.sBuildInfo.pbDefaultLen[wSymbolIndex], pLingCmnInfo->Private.sBuildInfo.pbDefaultCompLen[wSymbolIndex], pLingCmnInfo->Private.sBuildInfo.pwDefaultPos[wSymbolIndex]); if (pLingCmnInfo->Private.sBuildInfo.pwDefaultPos[wSymbolIndex] == UNDEFINED_STRING_INDEX) { fprintf(pF, "\n"); continue; } for (wIndex = 0; wIndex < pLingCmnInfo->Private.sBuildInfo.pbDefaultLen[wSymbolIndex]; ++wIndex) { ET9SYMB sSymb = pLingCmnInfo->Private.sBuildInfo.psDefaultSymbs[pLingCmnInfo->Private.sBuildInfo.pwDefaultPos[wSymbolIndex] + wIndex]; if (wIndex == (pLingCmnInfo->Private.sBuildInfo.pbDefaultLen[wSymbolIndex] - pLingCmnInfo->Private.sBuildInfo.pbDefaultCompLen[wSymbolIndex])) { fprintf(pF, "|"); } if (sSymb >= 0x20 && sSymb <= 0x7F && sSymb != '|') { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } fprintf(pF, "\n"); } fflush(pF); } #else /* ET9_DEBUGLOG2 */ #define WLOG2Captures(pF,pcComment,pLingInfo) #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL WLOG2Shrink(FILE *pF) { static long nShrinkCount = 0; static long long nRemovedLines = 0; const long nThreshold = 5000000; const long nKeepSize = nThreshold / 2; if (!pF || ftell(pF) < nThreshold) { return; } { char cLine[2048]; long nSize = ftell(pF); long nStart = nSize - nKeepSize; FILE *pfTmp = tmpfile(); if (!pfTmp) { fprintf(pF, "...Log file shrink failed, couldn't create tmp file\n\n"); return; } rewind(pF); while (!feof(pF) && ftell(pF) < nSize) { fgets(cLine, 2048, pF); if (ftell(pF) < nStart) { ++nRemovedLines; } else { fputs(cLine, pfTmp); } } rewind(pF); while (ftell(pF) < nSize) { fputs("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", pF); } rewind(pF); rewind(pfTmp); ++nShrinkCount; fprintf(pF, "...Total removed lines = %I64d, shrink count = %ld\n\n", nRemovedLines, nShrinkCount); while (!feof(pfTmp)) { fgets(cLine, 2048, pfTmp); fputs(cLine, pF); } fprintf(pF, "\n\n...Start point after shrink, shrink count = %ld\n\n", nShrinkCount); fflush(pF); fclose(pfTmp); } } #else /* ET9_DEBUGLOG2 */ #define WLOG2Shrink(pF) #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __LogUDB(ET9AWLingInfo * const pLingInfo, FILE *pF) { #if 1 ET9AWRUDBInfo ET9FARDATA * const pRUDB = pLingInfo->pLingCmnInfo->pRUDBInfo; ET9U8 ET9FARDATA *pRUDBCurrent; ET9U8 ET9FARDATA *pRUDBEnd; if (!pRUDB) { fprintf(pF, "\nNo UDB\n\n"); return; } fprintf(pF, "\nUDB, size %u, remaining %u, RDB words %u, UDB words %u\n\n", pRUDB->wDataSize, pRUDB->wRemainingMemory, pRUDB->wRDBWordCount, pRUDB->wUDBWordCount); pRUDBCurrent = ET9RUDBData(pRUDB) + pRUDB->wSizeOffset[0]; pRUDBEnd = pRUDBCurrent; pRUDBCurrent = _ET9AWMoveRUDBPtrRight(pRUDB, pRUDBCurrent, 1); while (pRUDBCurrent != pRUDBEnd) { const ET9UINT nCurRecType = _ET9AWGetRecordType(pRUDBCurrent); if (nCurRecType == ET9FREETYPE) { pRUDBCurrent = _ET9AWMoveRUDBPtrRight(pRUDB, pRUDBCurrent, _ET9AWGetRecordLength(pRUDB, pRUDBCurrent)); if (pRUDBCurrent == pRUDBEnd) { break; } } else { ET9UINT i; ET9U16 wWordSize; ET9U8 bLang; ET9U16 wFreq; ET9U16 wSaveFreq; fprintf(pF, " "); if (nCurRecType == ET9UDBTYPE) { bLang = 0; wWordSize = _ET9AWGetUDBWordLen(pRUDBCurrent); wFreq = (ET9U16)_ET9AWRUDBReadWord(pRUDB, _ET9AWMoveRUDBPtrRight(pRUDB, pRUDBCurrent, (ET9UINT)1)); wSaveFreq = wFreq; pRUDBCurrent = _ET9AWMoveRUDBPtrRight(pRUDB, pRUDBCurrent, UDB_RECORD_HEADER_SIZE); if (wFreq > ET9MAX_FREQ_COUNT) { fprintf(pF, "QUDB "); } else { fprintf(pF, "UDB "); wFreq = 0; } } else { wFreq = 0; bLang = (ET9U8)*(_ET9AWMoveRUDBPtrRight(pRUDB, pRUDBCurrent, (RDB_RECORD_HEADER_SIZE - 1))); wWordSize = _ET9AWGetRDBWordLen(pRUDBCurrent); wSaveFreq = (ET9U16)_ET9AWRUDBReadWord(pRUDB, _ET9AWMoveRUDBPtrRight(pRUDB, pRUDBCurrent, (ET9UINT)1)); pRUDBCurrent = _ET9AWMoveRUDBPtrRight(pRUDB, pRUDBCurrent, RDB_RECORD_HEADER_SIZE); fprintf(pF, "RDB "); } if (bLang) { fprintf(pF, " 0x%02X %5d ", bLang, wSaveFreq); } else if (wFreq) { fprintf(pF, " %2u %5d ", (wFreq - ET9MAX_FREQ_COUNT), wSaveFreq); } else { fprintf(pF, " %5d ", wSaveFreq); } { ET9UINT nLenUsed = 0; for (i = 0; i < wWordSize; ++i) { const ET9SYMB sSymb = (ET9SYMB)_ET9AWRUDBReadWord(pRUDB, (const ET9U8 ET9FARDATA *)pRUDBCurrent); pRUDBCurrent = _ET9AWMoveRUDBPtrRight(pRUDB, (const ET9U8 ET9FARDATA *)pRUDBCurrent, ET9SYMBOLWIDTH); if (sSymb >= 0x20 && sSymb <= 0xFF) { nLenUsed += fprintf(pF, "%c", (unsigned char)sSymb); } else { nLenUsed += fprintf(pF, "<%x>", (int)sSymb); } } } fprintf(pF, "\n"); } } fprintf(pF, "\n"); fflush(pF); #else ET9U16 wId = 0; ET9U16 wIndex; ET9U16 wCount; ET9U16 wBufLen; ET9SYMB psBuf[ET9MAXUDBWORDSIZE]; wBufLen = 0; if (ET9AWUDBGetWordCount(pLingInfo, &wCount)) { fprintf(pF, "no UDB active\n"); return; } fprintf(pF, "%d UDB entries\n", wCount); while (!ET9AWUDBGetWord(pLingInfo, psBuf, ET9MAXWORDSIZE, &wBufLen, 1)) { fprintf(pF, "UDB %03d: ", wId++); for (wIndex = 0; wIndex < wBufLen; ++wIndex) { ET9SYMB sSymb = psBuf[wIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F) { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } fprintf(pF, "\n"); } fprintf(pF, "\n"); fflush(pF); #endif } #else /* ET9_DEBUGLOG2 */ #define __LogUDB(pLingInfo,pF) #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __LogCDB(ET9AWLingInfo * const pLingInfo, FILE *pF) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWCDBInfo ET9FARDATA * const pCDB = pLingCmnInfo->pCDBInfo; { ET9UINT nIndex; for (nIndex = 0; nIndex < ET9NLM_CONTEXT_COUNT; ++nIndex) { fprintf(pF, " Context Word[%u]: ", nIndex); if (!pLingCmnInfo->Private.pContextWords[nIndex].wLen) { fprintf(pF, "[NONE]"); } else { WLOG2String(pF, "", pLingCmnInfo->Private.pContextWords[nIndex].sString, pLingCmnInfo->Private.pContextWords[nIndex].wLen, 0, 1); } fprintf(pF, "\n"); } fprintf(pF, "\n"); } if (!pCDB) { fprintf(pF, "\nNo CDB\n\n"); return; } fprintf(pF, "\nCDB, size %u\n\n", pCDB->wDataSize); { ET9UINT nDelimiterCount; ET9INT snSearchPosition; ET9INT snStopPoint; ET9UINT nBufferLen; static ET9SYMB psBuffer[100000]; nBufferLen = 0; nDelimiterCount = 0; snStopPoint = 0; snSearchPosition = (ET9INT)(pCDB->wDataEndOffset - (ET9U16)1 + ET9CDBDataAreaSymbs(pCDB)) % ET9CDBDataAreaSymbs(pCDB); for (; snSearchPosition != pCDB->wDataEndOffset;) { const ET9SYMB sSymb = *(ET9CDBData(pCDB) + snSearchPosition); if (sSymb == CDBDELIMITER) { ++nDelimiterCount; } else { nDelimiterCount = 0; } if (nDelimiterCount == 2) { fprintf(pF, "<<>>"); WLOG2String(pF, "", psBuffer, nBufferLen, 1, 1); fprintf(pF, "\n"); nBufferLen = 0; } if (nDelimiterCount < 2) { psBuffer[nBufferLen++] = sSymb; } if (!snSearchPosition) { snStopPoint = (ET9INT)pCDB->wDataEndOffset; snSearchPosition = (ET9INT)ET9CDBDataAreaSymbs(pCDB) - 1; } else { --snSearchPosition; } } if (nBufferLen) { WLOG2String(pF, "", psBuffer, nBufferLen, 1, 1); fprintf(pF, "\n"); } fprintf(pF, "\n"); } fflush(pF); } #else /* ET9_DEBUGLOG2 */ #define __LogCDB(pLingInfo,pF) #endif /* ET9_DEBUGLOG2 */ /* ************************************************************************************************************** */ /* * ************************************************************************************************************ */ /* ************************************************************************************************************** */ #ifdef ET9_DEBUG /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * * */ static void ET9LOCALCALL __VerifyLockedSymbs(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9U16 wLockPoint; ET9SymbInfo *pSymbInfo; wLockPoint = pWordSymbInfo->bNumSymbs; if (wLockPoint) { for (pSymbInfo = &pWordSymbInfo->SymbsInfo[wLockPoint - 1]; wLockPoint; --wLockPoint, --pSymbInfo) { /* ignore invalid Insure++ warnings */ if (pSymbInfo->bLocked) { break; } } } if (wLockPoint && wLockPoint >= wIndex) { ET9UINT nCount; ET9SYMB *psSymb; ET9SymbInfo *pSymbInfo; nCount = wLockPoint - wIndex; if (nCount > wLength) { nCount = wLength; } if (nCount > pWord->Base.wWordLen) { nCount = pWord->Base.wWordLen; } psSymb = pWord->Base.sWord; pSymbInfo = &pWordSymbInfo->SymbsInfo[wIndex]; for (; nCount; --nCount, ++psSymb, ++pSymbInfo) { ET9SYMB sLockedSymb = pSymbInfo->sLockedSymb; ET9SYMB sLockedCnvSymb = pSymbInfo->sLockedSymb; if (pLingInfo->Private.pConvertSymb != NULL) { pLingInfo->Private.pConvertSymb(pLingInfo->Private.pConvertSymbInfo, &sLockedCnvSymb); } /* this is NOT the final lock verification, could be before convert or after */ if (*psSymb != sLockedSymb && *psSymb != sLockedCnvSymb) { WLOG2Word(pLogFile2, "word verified", pWord); WLOG2( fprintf(pLogFile2, "__VerifyLockedSymbs, missmatch index = %d, word symb = ", (psSymb - pWord->Base.sWord)); if (*psSymb >= 0x20 && *psSymb <= 0x7F) { fprintf(pLogFile2, "'%c'", *psSymb); } else { fprintf(pLogFile2, "<%x>", *psSymb); } fprintf(pLogFile2, ", locked symb = "); if (sLockedSymb >= 0x20 && sLockedSymb <= 0x7F) { fprintf(pLogFile2, "'%c'", sLockedSymb); } else { fprintf(pLogFile2, "<%x>", sLockedSymb); } fprintf(pLogFile2, ", locked cnv symb = "); if (sLockedCnvSymb >= 0x20 && sLockedCnvSymb <= 0x7F) { fprintf(pLogFile2, "'%c'", sLockedCnvSymb); } else { fprintf(pLogFile2, "<%x>", sLockedCnvSymb); } fprintf(pLogFile2, "\n"); ) } else { ET9AssertLog(*psSymb == sLockedSymb || *psSymb == sLockedCnvSymb); } } } } #else #define __VerifyLockedSymbs(pLingInfo, pWord, wIndex, wLength) #endif /* ET9_DEBUG */ /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __GetStemsAllowed(ET9AWLingCmnInfo const * const pLingCmnInfo, const ET9U16 wLength) { return (ET9WORDSTEMS_MODE(pLingCmnInfo) && wLength >= pLingCmnInfo->Private.wWordStemsPoint) ? 1 : 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * * * * */ ET9INLINE static ET9U16 ET9LOCALCALL __GetMaxWordLength(ET9AWLingCmnInfo const * const pLingCmnInfo, const ET9U16 wLength) { return (wLength && (pLingCmnInfo->Base.pWordSymbInfo->Private.bRequiredInhibitCorrection || !ET9WORDCOMPLETION_MODE(pLingCmnInfo) || (wLength < pLingCmnInfo->Private.wWordCompletionPoint))) ? wLength : ET9MAXWORDSIZE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __IsInvalidSingleLetterWord(ET9AWLingInfo const * const pLingInfo, const ET9U32 dwLdbNum, ET9SimpleWord const * const pSimpleWord) { return (ET9BOOL)(pSimpleWord->wLen == 1 && !_ET9_IsSingleCharWord(pLingInfo, dwLdbNum, pSimpleWord->sString[0])); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __IsUsingMixedStyle(ET9AWLingCmnInfo const * const pLingCmnInfo) { if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_MIXED || pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_SUFFIX) { return 1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * / * * */ void ET9FARCALL _ET9_MakeWordTerm(ET9AWLingInfo * const pLingInfo, _ET9AW_Collection * const pCollection, const ET9UINT nWordIndex) { ET9AWPrivWordInfo * const pIndexWord = &pCollection->pWordList[pCollection->pnWordList[nWordIndex]]; ET9AssertLog(!pIndexWord->Base.bIsTerm); ET9AssertLog(pLingInfo->pLingCmnInfo->Private.bDoneBuilding); WLOG2Word(pLogFile2, "_ET9_MakeWordTerm", pIndexWord); { const ET9U8 bCost = (ET9U8)pIndexWord->Base.wWordCompLen; pIndexWord->Body.bEditDistSpc = pIndexWord->Body.bEditDistSpc + bCost; pIndexWord->Body.bEditDistStem = pIndexWord->Body.bEditDistStem + bCost; pIndexWord->Body.bEditDistSpcIns = pIndexWord->Body.bEditDistSpcIns + bCost; pIndexWord->Base.bIsTerm = 1; pIndexWord->Base.wWordCompLen = 0; } ++pLingInfo->pLingCmnInfo->Private.__SPATH_Status.nSpmClearedCompletionCount; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static ET9INT ET9LOCALCALL __PriorityCompareNWP(ET9AWPrivWordInfo const * const pTestWord, ET9AWPrivWordInfo const * const pSrcWord) { const ET9WORDSRC nTestWordSrc = GETCOMPSRC(pTestWord->Body.bWordSrc); const ET9WORDSRC nSrcWordSrc = GETCOMPSRC(pSrcWord->Body.bWordSrc); ET9AssertLog(pTestWord != NULL); ET9AssertLog(pSrcWord != NULL); if (pTestWord->Body.bNLMOrder && !pSrcWord->Body.bNLMOrder) { return 1; } if (pSrcWord->Body.bNLMOrder && !pTestWord->Body.bNLMOrder) { return -1; } { const ET9WORDSRC nModTestWordSrc = (nTestWordSrc == ET9WORDSRC_DLM) ? ET9WORDSRC_LDB : nTestWordSrc; const ET9WORDSRC nModSrcWordSrc = (nSrcWordSrc == ET9WORDSRC_DLM) ? ET9WORDSRC_LDB : nSrcWordSrc; if (nModTestWordSrc < nModSrcWordSrc) { return 1; } if (nModSrcWordSrc < nModTestWordSrc) { return -1; } } if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } if (pTestWord->Body.xWordFreq > pSrcWord->Body.xWordFreq) { return 1; } if (pSrcWord->Body.xWordFreq > pTestWord->Body.xWordFreq) { return -1; } if (pTestWord->Body.bIsCDBTrigram && !pSrcWord->Body.bIsCDBTrigram) { return 1; } if (pSrcWord->Body.bIsCDBTrigram && !pTestWord->Body.bIsCDBTrigram) { return -1; } if (pTestWord->Base.wWordLen > pSrcWord->Base.wWordLen) { return 1; } if (pSrcWord->Base.wWordLen > pTestWord->Base.wWordLen) { return -1; } if (pTestWord->Body.xWordFreq > pSrcWord->Body.xWordFreq) { return 1; } if (pSrcWord->Body.xWordFreq > pTestWord->Body.xWordFreq) { return -1; } if (pTestWord->Body.dwWordIndex < pSrcWord->Body.dwWordIndex) { return 1; } if (pSrcWord->Body.dwWordIndex < pTestWord->Body.dwWordIndex) { return -1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9INLINE static ET9INT ET9LOCALCALL __PriorityCompareClassic(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pTestWord, ET9AWPrivWordInfo const * const pSrcWord, const __SortStage eSortStage) { ET9WORDSRC nTestWordSrc; ET9WORDSRC nSrcWordSrc; ET9AssertLog(pTestWord != NULL); ET9AssertLog(pSrcWord != NULL); nTestWordSrc = GETCOMPSRC(pTestWord->Body.bWordSrc); nSrcWordSrc = GETCOMPSRC(pSrcWord->Body.bWordSrc); if (nTestWordSrc == ET9WORDSRC_EXACT && nSrcWordSrc != ET9WORDSRC_EXACT) { return 1; } else if (nSrcWordSrc == ET9WORDSRC_EXACT && nTestWordSrc != ET9WORDSRC_EXACT) { return -1; } if (nTestWordSrc == ET9WORDSRC_EXACTISH && nSrcWordSrc != ET9WORDSRC_EXACTISH) { return 1; } else if (nSrcWordSrc == ET9WORDSRC_EXACTISH && nTestWordSrc != ET9WORDSRC_EXACTISH) { return -1; } if (nTestWordSrc == nSrcWordSrc && (nSrcWordSrc == ET9WORDSRC_EXACT || nSrcWordSrc == ET9WORDSRC_EXACTISH)) { nTestWordSrc = GETBASESRC(pTestWord->Body.bWordSrc); nSrcWordSrc = GETBASESRC(pSrcWord->Body.bWordSrc); } if (GETRAWSRC(nTestWordSrc) == ET9WORDSRC_DLM && (pTestWord->Body.bIsWeak || pTestWord->Body.bIsQuarantine || pTestWord->Body.bIsHidden)) { nTestWordSrc = ET9WORDSRC_QUDB; } if (GETRAWSRC(nSrcWordSrc) == ET9WORDSRC_DLM && (pSrcWord->Body.bIsWeak || pSrcWord->Body.bIsQuarantine || pSrcWord->Body.bIsHidden)) { nSrcWordSrc = ET9WORDSRC_QUDB; } if (!pTestWord->Base.wWordLen) { return -1; } if (!pSrcWord->Base.wWordLen) { return 1; } if (eSortStage) { if (pTestWord->Base.bIsTerm && !pSrcWord->Base.bIsTerm) { return 1; } if (pSrcWord->Base.bIsTerm && !pTestWord->Base.bIsTerm) { return -1; } } else { if (ISPUNCTSRC(nTestWordSrc) && !pSrcWord->Base.bIsTerm) { return 1; } if (ISPUNCTSRC(nSrcWordSrc) && !pTestWord->Base.bIsTerm) { return -1; } if (pTestWord->Base.bIsTerm && !pSrcWord->Base.bIsTerm) { if (!ISGENUINESRC(nTestWordSrc) && ISGENUINESRC(nSrcWordSrc)) { return -1; } else if (pSrcWord->Body.bEditDistSpc + pSrcWord->Base.wWordCompLen < pTestWord->Body.bEditDistSpc) { return -1; } else { return 1; } } if (pSrcWord->Base.bIsTerm && !pTestWord->Base.bIsTerm) { if (!ISGENUINESRC(nSrcWordSrc) && ISGENUINESRC(nTestWordSrc)) { return 1; } else if (pTestWord->Body.bEditDistSpc + pTestWord->Base.wWordCompLen < pSrcWord->Body.bEditDistSpc) { return 1; } else { return -1; } } } if (eSortStage != SS_Collecting) { const ET9U8 bLangIndexCMP = pLingCmnInfo->Private.bNonActiveIndex; if (pTestWord->Base.bLangIndex != bLangIndexCMP && pSrcWord->Base.bLangIndex == bLangIndexCMP) { return 1; } if (pSrcWord->Base.bLangIndex != bLangIndexCMP && pTestWord->Base.bLangIndex == bLangIndexCMP) { return -1; } } if (nTestWordSrc < nSrcWordSrc) { return 1; } if (nSrcWordSrc < nTestWordSrc) { return -1; } if (pTestWord->Body.bIsCDBTrigram && !pSrcWord->Body.bIsCDBTrigram) { return 1; } if (pSrcWord->Body.bIsCDBTrigram && !pTestWord->Body.bIsCDBTrigram) { return -1; } if (pTestWord->Body.bEditDistSpc < pSrcWord->Body.bEditDistSpc) { return 1; } if (pSrcWord->Body.bEditDistSpc < pTestWord->Body.bEditDistSpc) { return -1; } if (pTestWord->Body.bEditDistStem < pSrcWord->Body.bEditDistStem && pTestWord->Body.bEditDistSpc && pSrcWord->Body.bEditDistSpc) { return 1; } if (pSrcWord->Body.bEditDistStem < pTestWord->Body.bEditDistStem && pTestWord->Body.bEditDistSpc && pSrcWord->Body.bEditDistSpc) { return -1; } if (eSortStage) { if (pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs <= ET9AWLDB_MAX_COLLECTION_TAG_LEN) { if (pTestWord->Body.bCollectionPrio > pSrcWord->Body.bCollectionPrio) { return 1; } if (pSrcWord->Body.bCollectionPrio > pTestWord->Body.bCollectionPrio) { return -1; } } } if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } if (pTestWord->Body.xWordFreq > pSrcWord->Body.xWordFreq) { return 1; } if (pSrcWord->Body.xWordFreq > pTestWord->Body.xWordFreq) { return -1; } if (pTestWord->Body.wTWordFreq + pTestWord->Body.wEWordFreq > pSrcWord->Body.wTWordFreq + pSrcWord->Body.wEWordFreq) { return 1; } if (pSrcWord->Body.wTWordFreq + pSrcWord->Body.wEWordFreq > pTestWord->Body.wTWordFreq + pTestWord->Body.wEWordFreq) { return -1; } if (pTestWord->Body.dwWordIndex < pSrcWord->Body.dwWordIndex) { return 1; } if (pSrcWord->Body.dwWordIndex < pTestWord->Body.dwWordIndex) { return -1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9INLINE static ET9INT ET9LOCALCALL __PriorityCompareMixed(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pTestWord, ET9AWPrivWordInfo const * const pSrcWord, const __SortStage eSortStage) { ET9WORDSRC nTestWordSrc; ET9WORDSRC nSrcWordSrc; ET9AssertLog(pTestWord != NULL); ET9AssertLog(pSrcWord != NULL); nTestWordSrc = GETCOMPSRC(pTestWord->Body.bWordSrc); nSrcWordSrc = GETCOMPSRC(pSrcWord->Body.bWordSrc); if (nTestWordSrc == ET9WORDSRC_EXACT && nSrcWordSrc != ET9WORDSRC_EXACT) { return 1; } else if (nSrcWordSrc == ET9WORDSRC_EXACT && nTestWordSrc != ET9WORDSRC_EXACT) { return -1; } if (nTestWordSrc == ET9WORDSRC_EXACTISH && nSrcWordSrc != ET9WORDSRC_EXACTISH) { return 1; } else if (nSrcWordSrc == ET9WORDSRC_EXACTISH && nTestWordSrc != ET9WORDSRC_EXACTISH) { return -1; } if (nTestWordSrc == nSrcWordSrc && (nSrcWordSrc == ET9WORDSRC_EXACT || nSrcWordSrc == ET9WORDSRC_EXACTISH)) { nTestWordSrc = GETBASESRC(pTestWord->Body.bWordSrc); nSrcWordSrc = GETBASESRC(pSrcWord->Body.bWordSrc); } if (eSortStage) { } else { if (ISPUNCTSRC(nTestWordSrc) && !pSrcWord->Base.bIsTerm) { return 1; } if (ISPUNCTSRC(nSrcWordSrc) && !pTestWord->Base.bIsTerm) { return -1; } } if (eSortStage != SS_Collecting) { const ET9U8 bLangIndexCMP = pLingCmnInfo->Private.bNonActiveIndex; if (pTestWord->Base.bLangIndex != bLangIndexCMP && pSrcWord->Base.bLangIndex == bLangIndexCMP) { return 1; } if (pSrcWord->Base.bLangIndex != bLangIndexCMP && pTestWord->Base.bLangIndex == bLangIndexCMP) { return -1; } } { const ET9INT snTestBuildable = ISBUILDABLESRC(nTestWordSrc); const ET9INT snSrcBuildable = ISBUILDABLESRC(nSrcWordSrc); if (snTestBuildable && !snSrcBuildable) { return 1; } if (snSrcBuildable && !snTestBuildable) { return -1; } if (!(snTestBuildable && snSrcBuildable)) { if (nTestWordSrc < nSrcWordSrc) { return 1; } if (nSrcWordSrc < nTestWordSrc) { return -1; } } } if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } if (pTestWord->Body.xSymbDist < pSrcWord->Body.xSymbDist) { return 1; } if (pSrcWord->Body.xSymbDist < pTestWord->Body.xSymbDist) { return -1; } if (eSortStage >= SS_Collecting) { if (pTestWord->Body.bCollectionPrio > pSrcWord->Body.bCollectionPrio) { return 1; } if (pSrcWord->Body.bCollectionPrio > pTestWord->Body.bCollectionPrio) { return -1; } } if (pTestWord->Body.xWordFreq > pSrcWord->Body.xWordFreq) { return 1; } if (pSrcWord->Body.xWordFreq > pTestWord->Body.xWordFreq) { return -1; } if (pTestWord->Body.wTWordFreq + pTestWord->Body.wEWordFreq > pSrcWord->Body.wTWordFreq + pSrcWord->Body.wEWordFreq) { return 1; } if (pSrcWord->Body.wTWordFreq + pSrcWord->Body.wEWordFreq > pTestWord->Body.wTWordFreq + pTestWord->Body.wEWordFreq) { return -1; } if (pTestWord->Base.bIsTerm && !pSrcWord->Base.bIsTerm) { return 1; } if (pSrcWord->Base.bIsTerm && !pTestWord->Base.bIsTerm) { return -1; } #if 0 if (pTestWord->Body.bEditDistFree < pSrcWord->Body.bEditDistFree) { return 1; } if (pSrcWord->Body.bEditDistFree < pTestWord->Body.bEditDistFree) { return -1; } #endif if (pTestWord->Body.dwWordIndex < pSrcWord->Body.dwWordIndex) { return 1; } if (pSrcWord->Body.dwWordIndex < pTestWord->Body.dwWordIndex) { return -1; } if (nTestWordSrc < nSrcWordSrc) { return 1; } if (nSrcWordSrc < nTestWordSrc) { return -1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9INLINE static ET9INT ET9LOCALCALL __PriorityCompareCompletionsPromoted(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pTestWord, ET9AWPrivWordInfo const * const pSrcWord, const __SortStage eSortStage) { ET9WORDSRC nTestWordSrc; ET9WORDSRC nSrcWordSrc; ET9AssertLog(pTestWord != NULL); ET9AssertLog(pSrcWord != NULL); nTestWordSrc = GETCOMPSRC(pTestWord->Body.bWordSrc); nSrcWordSrc = GETCOMPSRC(pSrcWord->Body.bWordSrc); if (nTestWordSrc == ET9WORDSRC_EXACT && nSrcWordSrc != ET9WORDSRC_EXACT) { return 1; } else if (nSrcWordSrc == ET9WORDSRC_EXACT && nTestWordSrc != ET9WORDSRC_EXACT) { return -1; } if (nTestWordSrc == ET9WORDSRC_EXACTISH && nSrcWordSrc != ET9WORDSRC_EXACTISH) { return 1; } else if (nSrcWordSrc == ET9WORDSRC_EXACTISH && nTestWordSrc != ET9WORDSRC_EXACTISH) { return -1; } if (nTestWordSrc == nSrcWordSrc && (nSrcWordSrc == ET9WORDSRC_EXACT || nSrcWordSrc == ET9WORDSRC_EXACTISH)) { nTestWordSrc = GETBASESRC(pTestWord->Body.bWordSrc); nSrcWordSrc = GETBASESRC(pSrcWord->Body.bWordSrc); } if (GETRAWSRC(nTestWordSrc) == ET9WORDSRC_DLM && (pTestWord->Body.bIsWeak || pTestWord->Body.bIsQuarantine || pTestWord->Body.bIsHidden)) { nTestWordSrc = ET9WORDSRC_QUDB; } if (GETRAWSRC(nSrcWordSrc) == ET9WORDSRC_DLM && (pSrcWord->Body.bIsWeak || pSrcWord->Body.bIsQuarantine || pSrcWord->Body.bIsHidden)) { nSrcWordSrc = ET9WORDSRC_QUDB; } if (!pTestWord->Base.wWordLen) { return -1; } if (!pSrcWord->Base.wWordLen) { return 1; } { const ET9U8 bTstEditDistStem = (ET9U8)(pTestWord->Body.bEditDistStem - ((pTestWord->Base.bIsTerm && pTestWord->Body.bIsTop5 && pTestWord->Body.bEditDistStem == 1 && (pTestWord->Body.bEditDistSpc - pTestWord->Body.bEditDistSpcTrp) == 0) ? 1 : 0)); const ET9U8 bSrcEditDistStem = (ET9U8)(pSrcWord->Body.bEditDistStem - ((pSrcWord->Base.bIsTerm && pSrcWord->Body.bIsTop5 && pSrcWord->Body.bEditDistStem == 1 && (pSrcWord->Body.bEditDistSpc - pSrcWord->Body.bEditDistSpcTrp) == 0) ? 1 : 0)); if (bTstEditDistStem < bSrcEditDistStem) { return 1; } if (bSrcEditDistStem < bTstEditDistStem) { return -1; } } if (eSortStage) { if (pTestWord->Base.bIsTerm && !pSrcWord->Base.bIsTerm) { return 1; } if (pSrcWord->Base.bIsTerm && !pTestWord->Base.bIsTerm) { return -1; } } else { if (ISPUNCTSRC(nTestWordSrc) && !pSrcWord->Base.bIsTerm) { return 1; } if (ISPUNCTSRC(nSrcWordSrc) && !pTestWord->Base.bIsTerm) { return -1; } if (pTestWord->Base.bIsTerm && !pSrcWord->Base.bIsTerm) { if (!ISGENUINESRC(nTestWordSrc) && ISGENUINESRC(nSrcWordSrc)) { return -1; } else if (pSrcWord->Body.bEditDistSpc + pSrcWord->Base.wWordCompLen < pTestWord->Body.bEditDistSpc) { return -1; } else { return 1; } } if (pSrcWord->Base.bIsTerm && !pTestWord->Base.bIsTerm) { if (!ISGENUINESRC(nSrcWordSrc) && ISGENUINESRC(nTestWordSrc)) { return 1; } else if (pTestWord->Body.bEditDistSpc + pTestWord->Base.wWordCompLen < pSrcWord->Body.bEditDistSpc) { return 1; } else { return -1; } } } /* repeat stem distance check to separate what we merged above */ if (pTestWord->Body.bEditDistStem < pSrcWord->Body.bEditDistStem) { return 1; } if (pSrcWord->Body.bEditDistStem < pTestWord->Body.bEditDistStem) { return -1; } if (eSortStage != SS_Collecting) { const ET9U8 bLangIndexCMP = pLingCmnInfo->Private.bNonActiveIndex; if (pTestWord->Base.bLangIndex != bLangIndexCMP && pSrcWord->Base.bLangIndex == bLangIndexCMP) { return 1; } if (pSrcWord->Base.bLangIndex != bLangIndexCMP && pTestWord->Base.bLangIndex == bLangIndexCMP) { return -1; } } if (nTestWordSrc < nSrcWordSrc) { return 1; } if (nSrcWordSrc < nTestWordSrc) { return -1; } if (pTestWord->Body.bIsCDBTrigram && !pSrcWord->Body.bIsCDBTrigram) { return 1; } if (pSrcWord->Body.bIsCDBTrigram && !pTestWord->Body.bIsCDBTrigram) { return -1; } { const ET9U8 bTstEditDistSpc = (ET9U8)(pTestWord->Body.bEditDistSpc - pTestWord->Body.bEditDistSpcTrp); const ET9U8 bSrcEditDistSpc = (ET9U8)(pSrcWord->Body.bEditDistSpc - pSrcWord->Body.bEditDistSpcTrp); if (bTstEditDistSpc < bSrcEditDistSpc) { return 1; } if (bSrcEditDistSpc < bTstEditDistSpc) { return -1; } } if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } if (pTestWord->Body.xWordFreq > pSrcWord->Body.xWordFreq) { return 1; } if (pSrcWord->Body.xWordFreq > pTestWord->Body.xWordFreq) { return -1; } if (pTestWord->Body.wTWordFreq + pTestWord->Body.wEWordFreq > pSrcWord->Body.wTWordFreq + pSrcWord->Body.wEWordFreq) { return 1; } if (pSrcWord->Body.wTWordFreq + pSrcWord->Body.wEWordFreq > pTestWord->Body.wTWordFreq + pTestWord->Body.wEWordFreq) { return -1; } if (pTestWord->Body.dwWordIndex < pSrcWord->Body.dwWordIndex) { return 1; } if (pSrcWord->Body.dwWordIndex < pTestWord->Body.dwWordIndex) { return -1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9INLINE static ET9INT ET9LOCALCALL __PriorityCompareSuffix(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pTestWord, ET9AWPrivWordInfo const * const pSrcWord, const __SortStage eSortStage) { ET9_UNUSED(pLingCmnInfo); ET9_UNUSED(eSortStage); ET9AssertLog(pTestWord); ET9AssertLog(pSrcWord); if (pTestWord->Body.snCorrectionCost < pSrcWord->Body.snCorrectionCost) { return 1; } if (pSrcWord->Body.snCorrectionCost < pTestWord->Body.snCorrectionCost) { return -1; } #ifdef ET9_MGD_MODULE if (pTestWord->Body.bBestMatchMin > pSrcWord->Body.bBestMatchMin) { return 1; } if (pSrcWord->Body.bBestMatchMin > pTestWord->Body.bBestMatchMin) { return -1; } #endif if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } if (pTestWord->Body.dwWordIndex < pSrcWord->Body.dwWordIndex) { return 1; } if (pSrcWord->Body.dwWordIndex < pTestWord->Body.dwWordIndex) { return -1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static ET9INT ET9LOCALCALL __PriorityCompare(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pTestWord, ET9AWPrivWordInfo const * const pSrcWord, const __SortStage eSortStage) { ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9U8 bNumSymbs = pWordSymbInfo->bNumSymbs; ET9AssertLog(pLingCmnInfo != NULL); ET9AssertLog(pTestWord != NULL); ET9AssertLog(pSrcWord != NULL); /* rules shared by all prio functions */ /* limited */ if (pLingCmnInfo->Private.bDemoteLimitedRegionality && (pTestWord->Body.bLimitedCount || pSrcWord->Body.bLimitedCount)) { if (pTestWord->Body.bLimitedCount && !pSrcWord->Body.bLimitedCount) { return -1; } if (!pTestWord->Body.bLimitedCount && pSrcWord->Body.bLimitedCount) { return +1; } } /* punct over non punct len 1 */ if (bNumSymbs == 1 && pWordSymbInfo->SymbsInfo->bSymbType == ET9KTSMARTPUNCT) { if (ISPUNCTSRC(pTestWord->Body.bWordSrc) && !ISPUNCTSRC(pSrcWord->Body.bWordSrc) && !ISEXACTSRC(pSrcWord->Body.bWordSrc)) { return 1; } if (ISPUNCTSRC(pSrcWord->Body.bWordSrc) && !ISPUNCTSRC(pTestWord->Body.bWordSrc) && !ISEXACTSRC(pTestWord->Body.bWordSrc)) { return -1; } } /* disposable words are last of all */ if (ISDISPOSEWRD(pTestWord) || ISDISPOSEWRD(pSrcWord)) { if (!ISDISPOSEWRD(pTestWord) && ISDISPOSEWRD(pSrcWord)) { return 1; } if (!ISDISPOSEWRD(pSrcWord) && ISDISPOSEWRD(pTestWord)) { return -1; } if (ISDISPOSEWRD(pTestWord) && ISDISPOSEWRD(pSrcWord)) { const ET9WORDSRC nTestWordSrc = GETCOMPSRC(pTestWord->Body.bWordSrc); const ET9WORDSRC nSrcWordSrc = GETCOMPSRC(pSrcWord->Body.bWordSrc); if (nTestWordSrc < nSrcWordSrc) { return 1; } if (nSrcWordSrc < nTestWordSrc) { return -1; } if (pTestWord->Body.bWordQuality > pSrcWord->Body.bWordQuality) { return 1; } if (pSrcWord->Body.bWordQuality > pTestWord->Body.bWordQuality) { return -1; } if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } } return 0; } /* protected words should always be last (in a real list, thus before disposable's) */ if (ISPROTECTEDWRD(pTestWord) || ISPROTECTEDWRD(pSrcWord)) { if (!ISPROTECTEDWRD(pTestWord) && ISPROTECTEDWRD(pSrcWord)) { return 1; } if (!ISPROTECTEDWRD(pSrcWord) && ISPROTECTEDWRD(pTestWord)) { return -1; } if (ISPROTECTEDWRD(pTestWord) && ISPROTECTEDWRD(pSrcWord)) { const ET9WORDSRC nTestWordSrc = GETCOMPSRC(pTestWord->Body.bWordSrc); const ET9WORDSRC nSrcWordSrc = GETCOMPSRC(pSrcWord->Body.bWordSrc); if (nTestWordSrc < nSrcWordSrc) { return 1; } if (nSrcWordSrc < nTestWordSrc) { return -1; } if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } if (pTestWord->Body.dwWordIndex < pSrcWord->Body.dwWordIndex) { return 1; } if (pSrcWord->Body.dwWordIndex < pTestWord->Body.dwWordIndex) { return -1; } } return 0; } /* pool words (stems etc) is lower than all more "real" stuff */ if (ISSPOTSRC(pTestWord->Body.bWordSrc) || ISSPOTSRC(pSrcWord->Body.bWordSrc)) { if (!ISSPOTSRC(pTestWord->Body.bWordSrc) && ISSPOTSRC(pSrcWord->Body.bWordSrc)) { return 1; } if (!ISSPOTSRC(pSrcWord->Body.bWordSrc) && ISSPOTSRC(pTestWord->Body.bWordSrc)) { return -1; } if (ISSPOTSRC(pTestWord->Body.bWordSrc) && ISSPOTSRC(pSrcWord->Body.bWordSrc)) { const ET9WORDSRC nTestWordSrc = GETCOMPSRC(pTestWord->Body.bWordSrc); const ET9WORDSRC nSrcWordSrc = GETCOMPSRC(pSrcWord->Body.bWordSrc); if (nTestWordSrc < nSrcWordSrc) { return 1; } if (nSrcWordSrc < nTestWordSrc) { return -1; } if (pTestWord->Body.bWordQuality > pSrcWord->Body.bWordQuality) { return 1; } if (pSrcWord->Body.bWordQuality > pTestWord->Body.bWordQuality) { return -1; } if (pTestWord->Body.xTotFreq > pSrcWord->Body.xTotFreq) { return 1; } if (pSrcWord->Body.xTotFreq > pTestWord->Body.xTotFreq) { return -1; } } return 0; } /* apply current prio function rules */ if (!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { return __PriorityCompareNWP(pTestWord, pSrcWord); } else if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_MIXED) { return __PriorityCompareMixed(pLingCmnInfo, pTestWord, pSrcWord, eSortStage); } else if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_CLASSIC) { return __PriorityCompareClassic(pLingCmnInfo, pTestWord, pSrcWord, eSortStage); } else if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_COMPLETIONSPROMOTED) { return __PriorityCompareCompletionsPromoted(pLingCmnInfo, pTestWord, pSrcWord, eSortStage); } else if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_SUFFIX) { return __PriorityCompareSuffix(pLingCmnInfo, pTestWord, pSrcWord, eSortStage); } else { ET9AssertLog(0); return 0; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9UINT ET9LOCALCALL __ET9AWIsDiscreteInput(ET9AWLingInfo * const pLingInfo) { ET9U16 wLen; ET9SymbInfo *pSymbInfo; ET9AssertLog(pLingInfo != NULL); wLen = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs; pSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo; for (; wLen ; --wLen, ++pSymbInfo) { if (pSymbInfo->eInputType != ET9DISCRETEKEY) { return 0; } } return 1; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9UINT ET9LOCALCALL __ET9AWIsLikeExactMatch(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo const * const pWord, const ET9BOOL bAllowSecondaryLanguage) { ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; const ET9U8 bNumSymbs = pWordSymbInfo->bNumSymbs; ET9U8 bFreeCount; ET9AssertLog(pLingInfo); ET9AssertLog(pWord); /* not "TERM" */ if (bNumSymbs && pWord->Body.bWordSrc == ET9WORDSRC_TERM) { const ET9SymbClass eClassWord = _ET9_GetSymbolClass(pWord->Base.sWord[0]); const ET9SymbClass eClassInput = _ET9_GetSymbolClass(pWordSymbInfo->SymbsInfo[bNumSymbs - 1].DataPerBaseSym[0].sChar[0]); if (eClassWord != eClassInput) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - 'TERM'", pWord); return 0; } } /* potentially not secondary language */ if (!bAllowSecondaryLanguage) { const ET9U8 bLangIndexExclude = pLingInfo->pLingCmnInfo->Private.bNonActiveIndex; if (pWord->Base.bLangIndex == bLangIndexExclude) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - secondary language", pWord); return 0; } } /* count free symbs in word */ { ET9SYMB const *pSymb; ET9UINT nCount; bFreeCount = 0; pSymb = pWord->Base.sWord; for (nCount = pWord->Base.wWordLen; nCount; --nCount, ++pSymb) { if (_ET9_LanguageSpecific_FreeExactishSymb(*pSymb)) { ++bFreeCount; } } } /* keep cases separate for clarity */ if (!bNumSymbs) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - empty input", pWord); return 0; } if (pWord->Base.wWordLen != bNumSymbs && pWord->Base.wWordLen != bNumSymbs + bFreeCount) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - has length difference", pWord); return 0; } if (pWordSymbInfo->SymbsInfo[bNumSymbs - 1].bSymbType == ET9KTSMARTPUNCT) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - last input is terminal punctuation", pWord); return 0; } if (pWord->Body.bHasPrimEditDist) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - has prim edit distance", pWord); return 0; } if (!pWord->Base.bIsTerm || ISPUNCTSRC(pWord->Body.bWordSrc) || ISCOMPOUNDSRC(pWord->Body.bWordSrc) || ISBUILDAROUNDSRC(pWord->Body.bWordSrc) || !ISREALSRC(pWord->Body.bWordSrc)) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - stem, punct, compound, buildaround or 'not real'", pWord); return 0; } if (_ET9_IsNumericString(pWord->Base.sWord, pWord->Base.wWordLen)) { WLOG2BWord(pLogFile2, "..IsLikeExact (0) - is numeric string", pWord); return 0; } if (__ET9AWIsDiscreteInput(pLingInfo)) { WLOG2BWord(pLogFile2, "..IsLikeExact (1) - is discrete input", pWord); return 1; } { ET9UINT nCount; ET9SYMB const *psSymb; ET9SymbInfo *pSymbInfo; ET9U32 dwLdbNum; ET9UINT nRemovableFrees; nRemovableFrees = __ET9Min(bFreeCount, pWord->Base.wWordLen - bNumSymbs); psSymb = pWord->Base.sWord; pSymbInfo = pWordSymbInfo->SymbsInfo; dwLdbNum = (pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum; for (nCount = pWord->Base.wWordLen; nCount ; --nCount, ++psSymb) { if (pSymbInfo->eInputType == ET9DISCRETEKEY || pSymbInfo->eInputType == ET9MULTITAPKEY || pSymbInfo->eInputType == ET9CUSTOMSET) { ++pSymbInfo; continue; } { ET9UINT nSymCount; ET9UINT nFound; ET9SYMB sLower; ET9SYMB sUpper; ET9SYMB *psLowerAlts; ET9SYMB *psUpperAlts; nFound = 0; sUpper = _ET9SymToUpper(*psSymb, dwLdbNum); sLower = _ET9SymToLower(*psSymb, dwLdbNum); psLowerAlts = pSymbInfo->DataPerBaseSym->sChar; psUpperAlts = pSymbInfo->DataPerBaseSym->sUpperCaseChar; for (nSymCount = pSymbInfo->DataPerBaseSym->bNumSymsToMatch; nSymCount; --nSymCount, ++psLowerAlts, ++psUpperAlts) { if (sUpper == *psUpperAlts || sLower == *psLowerAlts) { nFound = 1; break; } } if (!nFound) { if (nRemovableFrees && _ET9_LanguageSpecific_FreeExactishSymb(*psSymb)) { --nRemovableFrees; WLOG2B(fprintf(pLogFile2, "..IsLikeExact, consumed removable apostrophe, pos = %d\n", (pSymbInfo - pWordSymbInfo->SymbsInfo));) } else { WLOG2B(fprintf(pLogFile2, "..IsLikeExact, fail pos = %d\n", (pSymbInfo - pWordSymbInfo->SymbsInfo));) WLOG2BWord(pLogFile2, "..IsLikeExact (0) - symbol not found", pWord); return 0; } } else if (nCount > nRemovableFrees) { ++pSymbInfo; } } } } WLOG2BWord(pLogFile2, "..IsLikeExact (1) - ok", pWord); return 1; } /* ************************************************************************************************************** */ /* * COLLECTION SET ORDER *************************************************************************************** */ /* ************************************************************************************************************** */ /* ============================================================================================================== */ typedef ET9BOOL ET9LOCALCALL __pWordListCOND(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord); static ET9BOOL ET9LOCALCALL __WordOrder_SpcCmplCOND(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { ET9_UNUSED(pLingCmnInfo); if (!(pWord->Body.bHasPrimEditDist && pWord->Base.wWordCompLen) || ISPROTECTEDWRD(pWord)) { return 0; } return 1; } static ET9BOOL ET9LOCALCALL __WordOrder_SpcTermCOND(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { ET9_UNUSED(pLingCmnInfo); if (!(pWord->Body.bHasPrimEditDist && !pWord->Base.wWordCompLen) || ISPROTECTEDWRD(pWord)) { return 0; } return 1; } static ET9BOOL ET9LOCALCALL __WordOrder_CmplCOND(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { ET9_UNUSED(pLingCmnInfo); if (!(!pWord->Body.bHasPrimEditDist && pWord->Base.wWordCompLen) || ISPROTECTEDWRD(pWord)) { return 0; } return 1; } static ET9BOOL ET9LOCALCALL __WordOrder_ProtectedCOND(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { ET9_UNUSED(pLingCmnInfo); if (ISPROTECTEDWRD(pWord)) { return 0; } return 1; } static ET9BOOL ET9LOCALCALL __WordOrder_CmplOnlyCOND(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { ET9_UNUSED(pLingCmnInfo); if (!pWord->Base.wWordCompLen) { return 0; } return 1; } static ET9BOOL ET9LOCALCALL __WordOrder_AnyCOND(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { ET9_UNUSED(pLingCmnInfo); ET9_UNUSED(pWord); return 1; } /* ============================================================================================================== */ typedef ET9INT ET9LOCALCALL __pWordListCMP(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWordTst, ET9AWPrivWordInfo const * const pWordSrc); static ET9INT ET9LOCALCALL __WordOrder_StringCMP(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWordTst, ET9AWPrivWordInfo const * const pWordSrc) { ET9_UNUSED(pLingCmnInfo); if (pWordTst->Body.dwStringHash < pWordSrc->Body.dwStringHash) { return -1; } else if (pWordTst->Body.dwStringHash > pWordSrc->Body.dwStringHash) { return +1; } if (pWordTst->Base.wWordLen < pWordSrc->Base.wWordLen) { return -1; } else if (pWordTst->Base.wWordLen > pWordSrc->Base.wWordLen) { return +1; } { ET9UINT nCount = pWordTst->Base.wWordLen; ET9SYMB const *ps1 = pWordTst->Base.sWord; ET9SYMB const *ps2 = pWordSrc->Base.sWord; for (; nCount; --nCount, ++ps1, ++ps2) { if (*ps1 != *ps2) { break; } } if (nCount) { if (*ps1 < *ps2) { return -1; } else if (*ps1 > *ps2) { return +1; } } } { const ET9UINT nDupeTst = (pWordTst->Body.bWordQuality == DUPE_QUALITY) ? 1 : 0; const ET9UINT nDupeSrc = (pWordSrc->Body.bWordQuality == DUPE_QUALITY) ? 1 : 0; if (nDupeTst < nDupeSrc) { return -1; } else if (nDupeTst > nDupeSrc) { return +1; } } return 0; } static ET9INT ET9LOCALCALL __WordOrder_PriorityCMP(ET9AWLingCmnInfo const * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWordTst, ET9AWPrivWordInfo const * const pWordSrc) { { const ET9INT snCmp = __PriorityCompare(pLingCmnInfo, pWordTst, pWordSrc, SS_Collecting); if (snCmp) { return snCmp; } } if (pWordTst < pWordSrc) { return -1; } else if (pWordTst > pWordSrc) { return +1; } return 0; } /* ============================================================================================================== */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __WordOrder_Initialize(ET9AWLingCmnInfo * const pLingCmnInfo) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; pNullNode->pLeft = pNullNode; pNullNode->pRight = pNullNode; pNullNode->sbLevel = 0; } #ifdef ET9_DEBUG /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * */ static void ET9LOCALCALL __WordOrder_Validate(ET9AWLingCmnInfo * const pLingCmnInfo, __pWordListCMP pCMP, _ET9AW_WordOrderNode * const pNode) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; if (pNode == pNullNode) { return; } if (pNode->pLeft != pNullNode) { const ET9INT snCmp = pCMP(pLingCmnInfo, pNode->pLeft->pWord, pNode->pWord); ET9AssertLog(snCmp < 0); } if (pNode->pRight != pNullNode) { const ET9INT snCmp = pCMP(pLingCmnInfo, pNode->pRight->pWord, pNode->pWord); ET9AssertLog(snCmp > 0); } __WordOrder_Validate(pLingCmnInfo, pCMP, pNode->pLeft); __WordOrder_Validate(pLingCmnInfo, pCMP, pNode->pRight); } #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_Find(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord, __pWordListCMP pCMP, _ET9AW_WordOrderNode * const pNode) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; if (pNode == pNullNode) { return pNullNode; } { const ET9INT snCmp = pCMP(pLingCmnInfo, pWord, pNode->pWord); if (snCmp < 0) { return __WordOrder_Find(pLingCmnInfo, pWord, pCMP, pNode->pLeft); } else if (snCmp > 0) { return __WordOrder_Find(pLingCmnInfo, pWord, pCMP, pNode->pRight); } else { return pNode; } } } #if 0 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_FindMin(ET9AWLingCmnInfo * const pLingCmnInfo, _ET9AW_WordOrderNode * const pNode) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; if (pNode == pNullNode) { return pNullNode; } else if (pNode->pLeft == pNullNode) { return pNode; } else { return __WordOrder_FindMin(pLingCmnInfo, pNode->pLeft); } } #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_FindMinConditional(ET9AWLingCmnInfo * const pLingCmnInfo, _ET9AW_WordOrderNode * const pNode, __pWordListCOND pCOND) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; if (pNode == pNullNode) { return pNullNode; } { _ET9AW_WordOrderNode * const pTmpNode = __WordOrder_FindMinConditional(pLingCmnInfo, pNode->pLeft, pCOND); if (pTmpNode != pNullNode) { return pTmpNode; } } if (pCOND(pLingCmnInfo, pNode->pWord)) { return pNode; } return __WordOrder_FindMinConditional(pLingCmnInfo, pNode->pRight, pCOND); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * * * * * */ static void ET9LOCALCALL __WordOrder_BuildAccessList(ET9AWLingCmnInfo * const pLingCmnInfo, _ET9AW_WordOrderNode * const pNode, ET9AWPrivWordInfo * const pFirstWord, ET9UINT * const pnList, ET9UINT * const pnCount) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; if (pNode == pNullNode) { return; } __WordOrder_BuildAccessList(pLingCmnInfo, pNode->pRight, pFirstWord, pnList, pnCount); pnList[(*pnCount)++] = (ET9UINT)(pNode->pWord - pFirstWord); __WordOrder_BuildAccessList(pLingCmnInfo, pNode->pLeft, pFirstWord, pnList, pnCount); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_SingleRotateWithLeft(_ET9AW_WordOrderNode * const pK2) { _ET9AW_WordOrderNode *pK1; pK1 = pK2->pLeft; pK2->pLeft = pK1->pRight; pK1->pRight = pK2; return pK1; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_SingleRotateWithRight(_ET9AW_WordOrderNode *pK1) { _ET9AW_WordOrderNode *pK2; pK2 = pK1->pRight; pK1->pRight = pK2->pLeft; pK2->pLeft = pK1; return pK2; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_Skew(_ET9AW_WordOrderNode *pNode) { if (pNode->pLeft->sbLevel == pNode->sbLevel) { pNode = __WordOrder_SingleRotateWithLeft(pNode); } return pNode; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_Split(_ET9AW_WordOrderNode *pNode) { if (pNode->pRight->pRight->sbLevel == pNode->sbLevel) { pNode = __WordOrder_SingleRotateWithRight(pNode); ++pNode->sbLevel; } return pNode; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_Insert(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord, _ET9AW_WordOrderNode * const pNodeToUse, __pWordListCMP pCMP, _ET9AW_WordOrderNode *pNode) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; if (pNode == pNullNode) { /* "Create" and return a one-node tree */ pNode = pNodeToUse; pNode->pWord = pWord; pNode->sbLevel = 1; pNode->pLeft = pNullNode; pNode->pRight = pNullNode; } else { const ET9INT snCmp = pCMP(pLingCmnInfo, pWord, pNode->pWord); if (snCmp < 0) { pNode->pLeft = __WordOrder_Insert(pLingCmnInfo, pWord, pNodeToUse, pCMP, pNode->pLeft); } else if (snCmp > 0) { pNode->pRight = __WordOrder_Insert(pLingCmnInfo, pWord, pNodeToUse, pCMP, pNode->pRight); } else { /* Otherwise it's a duplicate; not allowed */ ET9AssertLog(0); } } pNode = __WordOrder_Skew(pNode); pNode = __WordOrder_Split(pNode); return pNode; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * */ static _ET9AW_WordOrderNode* ET9LOCALCALL __WordOrder_Remove(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord, _ET9AW_WordOrderNode ** const ppNodeFreed, __pWordListCMP pCMP, _ET9AW_WordOrderNode *pNode) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; _ET9AW_WordOrderNode ** const ppDelete = &pLingCmnInfo->Private.sWordC.sPool.pTmp1; _ET9AW_WordOrderNode ** const ppLast = &pLingCmnInfo->Private.sWordC.sPool.pTmp2; if (pNode != pNullNode) { /* Step 1: Search down tree, set 'last' and 'delete' */ *ppLast = pNode; if (pCMP(pLingCmnInfo, pWord, pNode->pWord) < 0) { pNode->pLeft = __WordOrder_Remove(pLingCmnInfo, pWord, ppNodeFreed, pCMP, pNode->pLeft); } else { *ppDelete = pNode; pNode->pRight = __WordOrder_Remove(pLingCmnInfo, pWord, ppNodeFreed, pCMP, pNode->pRight); } /* Step 2: If at the bottom of the tree and item is present, we remove it */ if (pNode == *ppLast) { if (*ppDelete != pNullNode && pWord == (*ppDelete)->pWord) { (*ppDelete)->pWord = pNode->pWord; *ppDelete = pNullNode; pNode = pNode->pRight; *ppNodeFreed = *ppLast; } } /* Step 3: Otherwise, we are not at the bottom; rebalance */ else { if (pNode->pLeft->sbLevel < (pNode->sbLevel - 1) || pNode->pRight->sbLevel < (pNode->sbLevel - 1)) { if (pNode->pRight->sbLevel > --pNode->sbLevel) { pNode->pRight->sbLevel = pNode->sbLevel; } pNode = __WordOrder_Skew(pNode); pNode->pRight = __WordOrder_Skew(pNode->pRight); pNode->pRight->pRight = __WordOrder_Skew(pNode->pRight->pRight); pNode = __WordOrder_Split(pNode); pNode->pRight = __WordOrder_Split(pNode->pRight); } } } return pNode; } /* ============================================================================================================== */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __WordOrder_NewAdd(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { _ET9AW_WordCollection * const pWordC = &pLingCmnInfo->Private.sWordC; _ET9AW_Collection * const pCurrC = pWordC->pCurrC; ET9AssertLog(pCurrC->nTotalWords > 0); ET9AssertLog(pWord >= &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[0] && pWord < &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords]); pCurrC->pStringRoot = __WordOrder_Insert(pLingCmnInfo, pWord, &pCurrC->pStringOrder[pCurrC->nTotalWords - 1], __WordOrder_StringCMP, pCurrC->pStringRoot); pCurrC->pPriorityRoot = __WordOrder_Insert(pLingCmnInfo, pWord, &pCurrC->pPriorityOrder[pCurrC->nTotalWords - 1], __WordOrder_PriorityCMP, pCurrC->pPriorityRoot); #ifdef ET9_DEBUG __WordOrder_Validate(pLingCmnInfo, __WordOrder_StringCMP, pCurrC->pStringRoot); __WordOrder_Validate(pLingCmnInfo, __WordOrder_PriorityCMP, pCurrC->pPriorityRoot); #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __WordOrder_UpdateOut(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord, const ET9BOOL bUpdateString, const ET9BOOL bUpdatePriority) { _ET9AW_WordCollection * const pWordC = &pLingCmnInfo->Private.sWordC; _ET9AW_Collection * const pCurrC = pWordC->pCurrC; _ET9AW_WordOrderNode ** const ppStringSave = &pLingCmnInfo->Private.sWordC.sPool.pTmp3; _ET9AW_WordOrderNode ** const ppPrioritySave = &pLingCmnInfo->Private.sWordC.sPool.pTmp4; *ppStringSave = NULL; *ppPrioritySave = NULL; ET9AssertLog(pWord >= &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[0] && pWord < &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords]); if (bUpdateString) { pCurrC->pStringRoot = __WordOrder_Remove(pLingCmnInfo, pWord, ppStringSave, __WordOrder_StringCMP, pCurrC->pStringRoot); } if (bUpdatePriority) { pCurrC->pPriorityRoot = __WordOrder_Remove(pLingCmnInfo, pWord, ppPrioritySave, __WordOrder_PriorityCMP, pCurrC->pPriorityRoot); } #ifdef ET9_DEBUG __WordOrder_Validate(pLingCmnInfo, __WordOrder_StringCMP, pCurrC->pStringRoot); __WordOrder_Validate(pLingCmnInfo, __WordOrder_PriorityCMP, pCurrC->pPriorityRoot); #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __WordOrder_UpdateIn(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord, const ET9BOOL bUpdateString, const ET9BOOL bUpdatePriority) { _ET9AW_WordCollection * const pWordC = &pLingCmnInfo->Private.sWordC; _ET9AW_Collection * const pCurrC = pWordC->pCurrC; _ET9AW_WordOrderNode ** const ppStringSave = &pLingCmnInfo->Private.sWordC.sPool.pTmp3; _ET9AW_WordOrderNode ** const ppPrioritySave = &pLingCmnInfo->Private.sWordC.sPool.pTmp4; ET9AssertLog(!bUpdateString || *ppStringSave); ET9AssertLog(!bUpdatePriority || *ppPrioritySave); ET9AssertLog(pWord >= &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[0] && pWord < &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords]); if (bUpdateString) { pCurrC->pStringRoot = __WordOrder_Insert(pLingCmnInfo, pWord, *ppStringSave, __WordOrder_StringCMP, pCurrC->pStringRoot); } if (bUpdatePriority) { pCurrC->pPriorityRoot = __WordOrder_Insert(pLingCmnInfo, pWord, *ppPrioritySave, __WordOrder_PriorityCMP, pCurrC->pPriorityRoot); } #ifdef ET9_DEBUG __WordOrder_Validate(pLingCmnInfo, __WordOrder_StringCMP, pCurrC->pStringRoot); __WordOrder_Validate(pLingCmnInfo, __WordOrder_PriorityCMP, pCurrC->pPriorityRoot); #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9AWPrivWordInfo* ET9LOCALCALL __WordOrder_FindMinPriorityConditional(ET9AWLingCmnInfo * const pLingCmnInfo, __pWordListCOND pCOND) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; _ET9AW_Collection * const pCurrC = pLingCmnInfo->Private.sWordC.pCurrC; _ET9AW_WordOrderNode const * const pNode = __WordOrder_FindMinConditional(pLingCmnInfo, pCurrC->pPriorityRoot, pCOND); if (pNode == pNullNode) { return NULL; } else { return (ET9AWPrivWordInfo*)pNode->pWord; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __WordOrder_ExportPriorityAccessList(ET9AWLingCmnInfo * const pLingCmnInfo) { _ET9AW_Collection * const pCurrC = pLingCmnInfo->Private.sWordC.pCurrC; ET9UINT nCount; nCount = 0; __WordOrder_BuildAccessList(pLingCmnInfo, pCurrC->pPriorityRoot, pCurrC->pWordList, pCurrC->pnWordList, &nCount); ET9AssertLog(nCount == pCurrC->nTotalWords); } /* ************************************************************************************************************** */ /* ************************************************************************************************************** */ /* ************************************************************************************************************** */ #ifdef ET9_DEBUG /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __VerifyWordCounters(ET9AWLingCmnInfo * const pLingCmnInfo) { if (!pLingCmnInfo->Private.bSpcComplDuringSingleBuild) { ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords <= pLingCmnInfo->Private.ASpc.nMaxSpcTermCount); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords <= pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords <= pLingCmnInfo->Private.nMaxCompletionCount); } { ET9UINT nIndex; ET9UINT nSpcTermWords; ET9UINT nSpcCmplWords; ET9UINT nCompletionWords; ET9AWPrivWordInfo *pWord; nSpcTermWords = 0; nSpcCmplWords = 0; nCompletionWords = 0; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[nIndex]; if (pWord->Body.bHasPrimEditDist && pWord->Base.wWordCompLen) { ++nSpcCmplWords; } else if (pWord->Base.wWordCompLen) { ++nCompletionWords; } else if (pWord->Body.bHasPrimEditDist) { ++nSpcTermWords; } } if (!pLingCmnInfo->Private.bSpcComplDuringSingleBuild) { ET9AssertLog(nSpcTermWords == pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords); ET9AssertLog(nSpcCmplWords == pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords); ET9AssertLog(nCompletionWords == pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords); } } } #else #define __VerifyWordCounters(pLingCmnInfo) #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __ET9HandleInsertCounters(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo * const pOldWord, ET9AWPrivWordInfo * const pNewWord) { const ET9BOOL bSpcComplDuringSingleBuild = pLingCmnInfo->Private.bSpcComplDuringSingleBuild; /* validation*/ if (!bSpcComplDuringSingleBuild) { ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords <= pLingCmnInfo->Private.ASpc.nMaxSpcTermCount); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords <= pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords <= pLingCmnInfo->Private.nMaxCompletionCount); } /* log */ WLOG2BWord(pLogFile2, "__ET9HandleInsertCounters, pOldWordIn", pOldWord); WLOG2BWord(pLogFile2, "__ET9HandleInsertCounters, pNewWordIn", pNewWord); WLOG2B(fprintf(pLogFile2, "__ET9HandleInsertCounters, compl count = %d (before)\n", pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords);) /* update info */ if (pOldWord->Base.wWordCompLen && pOldWord->Body.bHasPrimEditDist) { --pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords; } else if (pOldWord->Base.wWordCompLen) { --pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords; } else if (pOldWord->Body.bHasPrimEditDist) { --pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords; } if (pNewWord->Base.wWordCompLen && pNewWord->Body.bHasPrimEditDist) { ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords; } else if (pNewWord->Base.wWordCompLen) { ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords; } else if (pNewWord->Body.bHasPrimEditDist) { ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords; } WLOG2B(fprintf(pLogFile2, "__ET9HandleInsertCounters, compl count = %d (after)\n", pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords);) /* validation*/ if (!bSpcComplDuringSingleBuild) { ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords <= pLingCmnInfo->Private.ASpc.nMaxSpcTermCount); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords <= pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords <= pLingCmnInfo->Private.nMaxCompletionCount); } /* update last pointers */ if (pLingCmnInfo->Private.sWordC.pCurrC->pLastWord == pOldWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastWord = NULL; } if (pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord == pOldWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord = NULL; } if (pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord == pOldWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord = NULL; } if (pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord == pOldWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord = NULL; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __ET9UpdateInsertCounters(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo * const pOldWord, ET9AWPrivWordInfo * const pNewWord) { __ET9HandleInsertCounters(pLingCmnInfo, pOldWord, pNewWord); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __ET9ReplaceAndUpdateInsertCounters(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo * const pOldWord, ET9AWPrivWordInfo * const pNewWord) { __WordOrder_UpdateOut(pLingCmnInfo, pOldWord, 1, 1); __ET9HandleInsertCounters(pLingCmnInfo, pOldWord, pNewWord); *pOldWord = *pNewWord; __WordOrder_UpdateIn(pLingCmnInfo, pOldWord, 1, 1); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static ET9INT ET9LOCALCALL __PriorityCmp(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9INT snIndex1, const ET9INT snIndex2) { ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; ET9AWPrivWordInfo * const pw1 = &pWordList[snIndex1]; ET9AWPrivWordInfo * const pw2 = &pWordList[snIndex2]; ET9INT snResult; ET9AssertLog(snIndex1 >= 0 && snIndex1 < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); ET9AssertLog(snIndex2 >= 0 && snIndex2 < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); snResult = __PriorityCompare(pLingCmnInfo, pw1, pw2, SS_Done); if (snResult == 0) { snResult = (pw1 < pw2) ? +1 : ((pw1 > pw2) ? -1 : 0); /* arrival order */ } return snResult; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __ValueSwap(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9INT snIndex1, const ET9INT snIndex2) { ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; ET9UINT * const pn1 = &pnWordList[snIndex1]; ET9UINT * const pn2 = &pnWordList[snIndex2]; const ET9UINT nTmp = *pn1; *pn1 = *pn2; *pn2 = nTmp; ET9AssertLog(*pn1 >= 0 && *pn1 < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); ET9AssertLog(*pn2 >= 0 && *pn2 < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); ET9AssertLog(snIndex1 >= 0 && snIndex1 < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); ET9AssertLog(snIndex2 >= 0 && snIndex2 < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ void ET9FARCALL _ET9AWSortSearchListQS(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9INT snLo, const ET9INT snHi, fWordListComparator fCompareFn) { ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; if (snHi > snLo) { /* There are at least two things to compare */ /* Note that unlike the standard simple QuickSort, we pick a pivot * in the middle. This is because our inputs are often sorted or * nearly sorted, so this gets a pivot closer to the median. */ const ET9INT snPivotPos = (snLo + snHi) / 2; const ET9INT snPivotItem = pnWordList[snPivotPos]; ET9INT snL = snLo; ET9INT snH = snHi; while (snL <= snH) { while (snL == snPivotPos || (snL <= snH && fCompareFn(pLingCmnInfo, pnWordList[snL], snPivotItem) > 0)) { ++snL; } while (snH == snPivotPos || (snL <= snH && fCompareFn(pLingCmnInfo, pnWordList[snH], snPivotItem) < 0)) { --snH; } if (snL <= snH) { __ValueSwap(pLingCmnInfo, snL, snH); ++snL; --snH; } } if (snL < snPivotPos) { /* Our pivot is to the right of the divide */ __ValueSwap(pLingCmnInfo, snL++, snPivotPos); } else if (snH > snPivotPos) { /* Our pivot is to the left of the divide */ __ValueSwap(pLingCmnInfo, snH--, snPivotPos); } ET9AssertLog(snL == snH + 2); ET9AssertLog(pnWordList[snH + 1] == (ET9UINT)snPivotItem); _ET9AWSortSearchListQS(pLingCmnInfo, snLo, snH, fCompareFn); _ET9AWSortSearchListQS(pLingCmnInfo, snL, snHi, fCompareFn); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static void ET9LOCALCALL __ET9AWSortPriorityList(ET9AWLingCmnInfo * const pLingCmnInfo) { __ProfileStart; WLOG2(fprintf(pLogFile2, "__ET9AWSortPriorityList, eCurrSelectionListMode = %s\n", SELLSTMODETOSTRING(pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode));) _ET9AWSortSearchListQS(pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, __PriorityCmp); __LogPartialSelList(pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "after _ET9AWSortSearchListQS", pLogFile2); __ProfileEnd(tAW_SelLst_SortPriorityList); #ifdef ET9_DEBUG { ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; ET9UINT pnCount[ET9MAXCOLLECTSIZE]; ET9UINT nIndex; for (nIndex = 0; nIndex < ET9MAXCOLLECTSIZE; ++nIndex) { pnCount[nIndex] = 0; } for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ++pnCount[pnWordList[nIndex]]; } } { ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; ET9UINT nIndex; ET9UINT nLook; for (nIndex = 0; nIndex + 1 < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { for (nLook = nIndex + 1; nLook < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nLook) { ET9AssertLog(__PriorityCmp(pLingCmnInfo, pnWordList[nIndex], pnWordList[nLook]) >= 0); } } } #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __ET9AWIsWordInList(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo * const pWord, ET9AWPrivWordInfo ** const ppWordFound) { _ET9AW_WordOrderNode * const pNullNode = &pLingCmnInfo->Private.sWordC.sPool.sNullNode; _ET9AW_Collection * const pCurrC = pLingCmnInfo->Private.sWordC.pCurrC; _ET9AW_WordOrderNode const * const pNode = __WordOrder_Find(pLingCmnInfo, pWord, __WordOrder_StringCMP, pCurrC->pStringRoot); if (pNode == pNullNode) { *ppWordFound = NULL; } else { *ppWordFound = (ET9AWPrivWordInfo*)pNode->pWord; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __UpdateCustomWordFreqs(ET9AWLingInfo * const pLingInfo) { ET9UINT nListIndex; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; WLOG2(fprintf(pLogFile2, "__UpdateCustomWordFreqs\n");) if (!__IsUsingMixedStyle(pLingCmnInfo)) { WLOG2(fprintf(pLogFile2, " not mixed list - no updates\n");) return; } for (nListIndex = 0; nListIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nListIndex) { ET9AWPrivWordInfo * const pWord = &pWordList[pnWordList[nListIndex]]; /* valid user/oem words only */ if (pWord->Body.dwWordIndex) { continue; } if ((GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_RUDB && pWord->Body.bIsUDBWord) || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_ASDB_SHORTCUT || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_MDB || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM) { if (GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM) { /* interpolate LDB/DLM */ WLOG2(fprintf(pLogFile2, " DLM, xWordFreq %14.2f", pWord->Body.xWordFreq);) pWord->Body.xWordFreq = (ET9FREQPART)(pLingCmnInfo->Private.fInterpolFactDLM * pWord->Body.xWordFreq); WLOG2(fprintf(pLogFile2, ", final %14.2f\n", pWord->Body.xWordFreq);) } else if (!pLingCmnInfo->Private.xMaxWordFreq) { /* no op */ } else if (GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_MDB) { pWord->Body.xWordFreq = pLingCmnInfo->Private.xMaxWordFreq; } else { const ET9FREQPART xBaseFreq = ET9_SUPP_DB_BASE_FREQ; if (pWord->Body.xWordFreq > xBaseFreq) { pWord->Body.xWordFreq = (ET9FREQPART)(pWord->Body.xWordFreq - xBaseFreq); } else { pWord->Body.xWordFreq = 1; } pWord->Body.xWordFreq += pLingCmnInfo->Private.xMaxWordFreq; /* extra penalty to improve prediction performance */ pWord->Body.xWordFreq /= 10.0f; } ET9AssertLog(pWord->Body.xWordFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __CheckPredictionLoop(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo * const pWord) { const ET9U32 dwWordIndex = pWord->Body.dwWordIndex - 1; ET9BOOL bContextDup = 0; if (pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { return; } WLOG2B(fprintf(pLogFile2, "__CheckPredictionLoop, dwWordIndex %6u, bNLMOrder %u\n", dwWordIndex, pWord->Body.bNLMOrder);) { ET9UINT nContextIndex; for (nContextIndex = (!pWord->Body.bNLMOrder ? 0 : pWord->Body.bNLMOrder - 1); nContextIndex < ET9NLM_CONTEXT_COUNT; ++nContextIndex) { WLOG2B(fprintf(pLogFile2, " [%u] wLen %2u dwWordIndex %6u\n", nContextIndex, pLingCmnInfo->Private.pContextWords[nContextIndex].wLen, pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[nContextIndex]);) if (pLingCmnInfo->Private.pContextWords[nContextIndex].wLen && dwWordIndex == pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[nContextIndex]) { bContextDup = 1; break; } } } if (bContextDup) { WLOG2(fprintf(pLogFile2, "__CheckPredictionLoop, [%05u] xWordFreq %14.2f (%14.2f) xInputScore %14.2f (%14.2f)\n", pWord->Body.dwWordIndex, (float)(pWord->Body.xTapFreq / 10), (float)pWord->Body.xTapFreq, (float)(pWord->Body.xInputScore / 10), (float)pWord->Body.xInputScore);) pWord->Body.xTapFreq /= 10; if (pWord->Body.xTapFreq == 0) { pWord->Body.xTapFreq = 1; } pWord->Body.xInputScore /= 10; if (pWord->Body.xInputScore == 0) { pWord->Body.xInputScore = 1; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __UpdateBuiltInWordFreqs(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; const ET9BOOL bTraceBuild = pLingCmnInfo->Private.bTraceBuild; const ET9BOOL bDLMActive = (pLingCmnInfo->pDLMInfo && ET9DLMENABLED(pLingCmnInfo)) ? 1 : 0; const ET9UINT nIntraPassCount = 2; const ET9UINT nPassCount = nIntraPassCount * (ET9AW_GetBilingualSupported(pLingInfo) ? 2 : 1); ET9UINT nPass; __ProfileStart; WLOG2(fprintf(pLogFile2, "__UpdateBuiltInWordFreqs\n");) pLingCmnInfo->Private.xMaxWordFreq = 0; for (nPass = 0; nPass < nPassCount; ++nPass) { ET9UINT nListIndex; const ET9U8 bTargetLangIndex = (ET9U8)(nPass < nIntraPassCount ? ET9AWFIRST_LANGUAGE : ET9AWSECOND_LANGUAGE); const ET9U32 dwTargetLdbNum = nPass < nIntraPassCount ? pLingCmnInfo->dwFirstLdbNum : pLingCmnInfo->dwSecondLdbNum; const ET9BOOL bUsingLM = bDLMActive || _ET9AW_IsUsingLM(pLingInfo); const ET9BOOL bPrefixPass = (nPass % 2) ? 1 : 0; ET9BOOL bHasValidContext = 0; ET9U32 dwCurrPrefixCheckSum = 0; WLOG2B(fprintf(pLogFile2, "__UpdateBuiltInWordFreqs, pass %u, bPrefixPass %u\n", nPass, bPrefixPass);) for (nListIndex = 0; nListIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nListIndex) { ET9AWPrivWordInfo * const pWord = &pWordList[pnWordList[nListIndex]]; const ET9U8 bLangIndex = pWord->Body.bLangIndexScoring; ET9AssertLog(pWord->Body.xTapFreq >= 0); ET9AssertLog(pWord->Body.xWordFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); if (!pWord->Body.dwWordIndex || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE || ISPUNCTSRC(pWord->Body.bWordSrc)) { continue; } if ((bPrefixPass && !pWord->Body.wPrefixLen) || (!bPrefixPass && pWord->Body.wPrefixLen)) { continue; } /* if curr lang, then find freq using known index */ if (bLangIndex == bTargetLangIndex || (bLangIndex == ET9AWBOTH_LANGUAGES && dwTargetLdbNum == pLingCmnInfo->Private.dwCurrActiveLanguage)) { if (bUsingLM) { if (pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs || pWord->Body.bNLMOrder) { #ifdef ET9_DEBUG_ if (pWord->Base.wWordLen < ET9MAXWORDSIZE && !GETATTRSRC(pWord->Body.bWordSrc) && (pWord->Base.bIsTerm || pWord->Base.wWordCompLen)) { ET9AWPrivWordInfo sWord; ET9U32 dwWordIndex = pWord->Body.dwWordIndex - 1; _ET9AWLdbWordsByIndex(pLingInfo, dwTargetLdbNum, &dwWordIndex, 1, 0, &sWord); { const ET9U16 wOffset = pWord->Body.wPrefixLen ? (((pWord->Base.wWordLen - pWord->Body.wPrefixLen) <= sWord.Base.wWordLen) ? pWord->Body.wPrefixLen : (pWord->Base.wWordLen - sWord.Base.wWordLen)) : (pWord->Base.wWordLen >= sWord.Base.wWordLen) ? (pWord->Base.wWordLen - sWord.Base.wWordLen) : 0; const ET9U16 wCmpLen = (pWord->Base.wWordLen - wOffset) <= sWord.Base.wWordLen ? (pWord->Base.wWordLen - wOffset) : sWord.Base.wWordLen; ET9U16 wCount; ET9SYMB *psA = &pWord->Base.sWord[wOffset]; ET9SYMB *psB = &sWord.Base.sWord[0]; for (wCount = wCmpLen; wCount; --wCount, ++psA, ++psB) { ET9AssertLog(*psA == *psB || _ET9SymToLower(*psA, dwTargetLdbNum) == _ET9SymToLower(*psB, dwTargetLdbNum)); } } WLOG2Word(pLogFile2, " word", pWord); WLOG2Word(pLogFile2, " frq", &sWord); } #endif if (pWord->Body.wPrefixLen) { const ET9U32 dwThisPrefixCheckSum = _ET9SymbStringCheckSum(pWord->Base.sWord, pWord->Body.wPrefixLen); if (dwCurrPrefixCheckSum != dwThisPrefixCheckSum) { bHasValidContext = 0; dwCurrPrefixCheckSum = dwThisPrefixCheckSum; } } if (!bHasValidContext) { WLOG2(fprintf(pLogFile2, "__UpdateBuiltInWordFreqs, updating context\n");) _ET9AWLdbTagContext(pLingInfo, dwTargetLdbNum, pWord); bHasValidContext = 1; } /* potentially interpolate LDB/DLM */ { const ET9FREQPART xDLMWordFreq = (GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM) ? pWord->Body.xWordFreq : 0; ET9AWPrivWordInfo sWordTmp = *pWord; /* must use a temp, might loose other attributes in the "get word freq" call */ WLOG2(fprintf(pLogFile2, "__UpdateBuiltInWordFreqs, xDLMWordFreq %14.2f", xDLMWordFreq);) _ET9AWLMGetWordFreq(pLingInfo, dwTargetLdbNum, &sWordTmp); WLOG2(fprintf(pLogFile2, ", LDB-freq %14.2f", sWordTmp.Body.xWordFreq);) pWord->Body.xWordFreq = (ET9FREQPART)((pLingCmnInfo->Private.fInterpolFactLDB * sWordTmp.Body.xWordFreq) + (pLingCmnInfo->Private.fInterpolFactDLM * xDLMWordFreq)); WLOG2(fprintf(pLogFile2, ", final %14.2f\n", pWord->Body.xWordFreq);) #ifdef ET9_DEBUG pWord->Body.sScoreContext = sWordTmp.Body.sScoreContext; #endif } } else { pWord->Body.xWordFreq = 1; } /* loops */ __CheckPredictionLoop(pLingCmnInfo, pWord); } /* update freqs */ ET9AssertLog(pWord->Body.xWordFreq >= 0); if (pLingCmnInfo->Private.xMaxWordFreq < pWord->Body.xWordFreq && !pWord->Body.bNLMOrder && !pWord->Base.wWordCompLen && (!(pWord->Body.bEditDistSpc - pWord->Body.bEditDistSpcTrp) || bTraceBuild)) { pLingCmnInfo->Private.xMaxWordFreq = pWord->Body.xWordFreq; } } ET9AssertLog(pWord->Body.xTapFreq >= 0); ET9AssertLog(pWord->Body.xWordFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); } } __ProfileEnd(tAW_SelLst_UpdateBuiltInWordFreqs); WLOG2(fprintf(pLogFile2, "__UpdateBuiltInWordFreqs, xMaxWordFreq = %14.2f\n", (float)pLingCmnInfo->Private.xMaxWordFreq);) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static ET9FREQ ET9LOCALCALL __CalculateWordTotFreq(ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo const * const pWord) { if (pLingCmnInfo->Private.__SPATH_Status.bHasSpmScores) { return (ET9FREQ)(pWord->Body.xInputScoreSpm * pWord->Body.xWordFreq); } else { return (ET9FREQ)(pWord->Body.xInputScore * pWord->Body.xWordFreq); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __CalculateFinalTotWordFreqs(ET9AWLingInfo * const pLingInfo) { ET9UINT nListIndex; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; ET9UINT * const pbWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; WLOG2(fprintf(pLogFile2, "__CalculateFinalTotWordFreqs\n");) /* fix to adjust scoring for some words */ { const ET9U32 dw_w1 = 0x5fdf262a; /* swype */ const ET9U32 dw_w2 = 0xe253460a; /* Swype */ const ET9U32 dw_w3 = 0x0197360a; /* SWYPE */ const ET9FREQPART xWordMinFreq = 150000.0f; for (nListIndex = 0; nListIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nListIndex) { ET9AWPrivWordInfo * const pWord = &pWordList[pbWordList[nListIndex]]; if (pWord->Base.wWordLen != 5) { continue; } if (pWord->Body.xWordFreq >= xWordMinFreq) { continue; } if (pWord->Body.dwStringHash != dw_w1 && pWord->Body.dwStringHash != dw_w2 && pWord->Body.dwStringHash != dw_w3) { continue; } WLOG2(fprintf(pLogFile2, " [%02u %6u] modifying word score (special word) %9.2f -> %9.2f\n", nListIndex, pWord->Body.dwWordIndex, (float)pWord->Body.xWordFreq, (float)xWordMinFreq);) pWord->Body.xWordFreq = xWordMinFreq; } } /* calculate final scores for all words */ for (nListIndex = 0; nListIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nListIndex) { ET9AWPrivWordInfo * const pWord = &pWordList[pbWordList[nListIndex]]; { const ET9FREQ xTotFreq = __CalculateWordTotFreq(pLingCmnInfo, pWord); WLOG2(fprintf(pLogFile2, " [%02u %6u] xTotFreq %20.0f xWordFreq %14.2f (%20.0f)\n", nListIndex, pWord->Body.dwWordIndex, (float)xTotFreq, (float)pWord->Body.xWordFreq, (float)pWord->Body.xTotFreq);) pWord->Body.xTotFreq = xTotFreq; } ET9AssertLog(pWord->Body.xWordFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); } } #ifdef ET9_USE_FLOAT_FREQS /*---------------------------------------------------------------------------*/ /* -IDR- * */ static const ET9FLOAT pfCorrectionData[] = { 2.601674e-015f, /* 1.3 ^ -128 */ 3.382177e-015f, /* 1.3 ^ -127 */ 4.396829e-015f, /* 1.3 ^ -126 */ 5.715878e-015f, /* 1.3 ^ -125 */ 7.430641e-015f, /* 1.3 ^ -124 */ 9.659833e-015f, /* 1.3 ^ -123 */ 1.255778e-014f, /* 1.3 ^ -122 */ 1.632512e-014f, /* 1.3 ^ -121 */ 2.122265e-014f, /* 1.3 ^ -120 */ 2.758945e-014f, /* 1.3 ^ -119 */ 3.586628e-014f, /* 1.3 ^ -118 */ 4.662616e-014f, /* 1.3 ^ -117 */ 6.061400e-014f, /* 1.3 ^ -116 */ 7.879820e-014f, /* 1.3 ^ -115 */ 1.024377e-013f, /* 1.3 ^ -114 */ 1.331689e-013f, /* 1.3 ^ -113 */ 1.731196e-013f, /* 1.3 ^ -112 */ 2.250555e-013f, /* 1.3 ^ -111 */ 2.925722e-013f, /* 1.3 ^ -110 */ 3.803438e-013f, /* 1.3 ^ -109 */ 4.944469e-013f, /* 1.3 ^ -108 */ 6.427810e-013f, /* 1.3 ^ -107 */ 8.356152e-013f, /* 1.3 ^ -106 */ 1.086300e-012f, /* 1.3 ^ -105 */ 1.412190e-012f, /* 1.3 ^ -104 */ 1.835846e-012f, /* 1.3 ^ -103 */ 2.386600e-012f, /* 1.3 ^ -102 */ 3.102580e-012f, /* 1.3 ^ -101 */ 4.033354e-012f, /* 1.3 ^ -100 */ 5.243360e-012f, /* 1.3 ^ -99 */ 6.816368e-012f, /* 1.3 ^ -98 */ 8.861278e-012f, /* 1.3 ^ -97 */ 1.151966e-011f, /* 1.3 ^ -96 */ 1.497556e-011f, /* 1.3 ^ -95 */ 1.946823e-011f, /* 1.3 ^ -94 */ 2.530869e-011f, /* 1.3 ^ -93 */ 3.290130e-011f, /* 1.3 ^ -92 */ 4.277169e-011f, /* 1.3 ^ -91 */ 5.560319e-011f, /* 1.3 ^ -90 */ 7.228414e-011f, /* 1.3 ^ -89 */ 9.396939e-011f, /* 1.3 ^ -88 */ 1.221602e-010f, /* 1.3 ^ -87 */ 1.588083e-010f, /* 1.3 ^ -86 */ 2.064507e-010f, /* 1.3 ^ -85 */ 2.683859e-010f, /* 1.3 ^ -84 */ 3.489017e-010f, /* 1.3 ^ -83 */ 4.535722e-010f, /* 1.3 ^ -82 */ 5.896438e-010f, /* 1.3 ^ -81 */ 7.665369e-010f, /* 1.3 ^ -80 */ 9.964980e-010f, /* 1.3 ^ -79 */ 1.295447e-009f, /* 1.3 ^ -78 */ 1.684081e-009f, /* 1.3 ^ -77 */ 2.189306e-009f, /* 1.3 ^ -76 */ 2.846098e-009f, /* 1.3 ^ -75 */ 3.699927e-009f, /* 1.3 ^ -74 */ 4.809904e-009f, /* 1.3 ^ -73 */ 6.252876e-009f, /* 1.3 ^ -72 */ 8.128738e-009f, /* 1.3 ^ -71 */ 1.056736e-008f, /* 1.3 ^ -70 */ 1.373757e-008f, /* 1.3 ^ -69 */ 1.785883e-008f, /* 1.3 ^ -68 */ 2.321648e-008f, /* 1.3 ^ -67 */ 3.018143e-008f, /* 1.3 ^ -66 */ 3.923586e-008f, /* 1.3 ^ -65 */ 5.100661e-008f, /* 1.3 ^ -64 */ 6.630859e-008f, /* 1.3 ^ -63 */ 8.620117e-008f, /* 1.3 ^ -62 */ 1.120615e-007f, /* 1.3 ^ -61 */ 1.456800e-007f, /* 1.3 ^ -60 */ 1.893839e-007f, /* 1.3 ^ -59 */ 2.461991e-007f, /* 1.3 ^ -58 */ 3.200588e-007f, /* 1.3 ^ -57 */ 4.160765e-007f, /* 1.3 ^ -56 */ 5.408994e-007f, /* 1.3 ^ -55 */ 7.031692e-007f, /* 1.3 ^ -54 */ 9.141199e-007f, /* 1.3 ^ -53 */ 1.188356e-006f, /* 1.3 ^ -52 */ 1.544863e-006f, /* 1.3 ^ -51 */ 2.008321e-006f, /* 1.3 ^ -50 */ 2.610818e-006f, /* 1.3 ^ -49 */ 3.394063e-006f, /* 1.3 ^ -48 */ 4.412281e-006f, /* 1.3 ^ -47 */ 5.735965e-006f, /* 1.3 ^ -46 */ 7.456755e-006f, /* 1.3 ^ -45 */ 9.693781e-006f, /* 1.3 ^ -44 */ 1.260191e-005f, /* 1.3 ^ -43 */ 1.638249e-005f, /* 1.3 ^ -42 */ 2.129723e-005f, /* 1.3 ^ -41 */ 2.768640e-005f, /* 1.3 ^ -40 */ 3.599232e-005f, /* 1.3 ^ -39 */ 4.679002e-005f, /* 1.3 ^ -38 */ 6.082702e-005f, /* 1.3 ^ -37 */ 7.907513e-005f, /* 1.3 ^ -36 */ 1.027977e-004f, /* 1.3 ^ -35 */ 1.336369e-004f, /* 1.3 ^ -34 */ 1.737280e-004f, /* 1.3 ^ -33 */ 2.258464e-004f, /* 1.3 ^ -32 */ 2.936003e-004f, /* 1.3 ^ -31 */ 3.816804e-004f, /* 1.3 ^ -30 */ 4.961846e-004f, /* 1.3 ^ -29 */ 6.450399e-004f, /* 1.3 ^ -28 */ 8.385518e-004f, /* 1.3 ^ -27 */ 1.090117e-003f, /* 1.3 ^ -26 */ 1.417152e-003f, /* 1.3 ^ -25 */ 1.842298e-003f, /* 1.3 ^ -24 */ 2.394988e-003f, /* 1.3 ^ -23 */ 3.113484e-003f, /* 1.3 ^ -22 */ 4.047529e-003f, /* 1.3 ^ -21 */ 5.261787e-003f, /* 1.3 ^ -20 */ 6.840323e-003f, /* 1.3 ^ -19 */ 8.892420e-003f, /* 1.3 ^ -18 */ 1.156014e-002f, /* 1.3 ^ -17 */ 1.502819e-002f, /* 1.3 ^ -16 */ 1.953664e-002f, /* 1.3 ^ -15 */ 2.539764e-002f, /* 1.3 ^ -14 */ 3.301692e-002f, /* 1.3 ^ -13 */ 4.292200e-002f, /* 1.3 ^ -12 */ 5.579860e-002f, /* 1.3 ^ -11 */ 7.253817e-002f, /* 1.3 ^ -10 */ 9.429963e-002f, /* 1.3 ^ -9 */ 1.225895e-001f, /* 1.3 ^ -8 */ 1.593664e-001f, /* 1.3 ^ -7 */ 2.071763e-001f, /* 1.3 ^ -6 */ 2.693291e-001f, /* 1.3 ^ -5 */ 3.501278e-001f, /* 1.3 ^ -4 */ 4.551662e-001f, /* 1.3 ^ -3 */ 5.917160e-001f, /* 1.3 ^ -2 */ 7.692308e-001f, /* 1.3 ^ -1 */ 1.000000e+000f, /* 1.3 ^ 0 */ 1.300000e+000f, /* 1.3 ^ 1 */ 1.690000e+000f, /* 1.3 ^ 2 */ 2.197000e+000f, /* 1.3 ^ 3 */ 2.856100e+000f, /* 1.3 ^ 4 */ 3.712929e+000f, /* 1.3 ^ 5 */ 4.826808e+000f, /* 1.3 ^ 6 */ 6.274850e+000f, /* 1.3 ^ 7 */ 8.157305e+000f, /* 1.3 ^ 8 */ 1.060450e+001f, /* 1.3 ^ 9 */ 1.378584e+001f, /* 1.3 ^ 10 */ 1.792160e+001f, /* 1.3 ^ 11 */ 2.329807e+001f, /* 1.3 ^ 12 */ 3.028750e+001f, /* 1.3 ^ 13 */ 3.937374e+001f, /* 1.3 ^ 14 */ 5.118586e+001f, /* 1.3 ^ 15 */ 6.654162e+001f, /* 1.3 ^ 16 */ 8.650410e+001f, /* 1.3 ^ 17 */ 1.124553e+002f, /* 1.3 ^ 18 */ 1.461919e+002f, /* 1.3 ^ 19 */ 1.900495e+002f, /* 1.3 ^ 20 */ 2.470643e+002f, /* 1.3 ^ 21 */ 3.211836e+002f, /* 1.3 ^ 22 */ 4.175387e+002f, /* 1.3 ^ 23 */ 5.428003e+002f, /* 1.3 ^ 24 */ 7.056404e+002f, /* 1.3 ^ 25 */ 9.173324e+002f, /* 1.3 ^ 26 */ 1.192532e+003f, /* 1.3 ^ 27 */ 1.550292e+003f, /* 1.3 ^ 28 */ 2.015379e+003f, /* 1.3 ^ 29 */ 2.619993e+003f, /* 1.3 ^ 30 */ 3.405990e+003f, /* 1.3 ^ 31 */ 4.427788e+003f, /* 1.3 ^ 32 */ 5.756124e+003f, /* 1.3 ^ 33 */ 7.482960e+003f, /* 1.3 ^ 34 */ 9.727848e+003f, /* 1.3 ^ 35 */ 1.264620e+004f, /* 1.3 ^ 36 */ 1.644006e+004f, /* 1.3 ^ 37 */ 2.137208e+004f, /* 1.3 ^ 38 */ 2.778370e+004f, /* 1.3 ^ 39 */ 3.611881e+004f, /* 1.3 ^ 40 */ 4.695445e+004f, /* 1.3 ^ 41 */ 6.104079e+004f, /* 1.3 ^ 42 */ 7.935302e+004f, /* 1.3 ^ 43 */ 1.031589e+005f, /* 1.3 ^ 44 */ 1.341066e+005f, /* 1.3 ^ 45 */ 1.743386e+005f, /* 1.3 ^ 46 */ 2.266401e+005f, /* 1.3 ^ 47 */ 2.946322e+005f, /* 1.3 ^ 48 */ 3.830218e+005f, /* 1.3 ^ 49 */ 4.979283e+005f, /* 1.3 ^ 50 */ 6.473068e+005f, /* 1.3 ^ 51 */ 8.414988e+005f, /* 1.3 ^ 52 */ 1.093948e+006f, /* 1.3 ^ 53 */ 1.422133e+006f, /* 1.3 ^ 54 */ 1.848773e+006f, /* 1.3 ^ 55 */ 2.403404e+006f, /* 1.3 ^ 56 */ 3.124426e+006f, /* 1.3 ^ 57 */ 4.061753e+006f, /* 1.3 ^ 58 */ 5.280279e+006f, /* 1.3 ^ 59 */ 6.864362e+006f, /* 1.3 ^ 60 */ 8.923670e+006f, /* 1.3 ^ 61 */ 1.160077e+007f, /* 1.3 ^ 62 */ 1.508100e+007f, /* 1.3 ^ 63 */ 1.960530e+007f, /* 1.3 ^ 64 */ 2.548689e+007f, /* 1.3 ^ 65 */ 3.313296e+007f, /* 1.3 ^ 66 */ 4.307284e+007f, /* 1.3 ^ 67 */ 5.599469e+007f, /* 1.3 ^ 68 */ 7.279310e+007f, /* 1.3 ^ 69 */ 9.463102e+007f, /* 1.3 ^ 70 */ 1.230203e+008f, /* 1.3 ^ 71 */ 1.599264e+008f, /* 1.3 ^ 72 */ 2.079043e+008f, /* 1.3 ^ 73 */ 2.702756e+008f, /* 1.3 ^ 74 */ 3.513583e+008f, /* 1.3 ^ 75 */ 4.567658e+008f, /* 1.3 ^ 76 */ 5.937955e+008f, /* 1.3 ^ 77 */ 7.719341e+008f, /* 1.3 ^ 78 */ 1.003514e+009f, /* 1.3 ^ 79 */ 1.304569e+009f, /* 1.3 ^ 80 */ 1.695939e+009f, /* 1.3 ^ 81 */ 2.204721e+009f, /* 1.3 ^ 82 */ 2.866137e+009f, /* 1.3 ^ 83 */ 3.725978e+009f, /* 1.3 ^ 84 */ 4.843771e+009f, /* 1.3 ^ 85 */ 6.296902e+009f, /* 1.3 ^ 86 */ 8.185972e+009f, /* 1.3 ^ 87 */ 1.064176e+010f, /* 1.3 ^ 88 */ 1.383429e+010f, /* 1.3 ^ 89 */ 1.798458e+010f, /* 1.3 ^ 90 */ 2.337995e+010f, /* 1.3 ^ 91 */ 3.039394e+010f, /* 1.3 ^ 92 */ 3.951212e+010f, /* 1.3 ^ 93 */ 5.136575e+010f, /* 1.3 ^ 94 */ 6.677547e+010f, /* 1.3 ^ 95 */ 8.680811e+010f, /* 1.3 ^ 96 */ 1.128505e+011f, /* 1.3 ^ 97 */ 1.467057e+011f, /* 1.3 ^ 98 */ 1.907174e+011f, /* 1.3 ^ 99 */ 2.479326e+011f, /* 1.3 ^ 100 */ 3.223124e+011f, /* 1.3 ^ 101 */ 4.190061e+011f, /* 1.3 ^ 102 */ 5.447079e+011f, /* 1.3 ^ 103 */ 7.081202e+011f, /* 1.3 ^ 104 */ 9.205562e+011f, /* 1.3 ^ 105 */ 1.196723e+012f, /* 1.3 ^ 106 */ 1.555740e+012f, /* 1.3 ^ 107 */ 2.022462e+012f, /* 1.3 ^ 108 */ 2.629200e+012f, /* 1.3 ^ 109 */ 3.417960e+012f, /* 1.3 ^ 110 */ 4.443348e+012f, /* 1.3 ^ 111 */ 5.776352e+012f, /* 1.3 ^ 112 */ 7.509257e+012f, /* 1.3 ^ 113 */ 9.762035e+012f, /* 1.3 ^ 114 */ 1.269064e+013f, /* 1.3 ^ 115 */ 1.649784e+013f, /* 1.3 ^ 116 */ 2.144719e+013f, /* 1.3 ^ 117 */ 2.788134e+013f, /* 1.3 ^ 118 */ 3.624574e+013f, /* 1.3 ^ 119 */ 4.711947e+013f, /* 1.3 ^ 120 */ 6.125531e+013f, /* 1.3 ^ 121 */ 7.963189e+013f, /* 1.3 ^ 122 */ 1.035215e+014f, /* 1.3 ^ 123 */ 1.345779e+014f, /* 1.3 ^ 124 */ 1.749512e+014f, /* 1.3 ^ 125 */ 2.274366e+014f, /* 1.3 ^ 126 */ 2.956676e+014f, /* 1.3 ^ 127 */ 3.843679e+014f /* 1.3 ^ 128 */ }; #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static void ET9LOCALCALL __CalculateTempTotWordFreq(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord) { #ifdef ET9_USE_FLOAT_FREQS ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { pWord->Body.xTotFreq = pWord->Body.xWordFreq; return; } { const ET9FREQPART xFinalTapFreq = (ET9FREQPART)(pWord->Body.xScaledTapFreq * pWord->Body.xScaledTapFreq); const ET9INT snMaxCorrectionExp = 128; const ET9INT snMinCorrectionExp = -128; ET9INT snCorrectionExp; ET9FREQPART xCorrectionFactor; snCorrectionExp = 64 - pWord->Body.snCorrectionCost; if (snCorrectionExp < snMinCorrectionExp) { snCorrectionExp = snMinCorrectionExp; } else if (snCorrectionExp > snMaxCorrectionExp) { snCorrectionExp = snMaxCorrectionExp; } xCorrectionFactor = pfCorrectionData[snCorrectionExp + snMaxCorrectionExp]; { const ET9FREQPART xInputScore = (ET9FREQPART)(xCorrectionFactor * xFinalTapFreq); const ET9FREQPART xWordScore = (pWord->Base.bIsTerm || pWord->Body.bCollectionPrio) ? (ET9FREQPART)ET9_DB_MAX_FREQ : pWord->Body.xWordFreq; pWord->Body.xTotFreq = (ET9FREQ)(xInputScore * xWordScore); ET9AssertLog(pWord->Body.xTotFreq); WLOG2B(fprintf(pLogFile2, "__CalculateTempTotWordFreq, xTotFreq %3.1f = xInputScore %3.1f * xWordScore %3.1f\n", pWord->Body.xTotFreq, xInputScore, xWordScore);) } } #else ET9_UNUSED(pLingInfo); pWord->Body.xTotFreq = (ET9FREQ)(pWord->Body.xTapFreq * pWord->Body.xWordFreq); #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __InheritLangProps(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pTarget, ET9AWPrivWordInfo const * const pDonor) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U32 dwPreviousLDBNum = pLingCmnInfo->Private.dwCurrActiveLanguage; ET9BOOL bInheritWordIndex = 0; if ((pDonor->Base.bIsTerm > pTarget->Base.bIsTerm && pDonor->Body.bWordQuality != SHAPED_QUALITY) || (pDonor->Base.bIsTerm == pTarget->Base.bIsTerm && pDonor->Body.bWordQuality > pTarget->Body.bWordQuality) || (pDonor->Base.bIsTerm == pTarget->Base.bIsTerm && pDonor->Body.bWordQuality == pTarget->Body.bWordQuality && pDonor->Body.bWordDesignation < pTarget->Body.bWordDesignation) || (pDonor->Base.bIsTerm == pTarget->Base.bIsTerm && pDonor->Body.bWordQuality == pTarget->Body.bWordQuality && pDonor->Body.bWordDesignation == pTarget->Body.bWordDesignation && ISGENUINESRC(pDonor->Body.bWordSrc) && !ISGENUINESRC(pTarget->Body.bWordSrc))) { bInheritWordIndex = 1; pTarget->Body.bLangIndexScoring = pDonor->Body.bLangIndexScoring; } else if (!pTarget->Body.dwWordIndex && pDonor->Body.dwWordIndex) { bInheritWordIndex = 1; pTarget->Body.bLangIndexScoring = pDonor->Body.bLangIndexScoring; } else if (pTarget->Body.dwWordIndex && !pDonor->Body.dwWordIndex) { /* no op */ } else if (pTarget->Body.bLangIndexScoring == pDonor->Body.bLangIndexScoring) { /* no op */ } else if (!ET9AW_GetBilingualSupported(pLingInfo)) { /* no op */ } else if (pTarget->Body.bLangIndexScoring == ET9AWUNKNOWN_LANGUAGE) { bInheritWordIndex = 1; pTarget->Body.bLangIndexScoring = pDonor->Body.bLangIndexScoring; } else if (pTarget->Body.bLangIndexScoring == ET9AWFIRST_LANGUAGE && pDonor->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) { if (dwPreviousLDBNum == pLingCmnInfo->dwSecondLdbNum) { bInheritWordIndex = 1; } pTarget->Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; } else if (pTarget->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE && pDonor->Body.bLangIndexScoring == ET9AWFIRST_LANGUAGE) { if (dwPreviousLDBNum == pLingCmnInfo->dwFirstLdbNum) { bInheritWordIndex = 1; } pTarget->Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; } if (bInheritWordIndex) { pTarget->Body.dwWordIndex = pDonor->Body.dwWordIndex; pTarget->Body.xWordFreqOrg = pDonor->Body.xWordFreqOrg; pTarget->Body.wPrefixLen = pDonor->Body.wPrefixLen; } /* "additive" public lang index */ if (pTarget->Body.bWordQuality < pDonor->Body.bWordQuality) { pTarget->Base.bLangIndex = pDonor->Base.bLangIndex; } else if (pTarget->Body.bWordQuality == pDonor->Body.bWordQuality) { if (pTarget->Base.bIsTerm < pDonor->Base.bIsTerm) { pTarget->Base.bLangIndex = pDonor->Base.bLangIndex; } else if (pTarget->Base.bIsTerm == pDonor->Base.bIsTerm) { if (pTarget->Body.bWordDesignation > pDonor->Body.bWordDesignation) { pTarget->Base.bLangIndex = pDonor->Base.bLangIndex; } else if (pTarget->Body.bWordDesignation == pDonor->Body.bWordDesignation) { if (!ISGENUINESRC(pTarget->Body.bWordSrc) && ISGENUINESRC(pDonor->Body.bWordSrc)) { pTarget->Base.bLangIndex = pDonor->Base.bLangIndex; } else if (ISGENUINESRC(pTarget->Body.bWordSrc) == ISGENUINESRC(pDonor->Body.bWordSrc)) { if (pTarget->Base.bLangIndex == ET9AWUNKNOWN_LANGUAGE) { pTarget->Base.bLangIndex = pDonor->Base.bLangIndex; } else if (pDonor->Base.bLangIndex == ET9AWUNKNOWN_LANGUAGE) { } else if (pTarget->Base.bLangIndex != pDonor->Base.bLangIndex) { pTarget->Base.bLangIndex = ET9AWBOTH_LANGUAGES; } } } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWSelLstCheckDups(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9BOOL bIsNWP = (ET9BOOL)!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs; ET9U8 bOldAmbigLangIndex; ET9U32 dwOldAmbigWordIndex; const ET9U8 bOldWordLangIndex = pWord->Body.bLangIndexScoring; ET9BOOL bOldAmbigIsDLM; const ET9U8 bOldWordIsDLM = (ET9BOOL)(GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM); ET9BOOL bCopyFreq = 0; ET9BOOL bDonorWon; ET9AWPrivWordInfo *pAmbigWord; ET9AssertLog(pWord); ET9AssertLog(pLingInfo); WLOG2BWord(pLogFile2, "__ET9AWSelLstCheckDups, pWord", pWord); /* copy initial scoring lang index to public one */ ET9Assert(pWord->Base.bLangIndex == 0xcc); pWord->Base.bLangIndex = pWord->Body.bLangIndexScoring; /* mark the word with quality info (origin) also deal with collapsing shortcuts (completion or spc) */ switch (GETRAWSRC(pWord->Body.bWordSrc)) { case ET9WORDSRC_LDB: case ET9WORDSRC_DLM: case ET9WORDSRC_MDB: case ET9WORDSRC_CDB: case ET9WORDSRC_CSP: case ET9WORDSRC_GDB: pWord->Body.bWordQuality = GENUINE_QUALITY; break; case ET9WORDSRC_LAS_SHORTCUT: case ET9WORDSRC_ASDB_SHORTCUT: if ((pWord->Base.wWordCompLen || pWord->Body.bHasPrimEditDist) && !bIsNWP) { WLOG2(fprintf(pLogFile2, "++ wWordCompLen %u, bHasPrimEditDist %u, bIsNWP %u, bWordSrc %u, raw bWordSrc %u\n", pWord->Base.wWordCompLen, pWord->Body.bHasPrimEditDist, bIsNWP, pWord->Body.bWordSrc, GETRAWSRC(pWord->Body.bWordSrc));) pWord->Body.bWordQuality = STEM_QUALITY; pWord->Body.bWordSrc = ET9WORDSRC_STEMPOOL; /* lower prio - less space */ WLOG2Word(pLogFile2, "__ET9AWSelLstCheckDups, AS into stem pool", pWord); } else if (!pWord->Base.wSubstitutionLen) { pWord->Body.bWordQuality = GENUINE_QUALITY; } else { pWord->Body.bWordQuality = LIMITED_QUALITY; } break; case ET9WORDSRC_RUDB: if (pWord->Body.bIsUDBWord) { pWord->Body.bWordQuality = GENUINE_QUALITY; } else { pWord->Body.bWordQuality = DISPOSABLE_QUALITY; pWord->Body.xWordFreqSec = pWord->Body.xWordFreq; pWord->Body.xWordFreq = 1; if (!pLingCmnInfo->Private.bUsingLM) { const ET9FREQPART xSuppDbBaseFreqUni = ET9_SUPP_DB_BASE_FREQ; if (pWord->Body.xWordFreqSec > xSuppDbBaseFreqUni) { pWord->Body.xWordFreqSec -= xSuppDbBaseFreqUni; } pWord->Body.xWordFreq = pWord->Body.xWordFreqSec; } } break; case ET9WORDSRC_STEMPOOL: if (pLingCmnInfo->Private.bStemsAllowed) { pWord->Body.bWordQuality = STEM_QUALITY; } else { pWord->Body.bWordQuality = DISPOSABLE_QUALITY; } break; default: pWord->Body.bWordQuality = SHAPED_QUALITY; break; } /* search list for a match */ __ET9AWIsWordInList(pLingCmnInfo, pWord, &pAmbigWord); if (!pAmbigWord) { return ET9STATUS_NONE; /* no match found */ } /* match found */ ET9AssertLog(pWord->Base.wWordLen == pAmbigWord->Base.wWordLen); WLOG2Word(pLogFile2, "dups: donor I", pWord); WLOG2Word(pLogFile2, "dups: target I", pAmbigWord); bOldAmbigLangIndex = pAmbigWord->Body.bLangIndexScoring; dwOldAmbigWordIndex = pAmbigWord->Body.dwWordIndex; bOldAmbigIsDLM = (ET9BOOL)(GETRAWSRC(pAmbigWord->Body.bWordSrc) == ET9WORDSRC_DLM); ET9AssertLog((pAmbigWord->Body.bWordQuality == DUPE_QUALITY) == (pWord->Body.bWordQuality == DUPE_QUALITY)); __WordOrder_UpdateOut(pLingCmnInfo, pAmbigWord, 0, 1); /* start by back propagating some attributes for consistency etc */ { /* stem distance for the same "word" can vary depending on how they got built, but the insert logic expects them to be the same for non terms. (another problem would be if a stem ever wins over a term...) */ if (!pWord->Base.bIsTerm && pWord->Body.bEditDistStem != pAmbigWord->Body.bEditDistStem) { WLOG2(fprintf(pLogFile2, "dups: adjusting donor's stem distance (stems)\n");) pWord->Body.bEditDistStem = pAmbigWord->Body.bEditDistStem; } /* when a term gets truncated (won't fit the word size) there is a rare case when it will turn into a stem (without edit distance) and possibly dupe with a spell correction that got to the same result. This will cause the record keeping of spell correction count to go wrong, since a word will inherit the spell correction property without pushing another candidate out. This special case will take out that spell correction value. There is also a case when flipping convert-symb during a build, that will cause this to happen for terms when there is a flush present. */ if (!pAmbigWord->Body.bHasPrimEditDist && pWord->Body.bHasPrimEditDist) { WLOG2(fprintf(pLogFile2, "dups: adjusting donor's edit distance (etc)\n");) pWord->Body.bEditDistSpc = pAmbigWord->Body.bEditDistSpc; pWord->Body.bEditDistStem = pAmbigWord->Body.bEditDistStem; pWord->Body.bEditDistFree = pAmbigWord->Body.bEditDistFree; pWord->Body.bEditDistSpcSbt = pAmbigWord->Body.bEditDistSpcSbt; pWord->Body.bEditDistSpcTrp = pAmbigWord->Body.bEditDistSpcTrp; pWord->Body.bEditDistSpcIns = pAmbigWord->Body.bEditDistSpcIns; pWord->Body.bEditDistSpcDel = pAmbigWord->Body.bEditDistSpcDel; pWord->Body.bHasPrimEditDist = pAmbigWord->Body.bHasPrimEditDist; pWord->Base.bIsSpellCorr = pAmbigWord->Base.bIsSpellCorr; } /* or else the prio test below will be no good */ if (ISEXACTSRC(pAmbigWord->Body.bWordSrc)) { pWord->Body.bWordSrc |= EXACTOFFSET; } } bDonorWon = (__PriorityCompare(pLingCmnInfo, pWord, pAmbigWord, SS_Collecting) > 0) ? 1 : 0; /* transfer tap freq and correction cost values - keep all the best */ if (pAmbigWord->Body.xTapFreq < pWord->Body.xTapFreq) { pAmbigWord->Body.xTapFreq = pWord->Body.xTapFreq; } if (pAmbigWord->Body.xScaledTapFreq < pWord->Body.xScaledTapFreq) { pAmbigWord->Body.xScaledTapFreq = pWord->Body.xScaledTapFreq; } if (pAmbigWord->Body.snCorrectionCost > pWord->Body.snCorrectionCost) { pAmbigWord->Body.snCorrectionCost = pWord->Body.snCorrectionCost; } if (pAmbigWord->Body.xSymbDist > pWord->Body.xSymbDist) { pAmbigWord->Body.xSymbDist = pWord->Body.xSymbDist; } /* inherit lang props */ __InheritLangProps(pLingInfo, pAmbigWord, pWord); /* there is a duplicate, if the duplicate is with the exact word, set bExactInList, but do not remove. Also, mark its source as SAMEASEXACT. It will tell us that an object matching the exact input is in the list, and the list processing at the end of __ET9AWDoSelLstBuild() will set this as the default object. Exceptions to this processing happen for numeric strings, words of length one, and words with completions. */ if (ISEXACTSRC(pAmbigWord->Body.bWordSrc)) { WLOG2(fprintf(pLogFile2, "dups: dupe with the exact\n");) if (pWord->Base.bIsTerm && ISBUILDABLESRC(pWord->Body.bWordSrc)) { pLingCmnInfo->Private.sWordC.pCurrC->dwStateBits |= ET9SLBUILDABLEEXACT; } if (pWord->Base.bIsTerm && (ISREQUIREDSRC(pWord->Body.bWordSrc) || pWord->Base.wWordLen != 1 || !_ET9_IsNumericString(pWord->Base.sWord, 1))) { /* pAmbigWord should have the src updated if it didn't have a proper one from the beginning or if the newer one is an improvement (or should it not? used to be always) used to NOT allow any numeric strings to win here, now all allowed exception 1) single digits are disallowed */ if (ISREALSRC(pWord->Body.bWordSrc) && !(ISPUNCTSRC(pWord->Body.bWordSrc) && pWord->Base.sWord[pWord->Base.wWordLen-1] != _ET9_GetTermPunctChar(pLingInfo, pLingCmnInfo->dwLdbNum, 0)) && /* this is the correct LDB to use */ (GETBASESRC(pAmbigWord->Body.bWordSrc) == ET9WORDSRC_NONE || bDonorWon)) { WLOG2(fprintf(pLogFile2, "dups: exact picks up attributes (lost)\n");) bCopyFreq = 1; pAmbigWord->Body.bWordSrc = (ET9U8)(GETBASESRC(pWord->Body.bWordSrc) | EXACTOFFSET); } } } else if (bDonorWon) { WLOG2(fprintf(pLogFile2, "dups: donor won\n");) /* pWord will never donate a stem attribute to a term, thus it must be adjusted to not cause a false chunk counter update */ ET9AssertLog(pWord->Base.bIsTerm && !pWord->Base.wWordCompLen || !pWord->Base.bIsTerm); if (pAmbigWord->Base.bIsTerm && !pWord->Base.bIsTerm) { WLOG2(fprintf(pLogFile2, "dups: adjusting donor stem to term\n");) pWord->Base.bIsTerm = 1; pWord->Base.wWordCompLen = 0; } if (pAmbigWord->Base.wWordCompLen < pWord->Base.wWordCompLen) { WLOG2(fprintf(pLogFile2, "dups: adjusting donor compl len to %u (from %u)\n", pAmbigWord->Base.wWordCompLen, pWord->Base.wWordCompLen);) pWord->Base.wWordCompLen = pAmbigWord->Base.wWordCompLen; } /* since we are keeping track of counts we must update that info here as well */ __ET9UpdateInsertCounters(pLingCmnInfo, pAmbigWord, pWord); /* copy freq info */ bCopyFreq = 1; /* propagate spc info */ pAmbigWord->Body.bEditDistSpc = pWord->Body.bEditDistSpc; pAmbigWord->Body.bEditDistStem = pWord->Body.bEditDistStem; pAmbigWord->Body.bEditDistFree = pWord->Body.bEditDistFree; pAmbigWord->Body.bEditDistSpcSbt = pWord->Body.bEditDistSpcSbt; pAmbigWord->Body.bEditDistSpcTrp = pWord->Body.bEditDistSpcTrp; pAmbigWord->Body.bEditDistSpcIns = pWord->Body.bEditDistSpcIns; pAmbigWord->Body.bEditDistSpcDel = pWord->Body.bEditDistSpcDel; pAmbigWord->Body.bHasPrimEditDist = pWord->Body.bHasPrimEditDist; pAmbigWord->Base.bIsSpellCorr = pWord->Base.bIsSpellCorr; /* This function may call two words a dup, even if the words have different completion lengths (see comments above, at top of loop), but only if word completion is off. If the word completion lengths are not the same, and a terminal word matches the stem of a longer word in the selection list, the longer word is replaced with the terminal word. If the word we are comparing two completion words with the same stem, it is not necessary to replace the characters to match the winning word, only its source and frequency data, since the completion will be removed in post processing. */ pAmbigWord->Base.bIsTerm = pWord->Base.bIsTerm; pAmbigWord->Base.wWordCompLen = pWord->Base.wWordCompLen; } /* transfer "gray" before changing word source */ if (pAmbigWord->Body.bIsGray || pWord->Body.bIsGray) { const ET9BOOL bAmbigIsUser = (ET9BOOL)(GETRAWSRC(pAmbigWord->Body.bWordSrc) == ET9WORDSRC_DLM || GETRAWSRC(pAmbigWord->Body.bWordSrc) == ET9WORDSRC_RUDB); const ET9BOOL bWordIsUser = (ET9BOOL)(GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_RUDB); if (bAmbigIsUser && bWordIsUser) { if (pWord->Body.bIsGray) { pAmbigWord->Body.bIsGray = pWord->Body.bIsGray; } } else if (bAmbigIsUser) { } else if (bWordIsUser) { if (pWord->Body.bIsGray || !(pWord->Body.bIsWeak || pWord->Body.bIsQuarantine || pWord->Body.bIsHidden)) { pAmbigWord->Body.bIsGray = pWord->Body.bIsGray; } } else { if (pWord->Body.bIsGray) { pAmbigWord->Body.bIsGray = pWord->Body.bIsGray; } } } /* transfer "black" before changing word source */ if (pAmbigWord->Body.bIsBlack || pWord->Body.bIsBlack) { const ET9BOOL bAmbigIsUser = (ET9BOOL)(GETRAWSRC(pAmbigWord->Body.bWordSrc) == ET9WORDSRC_DLM || GETRAWSRC(pAmbigWord->Body.bWordSrc) == ET9WORDSRC_RUDB); const ET9BOOL bWordIsUser = (ET9BOOL)(GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_RUDB); if (bAmbigIsUser && bWordIsUser) { if (pWord->Body.bIsBlack) { pAmbigWord->Body.bIsBlack = pWord->Body.bIsBlack; } } else if (bAmbigIsUser) { } else if (bWordIsUser) { if (pWord->Body.bIsBlack || !(pWord->Body.bIsWeak || pWord->Body.bIsQuarantine || pWord->Body.bIsHidden)) { pAmbigWord->Body.bIsBlack = pWord->Body.bIsBlack; } } else { if (pWord->Body.bIsBlack) { pAmbigWord->Body.bIsBlack = pWord->Body.bIsBlack; } } } /* transfer prefix len */ if (ISSPOTSRC(pAmbigWord->Body.bWordSrc) && !ISSPOTSRC(pWord->Body.bWordSrc)) { pAmbigWord->Body.wPrefixLen = pWord->Body.wPrefixLen; } /* transfer word source */ if (!ISEXACTSRC(pAmbigWord->Body.bWordSrc)) { const ET9U8 bWordBaseSrc = GETBASESRC(pWord->Body.bWordSrc); const ET9U8 bWordAttrSrc = GETATTRSRC(pWord->Body.bWordSrc); const ET9U8 bAmbigBaseSrc = GETBASESRC(pAmbigWord->Body.bWordSrc); const ET9U8 bAmbigAttrSrc = GETATTRSRC(pAmbigWord->Body.bWordSrc); const ET9U8 bBaseSrc = (ISSPOTSRC(bWordBaseSrc)) ? bAmbigBaseSrc : (ISSPOTSRC(bAmbigBaseSrc)) ? bWordBaseSrc : (bAmbigBaseSrc && bAmbigBaseSrc < bWordBaseSrc && !(ISPUNCTSRC(bWordBaseSrc) && pWord->Base.wWordLen == 1)) ? bAmbigBaseSrc : bWordBaseSrc; const ET9U8 bAttrSrc = (bAmbigAttrSrc > bWordAttrSrc) ? bAmbigAttrSrc : bWordAttrSrc; const ET9U8 bNewSrc = bBaseSrc | bAttrSrc; if (pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs || bDonorWon) { pAmbigWord->Body.bWordSrc = bNewSrc; } } /* transfer origin and udb tag */ if (pAmbigWord->Body.bWordQuality < pWord->Body.bWordQuality) { pAmbigWord->Body.bWordQuality = pWord->Body.bWordQuality; } if (pWord->Body.bIsUDBWord) { pAmbigWord->Body.bIsUDBWord = pWord->Body.bIsUDBWord; } /* transfer LM order etc */ if (pAmbigWord->Body.bNLMOrder < pWord->Body.bNLMOrder) { pAmbigWord->Body.bNLMOrder = pWord->Body.bNLMOrder; } if (pAmbigWord->Body.bIsCDBTrigram < pWord->Body.bIsCDBTrigram) { pAmbigWord->Body.bIsCDBTrigram = pWord->Body.bIsCDBTrigram; } if (pAmbigWord->Body.bCollectionPrio < pWord->Body.bCollectionPrio) { pAmbigWord->Body.bCollectionPrio = pWord->Body.bCollectionPrio; } if (!pAmbigWord->Body.bIsTop5 && pWord->Body.bIsTop5) { pAmbigWord->Body.bIsTop5 = pWord->Body.bIsTop5; } { ET9BOOL bApplicable = 0; if (pAmbigWord->Body.bWordQuality < pWord->Body.bWordQuality) { bApplicable = 1; } else if (pAmbigWord->Body.bWordQuality == pWord->Body.bWordQuality) { if (pAmbigWord->Base.bIsTerm < pWord->Base.bIsTerm) { bApplicable = 1; } else if (pAmbigWord->Base.bIsTerm == pWord->Base.bIsTerm) { if (pAmbigWord->Body.bWordDesignation > pWord->Body.bWordDesignation) { bApplicable = 1; } else if (pAmbigWord->Body.bWordDesignation == pWord->Body.bWordDesignation) { if (!ISGENUINESRC(pAmbigWord->Body.bWordSrc) && ISGENUINESRC(pWord->Body.bWordSrc)) { bApplicable = 1; } else if (ISGENUINESRC(pAmbigWord->Body.bWordSrc) == ISGENUINESRC(pWord->Body.bWordSrc)) { bApplicable = 1; } } } } if (pAmbigWord->Body.bIsWeak && !pWord->Body.bIsWeak && bApplicable) { pAmbigWord->Body.bIsWeak = pWord->Body.bIsWeak; } if (pAmbigWord->Body.bIsHidden && !pWord->Body.bIsHidden && bApplicable) { pAmbigWord->Body.bIsHidden = pWord->Body.bIsHidden; } if (pAmbigWord->Body.bIsQuarantine && !pWord->Body.bIsQuarantine && bApplicable) { pAmbigWord->Body.bIsQuarantine = pWord->Body.bIsQuarantine; } } if (ISEXACTSRC(pAmbigWord->Body.bWordSrc) && !pAmbigWord->Body.bIsQuarantine && pWord->Body.bIsQuarantine) { pAmbigWord->Body.bIsQuarantine = pWord->Body.bIsQuarantine; } if (pWord->Body.bIsRetain) { pAmbigWord->Body.bIsRetain = pWord->Body.bIsRetain; } /* if word doesn't have a substitution, OR higher priority word has it's own substitution, load it in */ if (!pAmbigWord->Base.wSubstitutionLen || pWord->Base.wSubstitutionLen) { pAmbigWord->Base.wSubstitutionLen = pWord->Base.wSubstitutionLen; if (pWord->Base.wSubstitutionLen) { WLOG2(fprintf(pLogFile2, "dups: inherited substitution\n");) _ET9SymCopy(pAmbigWord->Base.sSubstitution, pWord->Base.sSubstitution, pWord->Base.wSubstitutionLen); } } /* the word must buildable for even being considered as a shortcut */ if (pAmbigWord->Base.wSubstitutionLen && !ISGENUINESRC(pAmbigWord->Body.bWordSrc)) { WLOG2(fprintf(pLogFile2, "dups: non buildable source, substitution removed\n");) pAmbigWord->Base.wSubstitutionLen = 0; } /* tap and word freq transfer */ if (bOldAmbigIsDLM || bOldWordIsDLM) { if (bOldAmbigIsDLM && bOldWordIsDLM) { if (pAmbigWord->Body.xWordFreq < pWord->Body.xWordFreq) { pAmbigWord->Body.xWordFreq = pWord->Body.xWordFreq; } } else if (bOldWordIsDLM) { pAmbigWord->Body.xWordFreq = pWord->Body.xWordFreq; } if (bCopyFreq) { if (pAmbigWord->Body.xTapFreq != pWord->Body.xTapFreq || pAmbigWord->Body.xScaledTapFreq != pWord->Body.xScaledTapFreq) { WLOG2(fprintf(pLogFile2, " dups: replacing DLM tap freq\n");) pAmbigWord->Body.xTapFreq = pWord->Body.xTapFreq; pAmbigWord->Body.xScaledTapFreq = pWord->Body.xScaledTapFreq; pAmbigWord->Body.xSymbDist = pWord->Body.xSymbDist; } } __CalculateTempTotWordFreq(pLingInfo, pAmbigWord); ET9AssertLog(pAmbigWord->Body.xTapFreq >= 0); ET9AssertLog(pAmbigWord->Body.xTotFreq >= 0); ET9AssertLog(pAmbigWord->Body.xWordFreq >= 0); #ifdef ET9_DEBUG if (!bOldAmbigIsDLM && bOldWordIsDLM) { pAmbigWord->Body.sScoreContext = pWord->Body.sScoreContext; } #endif } else if (bCopyFreq) { WLOG2(fprintf(pLogFile2, " dups: replacing freq\n");) pAmbigWord->Body.xTapFreq = pWord->Body.xTapFreq; pAmbigWord->Body.xScaledTapFreq = pWord->Body.xScaledTapFreq; pAmbigWord->Body.xTotFreq = pWord->Body.xTotFreq; pAmbigWord->Body.xWordFreq = pWord->Body.xWordFreq; pAmbigWord->Body.xWordFreqSec = pWord->Body.xWordFreqSec; pAmbigWord->Body.xSymbDist = pWord->Body.xSymbDist; } else if (!pLingCmnInfo->Private.bUsingLM && __IsUsingMixedStyle(pLingCmnInfo) && bOldAmbigLangIndex == bOldWordLangIndex && GETRAWSRC(pAmbigWord->Body.bWordSrc) == ET9WORDSRC_RUDB && !pAmbigWord->Body.bIsUDBWord && ((GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_LDB && ISGENUINESRC(pWord->Body.bWordSrc)) || (ISGENUINESRC(pWord->Body.bWordSrc) == ISGENUINESRC(pAmbigWord->Body.bWordSrc)))) { WLOG2(fprintf(pLogFile2, " dups: merging freq\n");) pAmbigWord->Body.xWordFreq = pWord->Body.xWordFreq + pAmbigWord->Body.xWordFreqSec; __CalculateTempTotWordFreq(pLingInfo, pAmbigWord); ET9AssertLog(pAmbigWord->Body.xTapFreq >= 0); ET9AssertLog(pAmbigWord->Body.xTotFreq >= 0); ET9AssertLog(pAmbigWord->Body.xWordFreq >= 0); } if (pAmbigWord->Body.bLangIndexScoring != bOldAmbigLangIndex) { WLOG2(fprintf(pLogFile2, " dups: langIndex changed, %s -> %s\n", LINDEXTOSTRING(bOldAmbigLangIndex), LINDEXTOSTRING(pAmbigWord->Body.bLangIndexScoring));) } if (pAmbigWord->Body.dwWordIndex != dwOldAmbigWordIndex) { WLOG2(fprintf(pLogFile2, " dups: word index changed, %u -> %u\n", dwOldAmbigWordIndex, pAmbigWord->Body.dwWordIndex);) } if (pLingCmnInfo->Private.bUsingLM || __IsUsingMixedStyle(pLingCmnInfo)) { ET9AssertLog(!pWord->Body.dwWordIndex || pAmbigWord->Body.dwWordIndex || pWord->Base.bIsTerm != pAmbigWord->Base.bIsTerm || ET9AW_GetBilingualSupported(pLingInfo)); } __VerifyWordCounters(pLingCmnInfo); WLOG2Word(pLogFile2, "dups: donor F", pWord); WLOG2Word(pLogFile2, "dups: target F", pAmbigWord); __WordOrder_UpdateIn(pLingCmnInfo, pAmbigWord, 0, 1); return ET9STATUS_WORD_EXISTS; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9BOOL ET9LOCALCALL __ET9PrependWord(ET9AWPrivWordInfo * const pWord, ET9AWPrivWordInfo * const pPrefixWord, const ET9U16 wLengthToPrepend) { ET9AssertLog(pPrefixWord); /* discard result if already the prefix fills the word */ if (pPrefixWord->Base.wWordLen >= ET9MAXWORDSIZE) { return 0; } /* discard result if prefix is shorter than what's to be prepended */ if (pPrefixWord->Base.wWordLen < wLengthToPrepend) { return 0; } /* prepend */ ET9AssertLog(pWord); ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); ET9AssertLog(pPrefixWord->Base.wWordLen <= ET9MAXWORDSIZE); WLOG2B(fprintf(pLogFile2, "__ET9PrependWord, wLengthToPrepend = %d\n", wLengthToPrepend);) WLOG2BWord(pLogFile2, "__ET9PrependWord, pWord (in)", pWord); WLOG2BWord(pLogFile2, "__ET9PrependWord, pPrefixWord (in)", pPrefixWord); /* adjust pWord to fit the prefix first */ { ET9U16 wWordLen = pWord->Base.wWordLen; /* truncate needed? */ if (wWordLen + wLengthToPrepend > ET9MAXWORDSIZE) { const ET9U16 wTruncateCount = wWordLen + wLengthToPrepend - ET9MAXWORDSIZE; wWordLen = (ET9U16)(wWordLen - wTruncateCount); if (pWord->Base.wWordCompLen <= wTruncateCount) { pWord->Base.wWordCompLen = 0; } else { pWord->Base.wWordCompLen = (ET9U16)(pWord->Base.wWordCompLen - wTruncateCount); } } /* move symbs up */ { ET9U16 wCount; ET9SYMB *psSrc = &pWord->Base.sWord[wWordLen - 1]; ET9SYMB *psDst = &pWord->Base.sWord[wWordLen + wLengthToPrepend - 1]; for (wCount = wWordLen; wCount; --wCount, --psSrc, --psDst) { *psDst = *psSrc; } } pWord->Base.wWordLen = (ET9U16)(wWordLen + wLengthToPrepend); } /* copy in the prefix */ _ET9SymCopy(pWord->Base.sWord, pPrefixWord->Base.sWord, wLengthToPrepend); /* propagate distance */ pWord->Body.bEditDistSpc = (ET9U8)(pWord->Body.bEditDistSpc + pPrefixWord->Body.bEditDistSpc); pWord->Body.bEditDistStem = (ET9U8)(pWord->Body.bEditDistStem + pPrefixWord->Body.bEditDistStem); /* track prefix length */ pWord->Body.wPrefixLen = wLengthToPrepend; /* done */ WLOG2BWord(pLogFile2, "__ET9PrependWord, pWord (out)", pWord); return 1; } /*--------------------------------------------------------------------------- * * Function: insert helpers * *---------------------------------------------------------------------------*/ #ifdef ET9_ACTIVATE_SLST_STATS #define __ET9AWSelLstIncInsertCounter ++pLingCmnInfo->Private.sStats.dwInsertCount; ++pLingCmnInfo->Private.sStats.dwTotInsertCount; #define __ET9AWSelLstIncDuplicateCounter ++pLingCmnInfo->Private.sStats.dwTotInsertDuplicate; #define __ET9AWSelLstIncReplacingCounter ++pLingCmnInfo->Private.sStats.dwTotInsertReplacing; #define __ET9AWSelLstIncDiscardedCounter ++pLingCmnInfo->Private.sStats.dwTotInsertDiscarded; #else #define __ET9AWSelLstIncInsertCounter #define __ET9AWSelLstIncDuplicateCounter #define __ET9AWSelLstIncReplacingCounter #define __ET9AWSelLstIncDiscardedCounter #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static void ET9LOCALCALL __UpdateStringHash(ET9AWPrivWordInfo * const pWord) { ET9U32 dwHashValue = 0; ET9UINT nCount; ET9SYMB const * psSymb; psSymb = &pWord->Base.sWord[0]; for (nCount = pWord->Base.wWordLen; nCount; --nCount, ++psSymb) { dwHashValue = *psSymb + (65599 * dwHashValue); } pWord->Body.dwStringHash = dwHashValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWSelLstInsert(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9_FREQ_DESIGNATION bFreqIndicator, const ET9BOOL bPreScreenedDupe) { ET9STATUS wStatus = ET9STATUS_NONE; ET9AWPrivWordInfo sStemWord; ET9U32 dwLdbNum; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9U16 wLockPoint = pLingCmnInfo->Private.wCurrLockPoint; const ET9U16 wLeftWordLen = pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord.Base.wWordLen; ET9AssertLog(!pLingCmnInfo->Private.bDoneBuilding); ET9AssertLog(pWord); ET9AssertLog(pLingInfo); ET9AssertLog(pWordSymbInfo); ET9AssertLog(pWord->Body.xTapFreq >= 0); ET9AssertLog(pWord->Body.xWordFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); ET9AssertLog(pWord->Body.dwWordIndex < 0xFFFFFF00); ET9AssertLog(pWord->Body.snCorrectionCost >= -5000 && pWord->Body.snCorrectionCost <= 5000); ET9AssertLog(pWord->Base.bIsTerm && !pWord->Base.wWordCompLen || !pWord->Base.bIsTerm); ET9AssertLog((!pWord->Body.bEditDistSpcSbt && !pWord->Body.bEditDistSpcTrp && !pWord->Body.bEditDistSpcIns && !pWord->Body.bEditDistSpcDel) || (pWord->Body.bEditDistSpcSbt + pWord->Body.bEditDistSpcTrp + pWord->Body.bEditDistSpcIns + pWord->Body.bEditDistSpcDel == pWord->Body.bEditDistSpc)); ET9AssertLog(pWord->Body.bLangIndexScoring != ET9AWSECOND_LANGUAGE || ET9AW_GetBilingualSupported(pLingInfo)); ET9AssertLog(!pWord->Body.bIsRetain || ISEXACTSRC(pWord->Body.bWordSrc) || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_EXACT_LAST); #if 0 { static FILE *pf = NULL; if (!pf) { pf = fopen("zSpcops.txt", "w"); } if ((pWord->Body.bEditDistSpcSbt + pWord->Body.bEditDistSpcTrp + pWord->Body.bEditDistSpcIns + pWord->Body.bEditDistSpcDel) && (pWord->Body.bEditDistSpcSbt + pWord->Body.bEditDistSpcTrp + pWord->Body.bEditDistSpcIns + pWord->Body.bEditDistSpcDel) != pWord->Body.bEditDistSpc) { static ET9UINT nCount = 0; ++nCount; fprintf(pf, "### [%4u] ops %u tot %u diff %d\n", nCount, (pWord->Body.bEditDistSpcSbt + pWord->Body.bEditDistSpcTrp + pWord->Body.bEditDistSpcIns + pWord->Body.bEditDistSpcDel), pWord->Body.bEditDistSpc, pWord->Body.bEditDistSpc - (pWord->Body.bEditDistSpcSbt + pWord->Body.bEditDistSpcTrp + pWord->Body.bEditDistSpcIns + pWord->Body.bEditDistSpcDel)); fflush(pf); } } #endif WLOG2BWord(pLogFile2, "__ET9AWSelLstInsert, pWord", pWord); /* handle substitution expansion */ if (pWord->Base.wSubstitutionLen && ET9EXPANDAUTOSUB(pLingCmnInfo)) { const ET9BOOL bIsNWP = (ET9BOOL)!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs; const ET9BOOL bSuppressSubst = (ET9BOOL)((pWord->Base.wWordCompLen || pWord->Body.bEditDistSpc > 0) && !bIsNWP); WLOG2(fprintf(pLogFile2, "expanding auto-sub, bSuppressSubst %d\n", bSuppressSubst);) WLOG2Word(pLogFile2, "pWord", pWord); if (bSuppressSubst) { pWord->Base.wSubstitutionLen = 0; } else { ET9AssertLog(pWord->Base.wSubstitutionLen <= ET9MAXWORDSIZE); _ET9SymCopy(pWord->Base.sWord, pWord->Base.sSubstitution, pWord->Base.wSubstitutionLen); pWord->Base.wWordLen = pWord->Base.wSubstitutionLen; pWord->Base.wWordCompLen = 0; pWord->Base.wSubstitutionLen = 0; if (pWord->Base.wWordLen > pLingCmnInfo->Private.wMaxWordLength && !pWord->Body.bHasPrimEditDist && !pWord->Body.bEditDistFree) { ++pWord->Body.bEditDistFree; } } } /* handle spc/free tracking */ if (pWord->Body.bEditDistSpc || pWord->Body.bEditDistFree) { pLingCmnInfo->Private.bSpcDuringBuild = 1; } /* track inserts, before dups etc, mostly used as an indication of search "success" */ if ((pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts + 1) != 0) { ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; } if (GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_LDB) { pLingCmnInfo->Private.sWordC.pCurrC->bHasRealWord = 1; } /* update insert counter */ __ET9AWSelLstIncInsertCounter; /* assign primary edit distance */ pWord->Body.bHasPrimEditDist = pWord->Body.bEditDistSpc > 0; /* truncation handling */ if (wLeftWordLen) { ET9U16 wAvailableLen; if (wLeftWordLen >= ET9MAXWORDSIZE) { WLOG2(fprintf(pLogFile2, "too long left hand side (%d >= %d)\n", wLeftWordLen, ET9MAXWORDSIZE);) return ET9STATUS_ERROR; } wAvailableLen = ET9MAXWORDSIZE - wLeftWordLen; if (pWord->Body.bHasPrimEditDist && pWord->Base.wWordLen > wAvailableLen) { WLOG2Word(pLogFile2, "discarding spell correction that would become truncated", pWord); return ET9STATUS_NONE; } if (pWord->Base.bIsTerm && pWord->Base.wWordLen > wAvailableLen) { WLOG2Word(pLogFile2, "term that would become truncated becomes a stem", pWord); pWord->Base.bIsTerm = 0; } } /* possibly remove completions */ if (!bPreScreenedDupe && !pWord->Body.bHasPrimEditDist && !pWord->Body.bEditDistFree && pWord->Base.wWordLen > pLingCmnInfo->Private.wMaxWordLength) { ET9AssertLog((pWord->Base.wWordLen - pWord->Base.wWordCompLen) == pLingCmnInfo->Private.wMaxWordLength); WLOG2B(fprintf(pLogFile2, "unwanted completion - into stem\n");) pWord->Base.wWordLen = pLingCmnInfo->Private.wMaxWordLength; pWord->Base.wWordCompLen = 0; } /* too short for lock? */ /* ET9AssertLog(wLeftWordLen + pWord->Base.wWordLen >= wLockPoint); at some point this would be good to activate... */ if (wLeftWordLen + pWord->Base.wWordLen < wLockPoint && !pLingCmnInfo->Private.sWordC.pCurrC->bPureCollection) { WLOG2Word(pLogFile2, "discarding candidate that would be shorter than lock", pWord); return ET9STATUS_NONE; } /* possibly discard stems */ if (!pWord->Base.bIsTerm && !pWord->Base.wWordCompLen && !pLingCmnInfo->Private.bStemsAllowed) { pWord->Body.bWordSrc = ET9WORDSRC_STEMPOOL; WLOG2B(fprintf(pLogFile2, "unwanted stem - into stem pool\n");) } /* clear substitution if this is a non-completed stem of an asdb shortcut */ if (!pWord->Base.bIsTerm && !pWord->Base.wWordCompLen) { pWord->Base.wSubstitutionLen = 0; } /* less wanted words into protected terms */ if (pWord->Base.wWordLen < 3 && pWordSymbInfo->bNumSymbs < 3 && !pWord->Body.bIsInternalGesture && !pWord->Body.wGestureReturnVal && _ET9_LanguageSpecific_ApplyTermDemote(pLingInfo, pWord)) { if (pWordSymbInfo->bNumSymbs == 1 && pWord->Base.bIsTerm && pWord->Base.wWordLen == 1 && pLingCmnInfo->Private.bHasRegionalInfo && pWordSymbInfo->SymbsInfo[0].bSymbType != ET9KTSMARTPUNCT && GETCOMPSRC(pWord->Body.bWordSrc) != ET9WORDSRC_EXACT) { const ET9SYMB sFirstSymb = pWord->Base.sWord[0]; if ((!pWord->Body.dwWordIndex || pWord->Body.dwWordIndex > __SINGLE_KEYPRESS_MAX_WORD_INDEX) && !(sFirstSymb == pWordSymbInfo->SymbsInfo[0].DataPerBaseSym[0].sChar[0] || sFirstSymb == pWordSymbInfo->SymbsInfo[0].DataPerBaseSym[0].sUpperCaseChar[0])) { pWord->Body.bWordSrc = ET9WORDSRC_TERM; WLOG2B(fprintf(pLogFile2, "less wanted single char word - into protected term\n");) } } if (pWordSymbInfo->bNumSymbs == 2 && pWord->Base.bIsTerm && pWord->Base.wWordLen == 2 && !pWord->Body.bIsTop5 && __IsUsingMixedStyle(pLingCmnInfo) && GETCOMPSRC(pWord->Body.bWordSrc) != ET9WORDSRC_EXACT && !((pWord->Base.sWord[0] == pWordSymbInfo->SymbsInfo[0].DataPerBaseSym[0].sChar[0] || pWord->Base.sWord[0] == pWordSymbInfo->SymbsInfo[0].DataPerBaseSym[0].sUpperCaseChar[0]) && (pWord->Base.sWord[1] == pWordSymbInfo->SymbsInfo[1].DataPerBaseSym[0].sChar[0] || pWord->Base.sWord[1] == pWordSymbInfo->SymbsInfo[1].DataPerBaseSym[0].sUpperCaseChar[0]))) { pWord->Body.bWordSrc = ET9WORDSRC_TERM; WLOG2B(fprintf(pLogFile2, "less wanted two char word - into protected term\n");) } } /* validate */ ET9AssertLog(!pWord->Base.bIsTerm || pWord->Base.bIsTerm && !pWord->Base.wWordCompLen); #ifdef ET9_DEBUG { ET9U16 wCount; ET9SYMB *pSymb; ET9AWPrivWordInfo *pLeftHandWord = &pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord; ET9AssertLog(pWord->Body.bWordSrc != ET9WORDSRC_NONE); ET9AssertLog(GETBASESRC(pWord->Body.bWordSrc) != ET9WORDSRC_CAPTURE); ET9AssertLog(pWord->Base.wWordLen || pLeftHandWord->Base.wWordLen); ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); ET9AssertLog(pWord->Base.wWordCompLen <= pWord->Base.wWordLen); ET9AssertLog(pWord->Base.wSubstitutionLen <= ET9MAXSUBSTITUTIONSIZE); ET9AssertLog(pLeftHandWord->Base.wWordLen <= ET9MAXWORDSIZE); ET9AssertLog(pLeftHandWord->Base.wWordCompLen <= pLeftHandWord->Base.wWordLen); if (pWord->Base.wWordLen < ET9MAXWORDSIZE) { pWord->Base.sWord[pWord->Base.wWordLen] = 0; } wCount = pWord->Base.wWordLen; pSymb = pWord->Base.sWord; while (wCount--) { ET9AssertLog(*pSymb && *pSymb != (ET9SYMB)0xcccc); ++pSymb; } wCount = pLeftHandWord->Base.wWordLen; pSymb = pLeftHandWord->Base.sWord; while (wCount--) { ET9AssertLog(*pSymb && *pSymb != (ET9SYMB)0xcccc); ++pSymb; } } #endif /* save some info for stem insertion, avoid copy the whole word (performance) */ #if defined(_DEBUG) && defined(ET9_DEBUG) _InitPrivWordInfo(&sStemWord); #endif if (pWord->Base.wWordCompLen) { sStemWord.Base.wWordLen = (ET9U16)(wLeftWordLen + pWord->Base.wWordLen - pWord->Base.wWordCompLen); sStemWord.Base.wWordCompLen = 0; sStemWord.Base.wSubstitutionLen = 0; sStemWord.Base.bIsTerm = 0; sStemWord.Base.bIsDeletable = 0; sStemWord.Body = pWord->Body; sStemWord.Body.bIsGroupBase = 0; sStemWord.Body.bGroupCount = 0; sStemWord.Body.bWordSrc = (pLingCmnInfo->Private.bStemsAllowed && !pWord->Body.bEditDistStem) ? ET9WORDSRC_STEM : ET9WORDSRC_STEMPOOL; #ifdef ET9_USE_FLOAT_FREQS sStemWord.Body.xTapFreq = 0.0000000001f; #else sStemWord.Body.xTapFreq = 1; #endif /* ET9_USE_FLOAT_FREQS */ ET9AssertLog(sStemWord.Body.xTapFreq); ET9AssertLog(sStemWord.Body.xTotFreq); ET9AssertLog(sStemWord.Body.xWordFreq); } else { sStemWord.Body.bWordSrc = 0; /* silly compiler warning... */ sStemWord.Base.wWordLen = 0; /* silly compiler warning... */ } /* update word source */ if (bPreScreenedDupe) { } else if (bFreqIndicator == FREQ_BUILDAROUND) { if (pWord->Body.bWordSrc > ET9WORDSRC_SIMPLE_START && pWord->Body.bWordSrc < ET9WORDSRC_BUILDAROUND_START) { pWord->Body.bWordSrc += BUILDAROUND_SRC_ADDON; } } else if (bFreqIndicator == FREQ_BUILDCOMPOUND) { if (pWord->Body.bWordSrc > ET9WORDSRC_SIMPLE_START && pWord->Body.bWordSrc < ET9WORDSRC_BUILDAROUND_START) { pWord->Body.bWordSrc += BUILDCOMPOUND_SRC_ADDON; } } /* setup and save word designation */ pWord->Body.bWordDesignation = bFreqIndicator; /* prepend lefthandword */ pWord->Body.wPrefixLen = 0; if (wLeftWordLen && !bPreScreenedDupe && !pWord->Body.bIsSegment) { if (!__ET9PrependWord(pWord, &pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord, wLeftWordLen)) { return ET9STATUS_ERROR; } } /* update spc info - not just primary (after prepend) */ pWord->Base.bIsSpellCorr = pWord->Body.bEditDistSpc > 0; ET9AssertLog(pWord->Body.bEditDistSpc && pWord->Base.bIsSpellCorr || !pWord->Body.bEditDistSpc && !pWord->Base.bIsSpellCorr); ET9AssertLog(!pWord->Base.bIsSpellCorr || pLingCmnInfo->Private.bSpcDuringBuild); /* handle capslock and shift of prediction and capslock of completion */ dwLdbNum = (pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; if (pWord->Base.wWordCompLen && _ET9_LanguageSpecific_ApplyShifting(pLingInfo, pWord)) { if (pWord->Base.wWordCompLen == pWord->Base.wWordLen && /* prediction */ pWordSymbInfo->Private.eLastShiftState == ET9SHIFT) { pWord->Base.sWord[0] = _ET9SymToUpper(pWord->Base.sWord[0], dwLdbNum); } else if (pWordSymbInfo->Private.eLastShiftState == ET9CAPSLOCK) { ET9SYMB *psSymb; ET9UINT nComplLen; psSymb = &pWord->Base.sWord[pWord->Base.wWordLen - 1]; nComplLen = pWord->Base.wWordCompLen; while (nComplLen--) { *psSymb = _ET9SymToUpper(*psSymb, dwLdbNum); --psSymb; } } } #if 0 /* this code is unused uless we reinstate substitutions on prediction */ /* Handle capslock and shift for prediction substitution */ if (pWord->Base.wSubstitutionLen && (pWord->Base.wWordCompLen == pWord->Base.wWordLen )) { /* if all syms shifted, do same for substitution */ pSymb = pWord->Base.sSubstitution; if (pAlpSelList->Private.pWordSymbInfo->Private.bLastShiftState == ET9CAPSLOCK) { nComplLen = pWord->Base.wSubstitutionLen; while (nComplLen--) { *pSymb = _SymToUpper(*pSymb); ++pSymb; } } /* if first sym shifted, do same for substitution */ else if (pAlpSelList->Private.pWordSymbInfo->Private.bLastShiftState == ET9SHIFT) { *pSymb = _SymToUpper(*pSymb); } } #endif /* on-the-fly (before __ET9AWIsLikeExactMatch & __ET9AWSelLstCheckDups) */ if (pLingInfo->Private.pConvertSymb && !bPreScreenedDupe) { ET9STATUS wConvert; WLOG2BWord(pLogFile2, "__ET9AWSelLstInsert, before OTFM, pWord", pWord); wConvert = _ET9_ConvertBuildBuf(pLingInfo, &pWord->Base); /* host provided no conversion for a character, skip this word (unless it's the exact) */ if (wConvert == ET9STATUS_NO_CHAR || wConvert == ET9STATUS_ERROR) { if (ISEXACTSRC(pWord->Body.bWordSrc)) { WLOG2Word(pLogFile2, "host tried to prevent the exact to enter the list, not accepted...", pWord); } else { return ET9STATUS_ERROR; } } } #ifdef ET9_DEBUG /* verify locked chars */ if (wLockPoint && !pLingCmnInfo->Private.sWordC.pCurrC->bPureCollection) { ET9UINT nIndex; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9UINT nSymbIndex; if (GETBASESRC(pWord1->Body.bWordSrc) == ET9WORDSRC_MAGICSTRING) { continue; } for (nSymbIndex = 0; nSymbIndex < pWordSymbInfo->bNumSymbs; ++nSymbIndex) { ET9SymbInfo const * const pSymbInfo = &pWordSymbInfo->SymbsInfo[nSymbIndex]; if (pSymbInfo->bCurrBuildLocked) { if (nSymbIndex >= pWord1->Base.wWordLen) { WLOG2(fprintf(pLogFile2, "word too short for lock\n");) ET9AssertLog(0); continue; } else { const ET9SYMB sSymb = pWord1->Base.sWord[nSymbIndex]; ET9SYMB sLockedSymb = pSymbInfo->sLockedSymb; if (pLingInfo->Private.pConvertSymb != NULL) { pLingInfo->Private.pConvertSymb(pLingInfo->Private.pConvertSymbInfo, &sLockedSymb); } /* this is going into the list, must be exact match to converted lock */ if (sSymb != sLockedSymb && !pLingCmnInfo->Private.bExpandAsDuringBuild) { WLOG2Word(pLogFile2, "word verified", pWord1); WLOG2(fprintf(pLogFile2, "lock symb missmatch index = %u, pos = %d, locked symb = %x\n", nIndex, nSymbIndex, sLockedSymb);) } ET9AssertLog(sSymb == sLockedSymb || pLingCmnInfo->Private.bExpandAsDuringBuild); } } } } } #endif /* assign string hash value before any list search */ __UpdateStringHash(pWord); /* all completions should also have the stem go into the selection list */ if (pWord->Base.wWordCompLen && (!pLingCmnInfo->Private.sWordC.pCurrC->bHasRealWord || pLingCmnInfo->Private.bStemsAllowed) && (sStemWord.Body.bWordSrc != ET9WORDSRC_STEMPOOL || pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) && sStemWord.Base.wWordLen && sStemWord.Base.wWordLen <= ET9MAXWORDSIZE) { /* at this point pWord already holds a properly prepended and converted word, can be used for dupe supression */ ET9AWPrivWordInfo *pDupeWord; { const ET9U16 wOldWordLen = pWord->Base.wWordLen; const ET9U32 dwOldStringHash = pWord->Body.dwStringHash; pWord->Base.wWordLen = sStemWord.Base.wWordLen; __UpdateStringHash(pWord); __ET9AWIsWordInList(pLingCmnInfo, pWord, &pDupeWord); pWord->Base.wWordLen = wOldWordLen; pWord->Body.dwStringHash = dwOldStringHash; } if (pDupeWord) { if (pDupeWord->Body.bWordQuality <= DISPOSABLE_QUALITY) { if (sStemWord.Body.bWordSrc != ET9WORDSRC_STEMPOOL || pLingCmnInfo->Private.bStemsAllowed) { pDupeWord->Body.bWordQuality = STEM_QUALITY; } } } else { switch (sStemWord.Body.bWordSrc) { case ET9WORDSRC_STEMPOOL: if (pLingCmnInfo->Private.bStemsAllowed) { sStemWord.Body.bWordQuality = STEM_QUALITY; } else { sStemWord.Body.bWordQuality = DISPOSABLE_QUALITY; } break; default: sStemWord.Body.bWordQuality = SHAPED_QUALITY; break; } /* pWord holds converted, prepended etc symbs, good to go directly... */ _ET9SymCopy(sStemWord.Base.sWord, pWord->Base.sWord, sStemWord.Base.wWordLen); WLOG2BWord(pLogFile2, "__ET9AWSelLstInsert, attempting completion's stem", &sStemWord); __ET9AWSelLstInsert(pLingInfo, &sStemWord, bFreqIndicator, 1); WLOG2B(fprintf(pLogFile2, "__ET9AWSelLstInsert, done with stem attempt\n");) } } /* check if the substitution is the same as the shortcut (also without OTFM to be sure it never happens)) */ if (pWord->Base.wSubstitutionLen == pWord->Base.wWordLen) { ET9U16 wCmpLen; ET9SYMB *pStr1; ET9SYMB *pStr2; wCmpLen = pWord->Base.wWordLen; pStr1 = pWord->Base.sWord; pStr2 = pWord->Base.sSubstitution; ++wCmpLen; while (--wCmpLen) { if (*pStr1++ != *pStr2++) { break; } } if (!wCmpLen) { WLOG2Word(pLogFile2, "word will be rejected, substitution same as shortcut", pWord); return ET9STATUS_NONE; } } /* duplicate suppression */ if (!bPreScreenedDupe) { wStatus = __ET9AWSelLstCheckDups(pLingInfo, pWord); if (wStatus) { __ET9AWSelLstIncDuplicateCounter; return wStatus; } } /* post rejection of special cases */ /* reject spc words that ends in punct when the symb info ends in punct */ if (pWordSymbInfo->bNumSymbs && pWord->Body.bHasPrimEditDist && _ET9_IsPunctChar(pWord->Base.sWord[pWord->Base.wWordLen - 1])) { ET9U16 wLastSymbIndex = pWordSymbInfo->bNumSymbs - 1; ET9U8 bSymbType = pWordSymbInfo->SymbsInfo[wLastSymbIndex].bSymbType; if (bSymbType == ET9KTSMARTPUNCT || bSymbType == ET9KTPUNCTUATION) { WLOG2Word(pLogFile2, "__ET9PostWordRejection, spc punct end", pWord); --pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; /* will never become zero, that's the important part*/ __ET9AWSelLstIncDiscardedCounter; return ET9STATUS_NONE; } } /* reject completions (non terms) from secondary language */ if ((pWord->Base.wWordCompLen && pWord->Base.wWordCompLen < pWord->Base.wWordLen) && pWord->Body.bLangIndexScoring == pLingCmnInfo->Private.bNonActiveIndex) { WLOG2Word(pLogFile2, "__ET9PostWordRejection, secondary completion", pWord); --pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; /* will never become zero, that's the important part*/ __ET9AWSelLstIncDiscardedCounter; return ET9STATUS_NONE; } /* flags */ if (pWord->Body.bHasPrimEditDist && pWord->Base.wWordCompLen) { pLingCmnInfo->Private.bSpcComplDuringSingleBuild = 1; } /* -------------------- actual insert attempt --------------------------*/ /* validation - protected sources have neither completion nor distance nor stem distance */ WLOG2Word(pLogFile2, "__ET9AWSelLstInsert, attempting, pWord", pWord); ET9AssertLog(pWord->Base.bIsTerm != 0xc); ET9AssertLog(pWord->Base.bIsTerm != 0xcc); ET9AssertLog(!ISPROTECTEDWRD(pWord) || pWord->Body.bWordSrc == ET9WORDSRC_TERM || pWord->Body.bWordSrc == ET9WORDSRC_STEM || pWord->Body.bIsGray || pWord->Body.bIsBlack || !pWord->Body.bEditDistSpc && !pWord->Body.bEditDistStem && pWord->Base.bIsTerm); /* try adding it to the list, assume validations above */ if (pWord->Body.bHasPrimEditDist && pWord->Base.wWordCompLen && pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords >= pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount && pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) { /* spc cmpl word and spc cmpl part of list is full, discard or replace */ if (!pLingCmnInfo->Private.bSpcComplDuringSingleBuild) { ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords == pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount); } if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord = __WordOrder_FindMinPriorityConditional(pLingCmnInfo, __WordOrder_SpcCmplCOND); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord); ET9AssertLog(!ISPROTECTEDWRD(pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord)); } if (__PriorityCompare(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord, pWord, SS_Collecting) > 0) { ET9AssertLog(!ISPROTECTEDWRD(pWord)); __ET9AWSelLstIncDiscardedCounter; return wStatus; } __ET9AWSelLstIncReplacingCounter; __ET9ReplaceAndUpdateInsertCounters(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcCmplWord, pWord); } else if (pWord->Body.bHasPrimEditDist && pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords >= pLingCmnInfo->Private.ASpc.nMaxSpcTermCount && pLingCmnInfo->Private.ASpc.nMaxSpcTermCount < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) { /* spc term word and spc term part of list is full, discard or replace */ if (!pLingCmnInfo->Private.bSpcComplDuringSingleBuild) { ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords == pLingCmnInfo->Private.ASpc.nMaxSpcTermCount); } if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord = __WordOrder_FindMinPriorityConditional(pLingCmnInfo, __WordOrder_SpcTermCOND); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord); ET9AssertLog(!ISPROTECTEDWRD(pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord)); if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord) { return ET9STATUS_ERROR; } } if (__PriorityCompare(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord, pWord, SS_Collecting) > 0) { ET9AssertLog(!ISPROTECTEDWRD(pWord)); __ET9AWSelLstIncDiscardedCounter; return wStatus; } __ET9AWSelLstIncReplacingCounter; __ET9ReplaceAndUpdateInsertCounters(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastSpcTermWord, pWord); } else if (pWord->Base.wWordCompLen && pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords >= pLingCmnInfo->Private.nMaxCompletionCount && pLingCmnInfo->Private.nMaxCompletionCount < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) { /* completion word and completion part of list is full, discard or replace */ if (!pLingCmnInfo->Private.bSpcComplDuringSingleBuild) { ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords == pLingCmnInfo->Private.nMaxCompletionCount); } if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord = __WordOrder_FindMinPriorityConditional(pLingCmnInfo, __WordOrder_CmplCOND); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord); ET9AssertLog(!ISPROTECTEDWRD(pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord)); if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord) { return ET9STATUS_ERROR; } } if (__PriorityCompare(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord, pWord, SS_Collecting) > 0) { ET9AssertLog(!ISPROTECTEDWRD(pWord)); __ET9AWSelLstIncDiscardedCounter; return wStatus; } __ET9AWSelLstIncReplacingCounter; __ET9ReplaceAndUpdateInsertCounters(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord, pWord); } else if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) { /* list is not full, get next empty slot */ ET9AssertLog(!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1].Base.wWordLen > 0); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords == pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize || pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords].Base.wWordLen == 0); { ET9AWPrivWordInfo * const pTargetWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords++]; *pTargetWord = *pWord; if (pWord->Body.bHasPrimEditDist && pWord->Base.wWordCompLen) { ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcCmplWords; } else if (pWord->Base.wWordCompLen) { ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalCompletionWords; } else if (pWord->Body.bHasPrimEditDist) { ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalSpcTermWords; } __WordOrder_NewAdd(pLingCmnInfo, pTargetWord); } } else { /* list is full, discard or replace */ ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords == pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastWord = __WordOrder_FindMinPriorityConditional(pLingCmnInfo, __WordOrder_ProtectedCOND); if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastWord = __WordOrder_FindMinPriorityConditional(pLingCmnInfo, __WordOrder_AnyCOND); } ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->pLastWord); if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastWord) { return ET9STATUS_ERROR; } } if (ISPROTECTEDWRD(pWord) && !ISPROTECTEDWRD(pLingCmnInfo->Private.sWordC.pCurrC->pLastWord)) { /* protected wins over non protected ones, except for protected stems that only wins over completions */ if (GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_STEM) { if (!pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord) { pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord = __WordOrder_FindMinPriorityConditional(pLingCmnInfo, __WordOrder_CmplOnlyCOND); ET9AssertLog(!pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord || !ISPROTECTEDWRD(pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord)); } if (pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord) { /* the lowest completion will be replaced, this path is working on pLastWord, needs to assign it */ pLingCmnInfo->Private.sWordC.pCurrC->pLastWord = pLingCmnInfo->Private.sWordC.pCurrC->pLastCompletionWord; } else { /* nothing to replace, discard */ __ET9AWSelLstIncDiscardedCounter; return wStatus; } } } else if (__PriorityCompare(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastWord, pWord, SS_Collecting) > 0) { __ET9AWSelLstIncDiscardedCounter; return wStatus; } __ET9AWSelLstIncReplacingCounter; __ET9ReplaceAndUpdateInsertCounters(pLingCmnInfo, pLingCmnInfo->Private.sWordC.pCurrC->pLastWord, pWord); } __VerifyWordCounters(pLingCmnInfo); return wStatus; } /* ************************************************************************************************************** */ /* * PRIVATE **************************************************************************************************** */ /* ************************************************************************************************************** */ /* tap */ #ifndef _ET9_MIXED_WEIGHT_EditDistStem_tap #define _ET9_MIXED_WEIGHT_EditDistStem_tap 3 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistFree_tap #define _ET9_MIXED_WEIGHT_EditDistFree_tap 0 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcSbt_tap #define _ET9_MIXED_WEIGHT_EditDistSpcSbt_tap 7 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcTrp_tap #define _ET9_MIXED_WEIGHT_EditDistSpcTrp_tap 10 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcIns_tap #define _ET9_MIXED_WEIGHT_EditDistSpcIns_tap 10 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcDel_tap #define _ET9_MIXED_WEIGHT_EditDistSpcDel_tap 3 #endif #ifndef _ET9_MIXED_WEIGHT_Acronym_tap #define _ET9_MIXED_WEIGHT_Acronym_tap 4 #endif #ifndef _ET9_MIXED_WEIGHT_CompletionReg_tap #define _ET9_MIXED_WEIGHT_CompletionReg_tap 20 #endif #ifndef _ET9_MIXED_WEIGHT_CompletionSpc_tap #define _ET9_MIXED_WEIGHT_CompletionSpc_tap 10 #endif #ifndef _ET9_MIXED_WEIGHT_Top5_tap #define _ET9_MIXED_WEIGHT_Top5_tap 0 #endif #ifndef _ET9_MIXED_WEIGHT_Length_tap #define _ET9_MIXED_WEIGHT_Length_tap 9 #endif #ifndef _ET9_MIXED_WEIGHT_PowBase_tap #define _ET9_MIXED_WEIGHT_PowBase_tap 1.4f #endif #ifndef _ET9_MIXED_WEIGHT_CostFactor_tap #define _ET9_MIXED_WEIGHT_CostFactor_tap 0.8f #endif #ifndef _ET9_MIXED_WEIGHT_TapFreqExp_tap #define _ET9_MIXED_WEIGHT_TapFreqExp_tap 2.4f #endif /* trace */ #ifndef _ET9_MIXED_WEIGHT_EditDistStem_trace #define _ET9_MIXED_WEIGHT_EditDistStem_trace 4 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistFree_trace #define _ET9_MIXED_WEIGHT_EditDistFree_trace 1 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcSbt_trace #define _ET9_MIXED_WEIGHT_EditDistSpcSbt_trace 18 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcTrp_trace #define _ET9_MIXED_WEIGHT_EditDistSpcTrp_trace 4 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcIns_trace #define _ET9_MIXED_WEIGHT_EditDistSpcIns_trace 12 #endif #ifndef _ET9_MIXED_WEIGHT_EditDistSpcDel_trace #define _ET9_MIXED_WEIGHT_EditDistSpcDel_trace 6 #endif #ifndef _ET9_MIXED_WEIGHT_SymbDist_trace #define _ET9_MIXED_WEIGHT_SymbDist_trace 0.0f #endif #ifndef _ET9_MIXED_WEIGHT_Acronym_trace #define _ET9_MIXED_WEIGHT_Acronym_trace 8 #endif #ifndef _ET9_MIXED_WEIGHT_Top5_trace #define _ET9_MIXED_WEIGHT_Top5_trace 0 #endif #ifndef _ET9_MIXED_WEIGHT_Pattern_trace #define _ET9_MIXED_WEIGHT_Pattern_trace 3 #endif #ifndef _ET9_MIXED_WEIGHT_TraceLenLo_trace #define _ET9_MIXED_WEIGHT_TraceLenLo_trace -6.0f #endif #ifndef _ET9_MIXED_WEIGHT_TraceLenHi_trace #define _ET9_MIXED_WEIGHT_TraceLenHi_trace +9.0f #endif #ifndef _ET9_MIXED_WEIGHT_TraceLen_trace #define _ET9_MIXED_WEIGHT_TraceLen_trace 3 #endif #ifndef _ET9_MIXED_WEIGHT_PowBase_trace #define _ET9_MIXED_WEIGHT_PowBase_trace 1.6f #endif #ifndef _ET9_MIXED_WEIGHT_CostFactor_trace #define _ET9_MIXED_WEIGHT_CostFactor_trace 0.8f #endif #ifndef _ET9_MIXED_WEIGHT_TapFreqExp_trace #define _ET9_MIXED_WEIGHT_TapFreqExp_trace 2.4f #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AWSelLstAdd(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9BOOL bUsingALM = pLingCmnInfo->Private.bUsingALM; const ET9BOOL bUsingDLM = pLingCmnInfo->Private.bUsingDLM; const ET9BOOL bTraceBuild = pLingCmnInfo->Private.bTraceBuild; ET9AssertLog(pWord); ET9AssertLog(pLingInfo); ET9AssertLog(pWord->Body.xTapFreq >= 0); ET9AssertLog(pWord->Body.xWordFreq >= 0); /* assure non zero and then store original */ if (pWord->Body.xWordFreq == 0) { #ifdef ET9_USE_FLOAT_FREQS pWord->Body.xWordFreq = 0.0000000001f; #else pWord->Body.xWordFreq = 1; #endif } pWord->Body.xWordFreqOrg = pWord->Body.xWordFreq; /* compLen is set in Add and in preAdd... */ if (pWord->Base.wWordCompLen > ET9MAXWORDSIZE) { /* keep pre-calculated value of zero */ pWord->Base.wWordCompLen = 0; } else if (pWord->Base.wWordCompLen && pWord->Base.wWordCompLen < pWord->Base.wWordLen) { /* keep pre-calculated value */ } else if (pWord->Body.bEditDistSpc || pWord->Base.wWordLen < wLength) { pWord->Base.wWordCompLen = 0; } else { pWord->Base.wWordCompLen = (ET9U16)(pWord->Base.wWordLen - wLength); } /* assign term attribute */ pWord->Base.bIsTerm = pWord->Base.wWordCompLen ? 0 : 1; /* store original tap freq also in the normalized attribute */ pWord->Body.xScaledTapFreq = pWord->Body.xTapFreq; if (!pWord->Body.xScaledTapFreq) { pWord->Body.xScaledTapFreq = 1; } /* modify tap freq */ if (__IsUsingMixedStyle(pLingCmnInfo)) { ET9ASPCFlexCompareData * const pCMP = &pLingCmnInfo->Private.ASpc.u.sCmpDataFlex; const ET9BOOL bUsingFlex = (ET9BOOL)pLingCmnInfo->Private.ASpc.bSpcFeatures; const ET9UINT nKeyCount = wLength; const ET9UINT nQualityCount = bUsingFlex ? pCMP->nQualityCount : nKeyCount; const ET9BOOL bIsNWP = (pWord->Base.wWordCompLen == pWord->Base.wWordLen) ? 1 : 0; ET9FLOAT fCost = 1; ET9FLOAT fBoost = 1; ET9INT snCorrectionCost = 0; ET9INT snCorrectionBoost = 0; if (pWord->Body.bEditDistStem) { ET9U8 bEditCount = pWord->Body.bEditDistStem + pWord->Body.bEditDistSpcTrp + 2 * pWord->Body.bEditDistSpcSbt; fCost += 10.0f * bEditCount; snCorrectionCost += ((bTraceBuild) ? _ET9_MIXED_WEIGHT_EditDistStem_trace : _ET9_MIXED_WEIGHT_EditDistStem_tap) * bEditCount; } if (pWord->Body.bEditDistSpc) { const ET9BOOL bRegionalSpc = (pLingCmnInfo->Private.bCurrSpcMode == ET9ASPCMODE_REGIONAL) ? 1 : 0; if (pWord->Body.bEditDistSpcSbt) { fCost += ((bRegionalSpc || bTraceBuild) ? 400.0f : 100.0f) * pWord->Body.bEditDistSpcSbt; snCorrectionCost += ((bTraceBuild) ? _ET9_MIXED_WEIGHT_EditDistSpcSbt_trace : _ET9_MIXED_WEIGHT_EditDistSpcSbt_tap) * pWord->Body.bEditDistSpcSbt; } if (pWord->Body.bEditDistSpcTrp) { fCost += ((bRegionalSpc || bTraceBuild) ? 100.0f : 25.0f) * pWord->Body.bEditDistSpcTrp; snCorrectionCost += ((bTraceBuild) ? _ET9_MIXED_WEIGHT_EditDistSpcTrp_trace : _ET9_MIXED_WEIGHT_EditDistSpcTrp_tap) * pWord->Body.bEditDistSpcTrp; } if (pWord->Body.bEditDistSpcIns) { fCost += ((bRegionalSpc || bTraceBuild) ? 200.0f : 50.0f) * pWord->Body.bEditDistSpcIns; snCorrectionCost += ((bTraceBuild) ? _ET9_MIXED_WEIGHT_EditDistSpcIns_trace : _ET9_MIXED_WEIGHT_EditDistSpcIns_tap) * pWord->Body.bEditDistSpcIns; } if (pWord->Body.bEditDistSpcDel) { fCost += ((bRegionalSpc || bTraceBuild) ? 200.0f : 50.0f) * pWord->Body.bEditDistSpcDel; snCorrectionCost += ((bTraceBuild) ? _ET9_MIXED_WEIGHT_EditDistSpcDel_trace : _ET9_MIXED_WEIGHT_EditDistSpcDel_tap) * pWord->Body.bEditDistSpcDel; } } if (pWord->Body.bEditDistFree) { fCost += (bTraceBuild) ? 1 : 0; snCorrectionCost += (bTraceBuild) ? _ET9_MIXED_WEIGHT_EditDistFree_trace : _ET9_MIXED_WEIGHT_EditDistFree_tap; if (GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_CDB) { fCost += 200; snCorrectionCost += 9; } if (pWord->Body.bIsWeak || pWord->Body.bIsHidden || pWord->Body.bIsQuarantine || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_QUDB || (GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_RUDB && pWord->Body.bIsUDBWord)) { fCost += 2 * pWord->Body.bEditDistFree; snCorrectionCost += pWord->Body.bEditDistFree; } } if (pWord->Body.xSymbDist > 0) { ET9AssertLog(bTraceBuild); #if 0 if (pWord->Body.xSymbDist < 8) { } else if (pWord->Body.xSymbDist < 20) { snCorrectionCost += 5; } else { snCorrectionCost += 10; } #else snCorrectionCost += (ET9INT)(_ET9_MIXED_WEIGHT_SymbDist_trace * pWord->Body.xSymbDist); #endif } if (pWord->Body.bIsAcronym && !bIsNWP) { snCorrectionCost += (bTraceBuild) ? _ET9_MIXED_WEIGHT_Acronym_trace : _ET9_MIXED_WEIGHT_Acronym_tap; } if (pWord->Base.wWordCompLen && !bIsNWP) { if (bUsingALM || bUsingDLM) { fCost += (pWord->Body.bEditDistSpc ? 30.0f : 20.0f); snCorrectionCost += (pWord->Body.bEditDistSpc ? _ET9_MIXED_WEIGHT_CompletionSpc_tap : _ET9_MIXED_WEIGHT_CompletionReg_tap); } else { const ET9FLOAT fCompLenValue = (ET9FLOAT)(pWord->Base.wWordCompLen + 1) / 2.0f; fCost += (pWord->Body.bEditDistSpc ? 3.0f : 2.0f) * fCompLenValue; snCorrectionCost += (pWord->Base.wWordCompLen + 1) / 2; } if (bTraceBuild && !ET9DEVSTATEINHBTRACECMPLDEMOTE_MODE(pLingCmnInfo->Private.dwDevStateBits)) { fCost += 1000; snCorrectionCost += 1000; } } if (pWord->Body.bLangIndexScoring == pLingCmnInfo->Private.bNonActiveIndex) { fCost += 50; snCorrectionCost += 5; } if (pWord->Body.bIsTop5 && !bIsNWP) { if (pWord->Body.dwWordIndex && pWord->Body.dwWordIndex <= 100) { snCorrectionBoost += (bTraceBuild) ? _ET9_MIXED_WEIGHT_Top5_trace : _ET9_MIXED_WEIGHT_Top5_tap; } else { snCorrectionBoost += (bTraceBuild) ? _ET9_MIXED_WEIGHT_Top5_trace : _ET9_MIXED_WEIGHT_Top5_tap; } } if (bTraceBuild) { const ET9U8 bSrc = GETBASESRC(pWord->Body.bWordSrc); if ((pWord->Base.wWordLen == 1 && (bSrc == ET9WORDSRC_LAS_SHORTCUT || bSrc == ET9WORDSRC_ASDB_SHORTCUT))) { fBoost *= 2; snCorrectionBoost += 2; } } else if (nQualityCount == nKeyCount) { const ET9U8 bSrc = GETBASESRC(pWord->Body.bWordSrc); if (bSrc == ET9WORDSRC_LAS_SHORTCUT || bSrc == ET9WORDSRC_ASDB_SHORTCUT) { if (pWord->Base.wWordLen == 1) { fBoost *= 10; snCorrectionBoost += 3; } } else if ((ET9UINT)(pWord->Base.wWordLen - pWord->Body.bEditDistFree) == nQualityCount) { fBoost *= 10; snCorrectionBoost += _ET9_MIXED_WEIGHT_Length_tap; } } WLOG2B(fprintf(pLogFile2, "_ET9AWSelLstAdd, nBoost %5.1f, nCost = %5.1f, %8.2f (%8.2f) (cc %u cb %u)\n", fBoost, fCost, ((pWord->Body.xTapFreq * fBoost) / fCost), (double)pWord->Body.xTapFreq, snCorrectionCost, snCorrectionBoost);) pWord->Body.xTapFreq = (ET9FREQPART)((pWord->Body.xTapFreq * fBoost) / fCost); pWord->Body.snCorrectionCost = snCorrectionCost - snCorrectionBoost; } else if (bTraceBuild) { if (pWord->Body.bEditDistFree) { ++pWord->Body.bEditDistStem; } pWord->Body.xTapFreq /= pWord->Body.bEditDistStem + 1; pWord->Body.snCorrectionCost = (ET9INT)pWord->Body.bEditDistStem; } if (!pWord->Body.xTapFreq) { pWord->Body.xTapFreq = 1; } /* assign tot freq */ __CalculateTempTotWordFreq(pLingInfo, pWord); /* validate */ ET9AssertLog(pWord->Body.xTapFreq >= 0); ET9AssertLog(pWord->Body.xWordFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); /* actual list insert */ return __ET9AWSelLstInsert(pLingInfo, pWord, bFreqIndicator, 0); } #ifdef _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ ET9INLINE static void ET9LOCALCALL __CharHashCmp (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nPosIndex, const ET9SYMB sChar, ET9U8 * const pbFreq, ET9U8 * const pbSymbDist, ET9U8 * const pbLocked, ET9BOOL * const pbExactMatch, ET9BOOL * const pbLimitedMatch) { ET9ADdbCharHashItem const * const pHashTable = &pLingCmnInfo->Private.ASpc.ddbCharHashInfo[nPosIndex]; ET9UINT nIndex; ET9Assert(pbFreq); ET9Assert(pbLocked); ET9Assert(pbExactMatch); ET9Assert(pbLimitedMatch); ET9Assert(nPosIndex < pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs); /* lock attribute */ *pbLocked = pLingCmnInfo->Private.ASpc.pbLockInfo[nPosIndex]; /* valid char? */ if (!sChar) { *pbFreq = 0; return; } /* look up char in table */ { ET9UINT nSentinel; nIndex = (sChar & 0xFF) << 1; for (nSentinel = ADDB_COMPARE_MAX_CHAR_HASH_SIZE; nSentinel; --nSentinel) { ET9Assert(nIndex < ADDB_COMPARE_MAX_CHAR_HASH_SIZE); { const ET9SYMB sItemChar = pHashTable->psHashTable[nIndex]; if (!sItemChar) { *pbFreq = 0; return; } else if (sItemChar == sChar) { break; } ++nIndex; if (nIndex >= ADDB_COMPARE_MAX_CHAR_HASH_SIZE) { nIndex = 0; } } } ET9Assert(nSentinel); } /* set attributes */ { ET9U8 bSymDataIndex; bSymDataIndex = pHashTable->pbSymbIndex[nIndex]; if (bSymDataIndex >= 0x80) { *pbExactMatch = 1; bSymDataIndex -= 0x80; } else { *pbExactMatch = 0; } { ET9SymbInfo const * const pSymbInfo = &pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo[nPosIndex]; ET9DataPerBaseSym const * const pSymData = &pSymbInfo->DataPerBaseSym[bSymDataIndex]; ET9Assert(bSymDataIndex < pSymbInfo->bNumBaseSyms); *pbFreq = pSymData->bSymFreq; *pbSymbDist = pSymData->bSymDist; *pbLimitedMatch = pSymData->bLimited; } } } #endif /* _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * * */ static void ET9LOCALCALL __GetCodeFreqStd (ET9AWLingInfo * const pLingInfo, const ET9INT snIndex, const ET9SYMB sSymb, const ET9BOOL bExactMatchOnly, ET9U8 * const pbMatch, ET9U8 * const pbFreq, ET9U8 * const pbLocked, ET9BOOL * const pbLimited, ET9U32 dwLdbId) { ET9INT snLockCompare; ET9INT snFirstLevelExact; ET9SymbInfo *pSymbInfo; ET9DataPerBaseSym *pSymData; ET9SYMB *pLower; ET9SYMB *pUpper; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; __STAT_INC_CmpOther; #if 0 WLOG2(fprintf(pLogFile2, "__GetCodeFreqStd, snIndex = %d, sSymb = %x\n", snIndex, (int)sSymb);) #endif ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pbMatch != NULL); ET9AssertLog(pbFreq != NULL); ET9AssertLog(pbLocked != NULL); ET9AssertLog(pWordSymbInfo != NULL); ET9AssertLog(snIndex < pWordSymbInfo->bNumSymbs); /* assign a proper default value, especially for "no match" */ *pbFreq = 0xcc; *pbLimited = 0; /* this is not supposed to happen, just for extra safety in "release" mode, should be after the assert above */ if (snIndex >= pWordSymbInfo->bNumSymbs) { *pbLocked = 0; *pbMatch = ET9_SPC_ED_MATCH_NONE; return; } ET9AssertLog(pLingCmnInfo->Private.ASpc.bSpcState == ET9_SPC_STATE_STD_INIT_OK); /* hash info compare? */ #ifdef _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH if (snIndex < ADDB_COMPARE_MAX_CHAR_HASH_POS) { ET9U8 bSymbDist; ET9BOOL bExactMatch; __CharHashCmp(pLingCmnInfo, snIndex, sSymb, pbFreq, &bSymbDist, pbLocked, &bExactMatch, pbLimited); if (*pbFreq) { if (bExactMatch) { *pbMatch = ET9_SPC_ED_MATCH_EXACT; } else if (!bExactMatchOnly) { *pbMatch = ET9_SPC_ED_MATCH_FULL; } else { *pbMatch = ET9_SPC_ED_MATCH_NONE; } } else { *pbMatch = ET9_SPC_ED_MATCH_NONE; } return; } #endif /* _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH */ /* check bit hash */ #ifdef _ET9_ACTIVATE_ADDB_COMPARE_BIT_HASH /* lock info */ *pbLocked = pLingCmnInfo->Private.ASpc.pbLockInfo[snIndex]; { const ET9UINT nCode = sSymb & 0xFF; if (!__ET9BitCheck(pLingCmnInfo->Private.ASpc.ddbHashInfo[snIndex], nCode)) { *pbMatch = ET9_SPC_ED_MATCH_NONE; return; } } #endif /* _ET9_ACTIVATE_ADDB_COMPARE_BIT_HASH */ /* search */ if (pWordSymbInfo->SymbsInfo[snIndex].bSymbType == ET9KTSMARTPUNCT) { snFirstLevelExact = 0; *pbMatch = ET9_SPC_ED_MATCH_EXACT; } else { switch (pWordSymbInfo->SymbsInfo[snIndex].eInputType) { case ET9DISCRETEKEY: case ET9CUSTOMSET: case ET9MULTITAPKEY: snFirstLevelExact = 0; *pbMatch = ET9_SPC_ED_MATCH_EXACT; break; default: snFirstLevelExact = 1; *pbMatch = ET9_SPC_ED_MATCH_EXACT; break; } } snLockCompare = *pbLocked == ET9_LOCKED_SYMB_VALUE_AND_POS; pSymbInfo = &pWordSymbInfo->SymbsInfo[snIndex]; if (snLockCompare) { const ET9SYMB sLockedSymb = pSymbInfo->sLockedSymb; *pbLocked = ET9_LOCKED_SYMB_VALUE_AND_POS; if (sSymb == sLockedSymb || sSymb == _ET9SymToOther(sLockedSymb, dwLdbId)) { *pbFreq = 1; *pbMatch = ET9_SPC_ED_MATCH_EXACT; return; } } else { ET9INT snKeyIndex; pSymData = pSymbInfo->DataPerBaseSym; snKeyIndex = (ET9INT)((bExactMatchOnly && snFirstLevelExact) ? 1 : pSymbInfo->bNumBaseSyms); while (snKeyIndex--) { if (pSymData->bLimited && pLingCmnInfo->Private.bCurrLimitedRegionality) { } else if (pSymData->bLimited && snIndex && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[snIndex - 1] && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[snIndex - 1] <= pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs && _ET9_IsPunctOrNumeric(pSymbInfo->DataPerBaseSym[0].sChar[0])) { /* disallow limited symbols for punct/numeric buildaround points after a flush */ } else { ET9INT k; k = pSymData->bNumSymsToMatch; pLower = pSymData->sChar; pUpper = pSymData->sUpperCaseChar; while (k--) { if (sSymb == *pLower++ || sSymb == *pUpper++) { *pbFreq = pSymData->bSymFreq; *pbLimited = pSymData->bLimited; return; } } } ++pSymData; if (snFirstLevelExact) { *pbMatch = ET9_SPC_ED_MATCH_FULL; } } } *pbMatch = ET9_SPC_ED_MATCH_NONE; } /*--------------------------------------------------------------------------- * undef's and control defs for edit distance calculation *---------------------------------------------------------------------------*/ #undef ET9_SPC_ED_GetMatchOnly #undef ET9_SPC_ED_GetMatchInfo #undef ET9_SPC_ED_GetWordSymb #undef ET9_SPC_ED_USE_COMPARE_CACHE #undef __ET9AWCalcEditDistance #ifdef _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH #else #define ET9_SPC_ED_USE_COMPARE_CACHE #endif /* _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ #define ET9_SPC_ED_GetMatchInfo(index1, index2, bExactMatchOnly, pbMatch, pbFreq, pbLocked, pbLimited) \ ( \ __GetCodeFreqStd(pLingInfo, \ ((index1) + (wIndex)), \ (pWord->Base.sWord[index2]), \ (bExactMatchOnly), \ (&pbMatch), \ (&pbFreq), \ (&pbLocked), \ (&pbLimited), \ ((pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum)) \ ) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ #define ET9_SPC_ED_GetWordSymb(index, psSymb) \ { \ (psSymb) = pWord->Base.sWord[index]; \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ #define __ET9AWCalcEditDistance __ET9AWCalcEditDistanceASLST #include "et9aspci.h" /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static ET9SYMB ET9LOCALCALL __AdbGetCharAtPos(ET9AWPrivWordInfo const * const pWord, const ET9UINT nPos) { return pWord->Base.sWord[nPos]; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * */ ET9INLINE static ET9U8 ET9LOCALCALL __GetSymbFreq(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nPos, const ET9SYMB sSymb, ET9U8 * const pbSymbDist, ET9BOOL * const pbExactMatch, ET9BOOL * const pbLimitedMatch) { ET9ASPCFlexCompareData * const pCMP = &pLingCmnInfo->Private.ASpc.u.sCmpDataFlex; ET9SymbInfo const * const pSymbInfo = &pCMP->pFirstSymb[nPos]; ET9UINT nBaseCount; ET9DataPerBaseSym const *pDPBS; #if 0 WLOG2(fprintf(pLogFile2, "__GetSymbFreq, nPos = %d, sSymb = %x, nFirstOffset = %u\n", nPos, (int)sSymb, pCMP->nFirstOffset);) #endif /* default values */ if (pbSymbDist) { *pbSymbDist = 0; } if (pbExactMatch) { *pbExactMatch = 0; } if (pbLimitedMatch) { *pbLimitedMatch = 0; } ET9AssertLog(pLingCmnInfo->Private.ASpc.bSpcState == ET9_SPC_STATE_FLEX_INIT_OK || (pLingCmnInfo->Private.ASpc.bSpcState == ET9_SPC_STATE_STD_INIT_OK && !pLingCmnInfo->Private.bTraceBuild)); /* hash info compare? */ #ifdef _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH { const ET9UINT nPosIndex = nPos + pCMP->nFirstOffset; if (nPosIndex < ADDB_COMPARE_MAX_CHAR_HASH_POS) { ET9U8 bFreq; ET9U8 bLocked; ET9U8 bSymbDist; ET9BOOL bExactMatch; ET9BOOL bLimitedMatch; __CharHashCmp(pLingCmnInfo, nPosIndex, sSymb, &bFreq, (pbSymbDist ? pbSymbDist : &bSymbDist), &bLocked, (pbExactMatch ? pbExactMatch : &bExactMatch), (pbLimitedMatch ? pbLimitedMatch : &bLimitedMatch)); return bFreq; } } #endif /* _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH */ /* check bit hash */ #ifdef _ET9_ACTIVATE_ADDB_COMPARE_BIT_HASH { const ET9UINT nCode = sSymb & 0xFF; if (!__ET9BitCheck(pLingCmnInfo->Private.ASpc.ddbHashInfo[nPos + pCMP->nFirstOffset], nCode)) { return 0; } } #endif /* _ET9_ACTIVATE_ADDB_COMPARE_BIT_HASH */ /* full get */ if (pCMP->psLockSymb[nPos]) { if (sSymb == pCMP->psLockSymb[nPos] || sSymb == pCMP->psLockSymbOther[nPos]) { if (pbExactMatch) { *pbExactMatch = 1; } return 1; } else { return 0; } } pDPBS = pSymbInfo->DataPerBaseSym; for (nBaseCount = pSymbInfo->bNumBaseSyms; nBaseCount ; --nBaseCount, ++pDPBS) { ET9SYMB const * pLower; ET9SYMB const * pUpper; ET9UINT nSymsCount; if (pDPBS->bLimited && pLingCmnInfo->Private.bCurrLimitedRegionality) { continue; } if (pDPBS->bLimited) { const ET9UINT nIndex = pCMP->nFirstOffset + nPos; if (nIndex && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[nIndex - 1] && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[nIndex - 1] <= pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs && _ET9_IsPunctOrNumeric(pSymbInfo->DataPerBaseSym[0].sChar[0])) { /* disallow limited symbols for punct/numeric buildaround points after a flush */ continue; } } pLower = pDPBS->sChar; pUpper = pDPBS->sUpperCaseChar; for (nSymsCount = pDPBS->bNumSymsToMatch; nSymsCount ; --nSymsCount, ++pLower, ++pUpper) { if (sSymb == *pLower || sSymb == *pUpper) { ET9U8 bSymFreq; bSymFreq = pDPBS->bSymFreq; if (!bSymFreq) { bSymFreq = 1; } if (pbSymbDist) { *pbSymbDist = pDPBS->bSymDist; } if (pbExactMatch) { *pbExactMatch = (ET9BOOL)(pDPBS == pSymbInfo->DataPerBaseSym || pSymbInfo->eInputType == ET9DISCRETEKEY || pSymbInfo->eInputType == ET9MULTITAPKEY || pSymbInfo->eInputType == ET9CUSTOMSET || pSymbInfo->bSymbType == ET9KTSMARTPUNCT); } if (pbLimitedMatch) { *pbLimitedMatch = pDPBS->bLimited; } return bSymFreq; } } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9U8 ET9FARCALL _ET9_GetSymbFreq(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nPos, const ET9SYMB sSymb, ET9BOOL * const pbExactMatch, ET9BOOL * const pbLimitedMatch) { return __GetSymbFreq(pLingCmnInfo, nPos, sSymb, NULL, pbExactMatch, pbLimitedMatch); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * * */ static ET9STATUS ET9LOCALCALL __ADB_CalcEditDistanceFlex(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9ASPCFlexCompareData * const pCMP = &pLingCmnInfo->Private.ASpc.u.sCmpDataFlex; ET9U8 const * const pbLockInfo = pCMP->pbLockInfo; ET9U8 const * const pbIsFreqPos = pCMP->pbIsFreqPos; ET9U8 const * const pbIsQualityKey = pCMP->pbIsQualityKey; const ET9U8 bMaxEditDist = pLingCmnInfo->Private.bCurrMaxEditDistance; const ET9U16 wCurrMinSourceLength = pLingCmnInfo->Private.wCurrMinSourceLength; const ET9BOOL bAllowSpcCmpl = pCMP->bAllowSpcCmpl; const ET9BOOL bAllowFreePunct = pCMP->bAllowFreePunct; const ET9BOOL bAllowFreeDouble = pCMP->bAllowFreeDouble; const ET9UINT nKeyCount = wLength; const ET9UINT nSymbCount = pWord->Base.wWordLen; const ET9UINT nQualityCount = pCMP->nQualityCount; ET9_UNUSED(wIndex); /* used in "init" */ ET9AssertLog(pLingCmnInfo->Private.ASpc.bSpcState == ET9_SPC_STATE_FLEX_INIT_OK); _ET9AWValidateFlexArea(pLingCmnInfo); WLOG8B({ fprintf(pLogFile8, "__ADB_CalcEditDistanceFlex, spc cmpl %d, free punct %d, free double %d, word ", bAllowSpcCmpl, bAllowFreePunct, bAllowFreeDouble); { ET9U16 wIndex; for (wIndex = 0; wIndex < pWord->Base.wWordLen; ++wIndex) { fprintf(pLogFile8, "%c", pWord->Base.sWord[wIndex]); } } fprintf(pLogFile8, "\n"); }) /* screening */ if (nSymbCount < wCurrMinSourceLength) { WLOG8B(fprintf(pLogFile8, " no match (too short, %u < %u - %u)\n", nSymbCount, nQualityCount, bMaxEditDist);) _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } if (nQualityCount == 1 && nSymbCount > nQualityCount) { ET9BOOL bMatchExact; (void)__GetSymbFreq(pLingCmnInfo, 0, __AdbGetCharAtPos(pWord, 0), NULL, &bMatchExact, NULL); if (!bMatchExact) { WLOG8B(fprintf(pLogFile8, " special completion suppression at length 1\n");) _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } /* screen calculation */ { ET9UINT nR; ET9UINT nC; ET9BOOL bInStem = 1; /* matrix */ for (nC = 1; nC <= nSymbCount; ++nC) { const ET9U16 wCharCode = __AdbGetCharAtPos(pWord, nC - 1); if (bInStem) { if (pCMP->pwPrevWordSC[nC] == wCharCode) { continue; } else { bInStem = 0; } } pCMP->pwPrevWordSC[nC] = wCharCode; { const ET9BOOL bIsFreeC = (ET9BOOL)(bAllowFreePunct && !pCMP->psLockSymb[nC - 1] && _ET9_IsFree(wCharCode)); ET9U8 bBestColEditDist; pCMP->ppbEditDist[0][nC - 0] = pCMP->ppbEditDist[0][nC - 1] + (bIsFreeC ? 0 : 1); bBestColEditDist = pCMP->ppbEditDist[0][nC]; for (nR = 1; nR <= nKeyCount; ++nR) { WLOG8B(fprintf(pLogFile8, "matrix %2d %2d\n", nR, nC);) /* initial */ pCMP->ppbEditDist[nR][nC] = _ET9_FLEX_TUBE_MAX_EDIT_DIST; /* pointless? */ if (pCMP->ppbEditDist[nR - 1][nC - 1] > bMaxEditDist && pCMP->ppbEditDist[nR - 1][nC - 0] > bMaxEditDist && pCMP->ppbEditDist[nR - 0][nC - 1] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " pointless (edit) [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = 0; continue; } { const ET9U8 bFreeR = !pbLockInfo[nR]; const ET9U8 bQualityR = pbIsQualityKey[nR]; const ET9U8 bMatchFreq = __GetSymbFreq(pLingCmnInfo, (nR - 1), wCharCode, NULL, NULL, NULL); /* */ pCMP->pbSubstFreqSC[nR][nC] = bMatchFreq; /* subst */ if (bMatchFreq) { WLOG8B(fprintf(pLogFile8, " subst-1, bMatchFreq %d (%u, max %u) [sc]\n", bMatchFreq, pCMP->ppbEditDist[nR - 1][nC - 1], bMaxEditDist);) pCMP->ppbEditDist[nR][nC] = pCMP->ppbEditDist[nR - 1][nC - 1]; } else if (pbLockInfo[nR] == ET9_LOCKED_SYMB_VALUE_AND_POS) { WLOG8B(fprintf(pLogFile8, " subst, non matching locked symb [sc]\n");) ET9Assert(pCMP->ppbEditDist[nR][nC] == _ET9_FLEX_TUBE_MAX_EDIT_DIST); continue; } else if (pCMP->ppbEditDist[nR - 1][nC - 1] < bMaxEditDist && bQualityR && bFreeR) { WLOG8B(fprintf(pLogFile8, " subst-2, ppbEditDist %d [sc]\n", pCMP->ppbEditDist[nR - 1][nC - 1]);) pCMP->ppbEditDist[nR][nC] = pCMP->ppbEditDist[nR - 1][nC - 1] + 1; } /* del */ if (!bQualityR) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " del-1 (%u, max %u) [sc]\n", bEditCost, bMaxEditDist);) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } else if (pCMP->ppbEditDist[nR - 1][nC - 0] < bMaxEditDist && bFreeR) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0] + 1; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " del-2 [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } /* ins */ if ((bAllowFreeDouble && bMatchFreq && bMatchFreq == pCMP->pbSubstFreqSC[nR][nC - 1]) || bIsFreeC) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-1 [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = pCMP->pbSubstFreqSC[nR][nC - 1]; pCMP->ppbEditDist[nR][nC] = bEditCost; } } else if (nR == nKeyCount && nC > nQualityCount && (!pCMP->ppbEditDist[nR - 0][nC - 1] || bAllowSpcCmpl)) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-2, nC %d, nQualityCount %d [sc]\n", nC, nQualityCount);) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } else if (pCMP->ppbEditDist[nR - 0][nC - 1] < bMaxEditDist) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1] + 1; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-3 (%u, max %u) [sc]\n", bEditCost, bMaxEditDist);) pCMP->pbSubstFreqSC[nR][nC] = bMatchFreq; pCMP->ppbEditDist[nR][nC] = bEditCost; } } /* best */ if (bBestColEditDist > pCMP->ppbEditDist[nR][nC]) { bBestColEditDist = pCMP->ppbEditDist[nR][nC]; } } } /* nR */ /* pre-empt? */ if (bBestColEditDist > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " no match (best edit) [sc], %d > %d\n", bBestColEditDist, bMaxEditDist);) pCMP->pwPrevWordSC[nC] = ALDB_CHAR_CODE_NONE; _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } } /* nC */ pCMP->pwPrevWordSC[nC] = ALDB_CHAR_CODE_NONE; WLOG8B({ fprintf(pLogFile8, "\n--- edit distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbEditDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n"); }) /* really a match? */ if (pCMP->ppbEditDist[nKeyCount][nSymbCount] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " no match (final edit), %d > %d (%d)\n", pCMP->ppbEditDist[nKeyCount][nSymbCount], bMaxEditDist, pCMP->ppbStemDist[nKeyCount][nSymbCount]);) _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } /* full calculation */ { ET9UINT nR; ET9UINT nC; ET9BOOL bInStem = 1; /* matrix */ for (nC = 1; nC <= nSymbCount; ++nC) { const ET9U16 wCharCode = __AdbGetCharAtPos(pWord, nC - 1); if (bInStem) { if (pCMP->pwPrevWordFC[nC] == wCharCode) { continue; } else { bInStem = 0; } } pCMP->pwPrevWordFC[nC] = wCharCode; { const ET9BOOL bIsFreeC = (ET9BOOL)(bAllowFreePunct && !pCMP->psLockSymb[nC - 1] && _ET9_IsFree(wCharCode)); pCMP->pbComplLenFC[nC] = 0; pCMP->ppbStemDist[0][nC - 0] = pCMP->ppbEditDist[0][nC - 0]; pCMP->ppbFreeDist[0][nC - 0] = (ET9U8)(nC - pCMP->ppbEditDist[0][nC - 0]); pCMP->ppbEditDistIns[0][nC - 0] = pCMP->ppbEditDist[0][nC - 0]; for (nR = 1; nR <= nKeyCount; ++nR) { WLOG8B(fprintf(pLogFile8, "matrix %2d %2d\n", nR, nC);) /* pointless? */ if (pCMP->ppbEditDist[nR][nC] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " pointless (edit %u > %u) [full]\n", pCMP->ppbEditDist[nR][nC], bMaxEditDist);) pCMP->ppbFreeDist[nR][nC] = _ET9_FLEX_TUBE_MAX_FREE_DIST; pCMP->ppbStemDist[nR][nC] = _ET9_FLEX_TUBE_MAX_STEM_DIST; pCMP->ppxStemFreq[nR][nC] = 1; pCMP->pbSubstFreqFC[nR][nC] = 0; continue; } { const ET9U8 bFreeR = !pbLockInfo[nR]; const ET9U8 bQualityR = pbIsQualityKey[nR]; ET9U8 bSymbDist; ET9BOOL bMatchExact; ET9BOOL bMatchLimited; const ET9U8 bMatchFreq = __GetSymbFreq(pLingCmnInfo, (nR - 1), wCharCode, &bSymbDist, &bMatchExact, &bMatchLimited); const ET9U8 bDefaultFreq = (bQualityR && nC < ET9_SPC_ED_MAX_FREQ_LEN) ? 10 : 1; ET9U8 bEditDist; ET9U8 bFreeDist; ET9U8 bStemDist; ET9FREQ xStemFreq; ET9FREQ xSymbDist; ET9U8 bEditDistSbt; ET9U8 bEditDistTrp; ET9U8 bEditDistIns; ET9U8 bEditDistDel; ET9U8 bLimitedCount; /* */ pCMP->pbSubstFreqFC[nR][nC] = bMatchFreq; /* subst */ if (bMatchFreq) { #ifdef ET9_USE_FLOAT_FREQS const ET9FREQ xFreq = (ET9FREQ)(!pbIsFreqPos[nR] ? bDefaultFreq : bMatchFreq); #else const ET9FREQ xFreq = (!pbIsFreqPos[nR] || bMatchFreq <= 17) ? 1 : ((bMatchFreq * 15) / 255); #endif const ET9FREQ xDist = (ET9FREQ)bSymbDist; WLOG8B(fprintf(pLogFile8, " subst-1, bMatchFreq %d (%u, max %u) [full]\n", bMatchFreq, pCMP->ppbEditDist[nR - 1][nC - 1], bMaxEditDist);) bFreeDist = pCMP->ppbFreeDist[nR - 1][nC - 1] + (bQualityR ? 0 : 1); bEditDist = pCMP->ppbEditDist[nR - 1][nC - 1]; bStemDist = pCMP->ppbStemDist[nR - 1][nC - 1] + (bMatchExact ? 0 : 1); xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 1] * xFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 1] + xDist; bEditDistSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 1]; bEditDistTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 1]; bEditDistIns = pCMP->ppbEditDistIns[nR - 1][nC - 1]; bEditDistDel = pCMP->ppbEditDistDel[nR - 1][nC - 1]; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 1] + (bMatchLimited ? 1 : 0); } else if (pbLockInfo[nR] == ET9_LOCKED_SYMB_VALUE_AND_POS) { WLOG8B(fprintf(pLogFile8, " subst, non matching locked symb [full]\n");) ET9AssertLog(pCMP->ppbEditDist[nR][nC] == _ET9_FLEX_TUBE_MAX_EDIT_DIST); continue; } else if (pCMP->ppbEditDist[nR - 1][nC - 1] < bMaxEditDist && bQualityR && bFreeR) { WLOG8B(fprintf(pLogFile8, " subst-2, ppbEditDist %d [full]\n", pCMP->ppbEditDist[nR - 1][nC - 1]);) bFreeDist = pCMP->ppbFreeDist[nR - 1][nC - 1]; bEditDist = pCMP->ppbEditDist[nR - 1][nC - 1] + 1; bStemDist = pCMP->ppbStemDist[nR - 1][nC - 1] + 1; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 1] * bDefaultFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 1]; bEditDistSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 1] + 1; bEditDistTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 1]; bEditDistIns = pCMP->ppbEditDistIns[nR - 1][nC - 1]; bEditDistDel = pCMP->ppbEditDistDel[nR - 1][nC - 1]; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 1]; } else { bFreeDist = _ET9_FLEX_TUBE_MAX_FREE_DIST; bEditDist = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bStemDist = _ET9_FLEX_TUBE_MAX_STEM_DIST; xStemFreq = 1; xSymbDist = 0; bEditDistSbt = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistTrp = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistIns = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistDel = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bLimitedCount = 0; } /* del */ if (!bQualityR) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 1][nC - 0]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 1][nC - 0]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 0]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 0]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 1][nC - 0]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 1][nC - 0]; if (_ET9_Flex_IsBetterOp(-1, 0)) { WLOG8B(fprintf(pLogFile8, " del-1 [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = 0; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 0]; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 0]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 0]; } } else if (pCMP->ppbEditDist[nR - 1][nC - 0] < bMaxEditDist && bFreeR) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 1][nC - 0]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0] + 1; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 1][nC - 0] + 1; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 0]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 0]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 1][nC - 0]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 1][nC - 0] + 1; if (_ET9_Flex_IsBetterOp(-1, 0)) { WLOG8B(fprintf(pLogFile8, " del-2 [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = 0; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 0] * bDefaultFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 0]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 0]; } } /* ins */ if ((bAllowFreeDouble && bMatchFreq && bMatchFreq == pCMP->pbSubstFreqFC[nR][nC - 1]) || bIsFreeC) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1] + 1; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-1 (free) [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = pCMP->pbSubstFreqFC[nR][nC - 1]; if (nR == nKeyCount) { pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1]; } bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } else if (nR == nKeyCount && nC > nQualityCount && (!pCMP->ppbEditDist[nR - 0][nC - 1] || bAllowSpcCmpl)) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-2 (cmpl), nC %d, nQualityCount %d [full]\n", nC, nQualityCount);) pCMP->pbSubstFreqFC[nR][nC] = 0; pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1] + 1; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } else if (pCMP->ppbEditDist[nR - 0][nC - 1] < bMaxEditDist) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1] + 1; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1] + 1; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1] + 1; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-3 (spc) [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = bMatchFreq; if (nR == nKeyCount) { pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1]; } bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } /* assure same as screening */ if (bEditDist != pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " inconsistent (%u <-> %u) [full]\n", bEditDist, pCMP->ppbEditDist[nR][nC]);) } ET9AssertLog(bEditDist == pCMP->ppbEditDist[nR][nC]); /* persist */ pCMP->ppbFreeDist[nR][nC] = bFreeDist; pCMP->ppbStemDist[nR][nC] = bStemDist; pCMP->ppxStemFreq[nR][nC] = xStemFreq; pCMP->ppxSymbDist[nR][nC] = xSymbDist; pCMP->ppbEditDistSbt[nR][nC] = bEditDistSbt; pCMP->ppbEditDistTrp[nR][nC] = bEditDistTrp; pCMP->ppbEditDistIns[nR][nC] = bEditDistIns; pCMP->ppbEditDistDel[nR][nC] = bEditDistDel; pCMP->ppbLimitedCount[nR][nC] = bLimitedCount; } } /* nR */ } } /* nC */ pCMP->pwPrevWordFC[nC] = ALDB_CHAR_CODE_NONE; WLOG8B({ fprintf(pLogFile8, "\n--- free distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbFreeDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- edist distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbEditDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- stem distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbStemDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- edit kinds ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%x%x%x%x ", pCMP->ppbEditDistSbt[nR][nC], pCMP->ppbEditDistTrp[nR][nC], pCMP->ppbEditDistIns[nR][nC], pCMP->ppbEditDistDel[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- stem freq ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%8.0f ", pCMP->ppxStemFreq[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- completion length ---\n\n"); { fprintf(pLogFile8, " : "); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->pbComplLenFC[nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n"); }) /* assure same as screening */ ET9AssertLog(pCMP->ppbEditDist[nKeyCount][nSymbCount] <= bMaxEditDist); { const ET9UINT nMatchRow = nKeyCount; const ET9UINT nMatchCol = nSymbCount; const ET9BOOL bAvoidStems = (pLingCmnInfo->Private.bTraceBuild || wIndex) ? 1 : 0; ET9U8 bComplLenFC_nSymbCount = pCMP->pbComplLenFC[nMatchCol]; /* assign distance */ pWord->Body.bEditDistSpc = pCMP->ppbEditDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistStem = pCMP->ppbStemDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistFree = pCMP->ppbFreeDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcSbt = pCMP->ppbEditDistSbt[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcTrp = pCMP->ppbEditDistTrp[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcIns = pCMP->ppbEditDistIns[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcDel = pCMP->ppbEditDistDel[nMatchRow][nMatchCol]; pWord->Body.bLimitedCount = pCMP->ppbLimitedCount[nMatchRow][nMatchCol]; /* adjust to prefer spell correction over completion when applicable */ if (bComplLenFC_nSymbCount && (pWord->Body.bEditDistSpc + bComplLenFC_nSymbCount) <= bMaxEditDist && (pWord->Body.bEditDistSpc || pLingCmnInfo->Private.bTraceBuild)) { const ET9U8 bCost = bComplLenFC_nSymbCount; WLOG8B(fprintf(pLogFile8, " completion to spc term\n");) bComplLenFC_nSymbCount = 0; pWord->Body.bEditDistSpc = pWord->Body.bEditDistSpc + bCost; pWord->Body.bEditDistStem = pWord->Body.bEditDistStem + bCost; pWord->Body.bEditDistSpcIns = pWord->Body.bEditDistSpcIns + bCost; } /* suppress completions before point */ if (bComplLenFC_nSymbCount && nQualityCount < pLingCmnInfo->Private.wWordCompletionPoint) { if (bAvoidStems) { WLOG8B(fprintf(pLogFile8, " suppressed completion before point\n");) __STAT_AWLdb_Flex_EDC_Done(NULL); _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } /**/ if (bComplLenFC_nSymbCount && !pWord->Body.bEditDistSpc && (pWordSymbInfo->bNumSymbs < pLingCmnInfo->Private.wWordCompletionPoint)) { if (bAvoidStems) { WLOG8B(fprintf(pLogFile8, " assigning edit distance - before completion point\n");) pWord->Body.bEditDistSpc = 1; } } #if 1 if (pWord->Base.wWordLen > pLingCmnInfo->Private.wMaxWordLength + pWord->Body.bEditDistFree) { pWord->Body.bEditDistFree = 0; pWord->Base.wWordCompLen = 0; } else { pWord->Base.wWordCompLen = bComplLenFC_nSymbCount; } #else if (pbComplLen[nMatchCol] && (!pWord->Body.bEditDistSpc || bAllowSpcCmpl) && pWord->Base.wWordLen <= pLingCmnInfo->Private.wMaxWordLength) { pWord->Base.wWordCompLen = pbComplLen[nMatchCol]; } else { pWord->Base.wWordCompLen = 0; } #endif if (!pWord->Base.wWordCompLen && pWord->Base.wWordLen > pWordSymbInfo->bNumSymbs) { if (!pWord->Body.bEditDistSpc && pWord->Base.wWordLen <= pLingCmnInfo->Private.wMaxWordLength) { WLOG8B(fprintf(pLogFile8, " longer than input and shorter than max - special zero compl len\n");) pWord->Base.wWordCompLen = 0xFFFF; } else if (pWord->Base.wWordLen == pLingCmnInfo->Private.wMaxWordLength + pWord->Body.bEditDistFree) { WLOG8B(fprintf(pLogFile8, " longer than input and longer than max - special zero compl len\n");) pWord->Base.wWordCompLen = 0xFFFF; } } /* assign tap freq */ #ifdef ET9_USE_FLOAT_FREQS pWord->Body.xTapFreq = pCMP->ppxStemFreq[nMatchRow][nMatchCol] / pCMP->fTapFreqDivider; #else /* ET9_USE_FLOAT_FREQS */ if (nQualityCount <= 4) { pWord->Body.xTapFreq = (ET9FREQPART)pCMP->ppxStemFreq[nMatchRow][nMatchCol]; } else { pWord->Body.xTapFreq = (ET9FREQPART)(pCMP->ppxStemFreq[nMatchRow][nMatchCol] >> (4 * (nQualityCount - 4))); } #endif /* ET9_USE_FLOAT_FREQS */ if (!pWord->Body.xTapFreq) { pWord->Body.xTapFreq = 1; } ET9AssertLog(pWord->Body.xTapFreq >= 0); /* assign symb distance */ pWord->Body.xSymbDist = pCMP->ppxSymbDist[nMatchRow][nMatchCol]; } /* done */ WLOG8B(fprintf(pLogFile8, " match, bEditDistStem %d, bEditDistSpc %d, wWordCompLen %d, \n\n", pWord->Body.bEditDistStem, pWord->Body.bEditDistSpc, pWord->Base.wWordCompLen);) _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NONE; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * * */ static ET9STATUS ET9LOCALCALL __CalcEditDistance(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9STATUS eStatus; if (wIndex + wLength > ET9MAXWORDSIZE) { ET9AssertLog(0); return ET9STATUS_ERROR; } if (pLingCmnInfo->Private.ASpc.bSpcFeatures) { eStatus = __ADB_CalcEditDistanceFlex(pLingInfo, pWord, wIndex, wLength); } else { eStatus = __ET9AWCalcEditDistance(pLingInfo, pWord, wIndex, wLength); } ET9AssertLog(pWord->Body.xTapFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); return eStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __MakeSymbFreqsUnique(ET9SymbInfo * const pSymbInfo) { if (pSymbInfo->eInputType == ET9DISCRETEKEY || pSymbInfo->eInputType == ET9MULTITAPKEY || pSymbInfo->eInputType == ET9CUSTOMSET || pSymbInfo->bSymbType == ET9KTSMARTPUNCT) { return; } if (!pSymbInfo->bNumBaseSyms) { return; } { ET9U8 pbUsedFreq[0x100]; ET9DataPerBaseSym *pDPBS; _ET9ClearMem(pbUsedFreq, sizeof(pbUsedFreq)); for (pDPBS = &pSymbInfo->DataPerBaseSym[pSymbInfo->bNumBaseSyms - 1]; pDPBS >= &pSymbInfo->DataPerBaseSym[0]; --pDPBS) { /* ignore invalid Insure++ warnings */ if (!pbUsedFreq[pDPBS->bSymFreq]) { pbUsedFreq[pDPBS->bSymFreq] = 1; continue; } { ET9UINT nIndex; for (nIndex = pDPBS->bSymFreq; nIndex < 0xFF && pbUsedFreq[nIndex]; ++nIndex) { } for (; nIndex > 0 && pbUsedFreq[nIndex]; --nIndex) { } ET9AssertLog(nIndex <= 0xFF); ET9AssertLog(!pbUsedFreq[nIndex]); WLOG2(fprintf(pLogFile2, " unique, %3u becomes %3u\n", pDPBS->bSymFreq, nIndex);) pDPBS->bSymFreq = (ET9U8)nIndex; pbUsedFreq[nIndex] = 1; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __MakeSymbFreqsValid(ET9AWLingCmnInfo const * const pLingCmnInfo) { ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9UINT nSymbCount; ET9SymbInfo *pSymbInfo; WLOG2(fprintf(pLogFile2, "__MakeSymbFreqsValid\n");) pSymbInfo = pWordSymbInfo->SymbsInfo; for (nSymbCount = pWordSymbInfo->bNumSymbs; nSymbCount; --nSymbCount, ++pSymbInfo) { if (!pSymbInfo->bFreqsInvalidated) { continue; } WLOG2(fprintf(pLogFile2, " [%2u] new symb\n", (pSymbInfo - pWordSymbInfo->SymbsInfo));) pSymbInfo->bFreqsInvalidated = 0; if (!pLingCmnInfo->Private.ASpc.bSpcFeatures) { continue; } __MakeSymbFreqsUnique(pSymbInfo); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * * * * */ ET9INLINE static ET9STATUS ET9LOCALCALL __SelLstWordSearch(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator, const ET9U8 bSpcMode) { ET9STATUS eStatus; eStatus = _ET9AWCalcEditDistanceInit(pLingInfo, wIndex, wLength, bSpcMode, pLingInfo->pLingCmnInfo->Private.ASpc.bSpcFeatures); if (eStatus) { return eStatus; } eStatus = _ET9AWSelLstWordSearch(pLingInfo, pWord, wIndex, wLength, bFreqIndicator); _ET9AWCalcEditDistanceDone(pLingInfo); return eStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWSelLstWordMatch(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wFirstTap, const ET9U8 bTotalSymbInputs, ET9U8 * const pbyMatched, const ET9_FREQ_DESIGNATION bFreqIndicator) { ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); ET9AssertLog(pbyMatched != NULL); #if 0 WLOG2(fprintf(pLogFile2, "_ET9AWSelLstWordMatch, wFirstTap = %d, bTotalSymbInputs = %d\n", wFirstTap, bTotalSymbInputs);) WLOG2Word(pLogFile2, "word to match", pWord); #endif /* check if the word matches the input */ if (__CalcEditDistance(pLingInfo, pWord, wFirstTap, bTotalSymbInputs)) { *pbyMatched = 0; return ET9STATUS_NONE; } /* check if the substitution itself is a better match */ if (pWord->Base.wSubstitutionLen && pWord->Body.bEditDistSpc && pWord->Base.wSubstitutionLen <= ET9MAXWORDSIZE && !_ET9FindSpacesAndUnknown(pWord->Base.sSubstitution, pWord->Base.wSubstitutionLen)) { ET9AWPrivWordInfo sSubstAsWord; _InitPrivWordInfo(&sSubstAsWord); sSubstAsWord.Body.bWordSrc = pWord->Body.bWordSrc; sSubstAsWord.Base.wWordLen = pWord->Base.wSubstitutionLen; _ET9SymCopy(sSubstAsWord.Base.sWord, pWord->Base.sSubstitution, pWord->Base.wSubstitutionLen); if (!__CalcEditDistance(pLingInfo, &sSubstAsWord, wFirstTap, bTotalSymbInputs)) { if (sSubstAsWord.Body.bEditDistSpc < pWord->Body.bEditDistSpc || (sSubstAsWord.Body.bEditDistSpc == pWord->Body.bEditDistSpc && sSubstAsWord.Body.bEditDistStem < pWord->Body.bEditDistStem)) { pWord->Base.wWordCompLen = 0; /* for WLOG2Word */ WLOG2(fprintf(pLogFile2, "_ET9AWSelLstWordMatch, replacing shortcut with its substitution\n");) WLOG2Word(pLogFile2, "shortcut", pWord); WLOG2Word(pLogFile2, "substitution", &sSubstAsWord); *pWord = sSubstAsWord; } } } /* we have a match */ *pbyMatched = 1; { const ET9U16 wWordCompLenSave = pWord->Base.wWordCompLen; _ET9AWSelLstWordPreAdd(pLingInfo, pWord, wFirstTap, bTotalSymbInputs, 0, bFreqIndicator); pWord->Base.wWordCompLen = wWordCompLenSave; } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * * * * */ ET9STATUS ET9FARCALL _ET9AWSelLstWordPreAdd (ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wFirstTap, const ET9U8 bInputLength, const ET9U8 bShiftPoint, const ET9_FREQ_DESIGNATION bFreqIndicator) { ET9S16 swIndex; ET9U16 wCount; ET9SymbInfo *pSymbInfo; ET9BOOL bLockCompare; ET9BOOL bAllShifted; ET9BOOL bAllUnshifted; ET9SYMB *psSymb; ET9SYMB *psLower; ET9U32 dwLdbNum; ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; ET9AssertLog(pLingInfo); ET9AssertLog(pWord); ET9_UNUSED(bFreqIndicator); /* verify word */ #ifdef ET9_DEBUG { ET9SYMB *pSymb = pWord->Base.sWord; ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); for (wCount = pWord->Base.wWordLen; wCount; --wCount) { ET9AssertLog(*pSymb && *pSymb != (ET9SYMB)0xcccc); ++pSymb; } } #endif /* compLen is set in Add and in preAdd... */ if (pWord->Body.bEditDistSpc || pWord->Base.wWordLen < bInputLength) { pWord->Base.wWordCompLen = 0; } else { pWord->Base.wWordCompLen = (ET9U16)(pWord->Base.wWordLen - bInputLength); } /* log word */ #if 0 WLOG2Word(pLogFile2, "_ET9AWSelLstWordPreAdd, pWord", pWord); #endif /* save the first char's original value */ pWord->Body.sPureFirstChar = pWord->Base.sWord[0]; /* from here on this function handles shift properties - skip for some languages */ if (!_ET9_LanguageSpecific_ApplyShifting(pLingInfo, pWord) || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE) { WLOG2(fprintf(pLogFile2, "_ET9AWSelLstWordPreAdd, skipping shift handling\n");) /* still need to comply with locked chars */ { bLockCompare = 0; swIndex = (ET9S16)((ET9S16)bInputLength - 1); ET9AssertLog(wFirstTap + swIndex < ET9MAXWORDSIZE); psSymb = &pWord->Base.sWord[bInputLength - 1]; pSymbInfo = &pWordSymbInfo->SymbsInfo[wFirstTap + swIndex]; dwLdbNum = (pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum; for (; swIndex >= 0; --swIndex, --pSymbInfo, --psSymb) { /* ignore invalid Insure++ warnings */ if (!bLockCompare && pSymbInfo->bLocked) { bLockCompare = 1; } if (swIndex == (ET9S16)(bInputLength - 1) && GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE) { continue; } if (bLockCompare) { ET9AssertLog(pSymbInfo->sLockedSymb && pSymbInfo->sLockedSymb != (ET9SYMB)0xcccc); ET9AssertLog(_ET9SymToLower(*psSymb, dwLdbNum) == _ET9SymToLower(pSymbInfo->sLockedSymb, dwLdbNum) || _ET9SymToUpper(*psSymb, dwLdbNum) == _ET9SymToUpper(pSymbInfo->sLockedSymb, dwLdbNum)); *psSymb = pSymbInfo->sLockedSymb; } } } return ET9STATUS_NONE; } /* acronym? capitalized? */ pWord->Body.bIsAcronym = 0; pWord->Body.bIsCapitalized = 0; if (bInputLength > 1 && pWord->Base.wWordLen > 1) { const ET9UINT nCmpLen = __ET9Max(bInputLength, pWord->Base.wWordLen); ET9SYMB *psCurr; ET9UINT nIndex; ET9UINT nShiftCount; ET9UINT nLowerCount; ET9UINT nUpperCount; ET9BOOL bAllShifted; bAllShifted = (bInputLength) ? 1 : (pWordSymbInfo->Private.eLastShiftState == ET9CAPSLOCK) ? 1 : 0; nShiftCount = 0; nLowerCount = 0; nUpperCount = 0; psCurr = &pWord->Base.sWord[0]; pSymbInfo = &pWordSymbInfo->SymbsInfo[wFirstTap]; for (nIndex = 0; nIndex < nCmpLen && !nLowerCount; ++nIndex, ++psCurr, ++pSymbInfo) { if (nIndex < bInputLength) { if (!pSymbInfo->eShiftState) { bAllShifted = 0; } if (pSymbInfo->eShiftState && (!pSymbInfo->bTraceIndex || pSymbInfo->bTraceProbability)) { ++nShiftCount; } } if (nIndex < pWord->Base.wWordLen) { if (_ET9SymIsLower(*psCurr, pWordSymbInfo->Private.dwLocale)) { ++nLowerCount; } else if (_ET9SymIsUpper(*psCurr, pWordSymbInfo->Private.dwLocale)) { ++nUpperCount; } } } /* the rule for acronym is simplified to to improve performance and intended to catch the case without "input shift" */ if (nUpperCount > 1 && nLowerCount <= 1 && nShiftCount < nUpperCount && !bAllShifted) { WLOG2(fprintf(pLogFile2, "_ET9AWSelLstWordPreAdd, word is an acronym\n");) pWord->Body.bIsAcronym = 1; } /* the rule for capitalization is simplified to to improve performance and intended to catch the case without "input shift" */ if (nUpperCount == 1 && nLowerCount == 1 && nShiftCount == 0) { WLOG2(fprintf(pLogFile2, "_ET9AWSelLstWordPreAdd, word is capitalized\n");) pWord->Body.bIsCapitalized = 1; } } /* all shifted? */ { ET9U16 wShiftedCount = 0; bAllShifted = 1; bAllUnshifted = 1; pSymbInfo = &pWordSymbInfo->SymbsInfo[wFirstTap]; for (wCount = 0; wCount < bInputLength; ++wCount, ++pSymbInfo) { if (!__IsShiftNeutralSymb(pSymbInfo) || pWordSymbInfo->Private.eLastShiftState == ET9CAPSLOCK) { if (pSymbInfo->eShiftState) { ++wShiftedCount; bAllUnshifted = 0; /* if current word has shifting after a lockpoint, cancels any compounding downshift logic */ if (wCount > pLingInfo->pLingCmnInfo->Private.wCurrLockPoint) { pWordSymbInfo->Private.bCompoundingDownshift = 0; } } else if (pSymbInfo->bForcedLowercase || pSymbInfo->bAutoDowncase) { bAllShifted = 0; } else { bAllShifted = 0; bAllUnshifted = 0; break; } } } if (wShiftedCount <= 1) { bAllShifted = 0; } } /* handle shift for word */ bLockCompare = 0; swIndex = (ET9S16)((ET9S16)bInputLength - 1); ET9AssertLog(wFirstTap + swIndex < ET9MAXWORDSIZE); psSymb = &pWord->Base.sWord[bInputLength - 1]; pSymbInfo = &pWordSymbInfo->SymbsInfo[wFirstTap + swIndex]; dwLdbNum = (pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum; if (bShiftPoint > 0) { --pSymbInfo; } for (; swIndex >= 0; --swIndex, --pSymbInfo, --psSymb) { /* ignore invalid Insure++ warnings */ if (bShiftPoint > 0 && bShiftPoint == swIndex) { ++pSymbInfo; continue; } if (!bLockCompare && pSymbInfo->bLocked) { bLockCompare = 1; } if (bLockCompare) { ET9AssertLog(pSymbInfo->sLockedSymb && pSymbInfo->sLockedSymb != (ET9SYMB)0xcccc); ET9AssertLog(_ET9SymToLower(*psSymb, dwLdbNum) == _ET9SymToLower(pSymbInfo->sLockedSymb, dwLdbNum) || _ET9SymToUpper(*psSymb, dwLdbNum) == _ET9SymToUpper(pSymbInfo->sLockedSymb, dwLdbNum)); *psSymb = pSymbInfo->sLockedSymb; /* any locked character will not be downcased; safety clear */ pSymbInfo->bAutoDowncase = 0; } else if (!__IsShiftNeutralSymb(pSymbInfo) || pWordSymbInfo->Private.eLastShiftState == ET9CAPSLOCK) { if (bAllShifted || pSymbInfo->eShiftState) { ET9AssertLog(swIndex >= pWord->Base.wWordLen || _ET9SymToUpper(*psSymb, dwLdbNum) && _ET9SymToUpper(*psSymb, dwLdbNum) != (ET9SYMB)0xcccc); *psSymb = _ET9SymToUpper(*psSymb, dwLdbNum); /* this will write outside the actual string on short spc, but that should be ok... */ /* any shifted character will not be downcased; safety clear */ pSymbInfo->bAutoDowncase = 0; } else if ((pSymbInfo->bForcedLowercase || pSymbInfo->bAutoDowncase)) { *psSymb = _ET9SymToLower(*psSymb, dwLdbNum); } /* otherwise, if actively downcasing, make sure to 'mark' sym entry */ else if (pWordSymbInfo->Private.bCompoundingDownshift) { *psSymb = _ET9SymToLower(*psSymb, dwLdbNum); pSymbInfo->bAutoDowncase = 1; } } } /* handle caps lock for long spc and completion */ if (bAllShifted || pWordSymbInfo->Private.eLastShiftState == ET9CAPSLOCK) { swIndex = (ET9S16)(pWord->Base.wWordLen - 1); psSymb = &pWord->Base.sWord[swIndex]; for (; swIndex > bInputLength - 1; --swIndex, --psSymb) { *psSymb = _ET9SymToUpper(*psSymb, dwLdbNum); } } else if (pWordSymbInfo->Private.bCompoundingDownshift) { /* if actively downcasing.... */ /* make sure to lowercase all syms in completion */ swIndex = (ET9S16)(pWord->Base.wWordLen - 1); psSymb = &pWord->Base.sWord[swIndex]; for (; swIndex > bInputLength - 1; --swIndex, --psSymb) { *psSymb = _ET9SymToLower(*psSymb, dwLdbNum); } } /* handle shift for substitution */ if (pWord->Base.wSubstitutionLen) { psLower = pWord->Base.sSubstitution; /* if all syms shifted, do same for substitution */ if (bAllShifted) { for (wCount = pWord->Base.wSubstitutionLen; wCount; --wCount, ++psLower) { *psLower = _ET9SymToUpper(*psLower, dwLdbNum); } } else if (bAllUnshifted) { for (wCount = pWord->Base.wSubstitutionLen; wCount; --wCount, ++psLower) { *psLower = _ET9SymToLower(*psLower, dwLdbNum); } } /* if first sym shifted, do same for substitution */ else { pSymbInfo = &pWordSymbInfo->SymbsInfo[wFirstTap]; if (pSymbInfo->eShiftState && (!__IsShiftNeutralSymb(pSymbInfo) || pWordSymbInfo->Private.eLastShiftState == ET9CAPSLOCK)) { *psLower = _ET9SymToUpper(*psLower, dwLdbNum); } } } ET9AssertLog(pWord->Body.bIsAcronym <= 1); ET9AssertLog(pWord->Body.bIsCapitalized <= 1); ET9AssertLog(pWord->Body.bIsTop5 <= 1); ET9AssertLog(pWord->Body.bIsUDBWord <= 1); ET9AssertLog(pWord->Body.bWordDesignation <= 4); ET9AssertLog(pWord->Body.bNLMOrder <= ET9NLM_MAX_ORDER); ET9AssertLog(pWord->Body.bCollectionPrio <= ET9NLM_MAX_ORDER); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * * * * */ ET9STATUS ET9FARCALL _ET9AWSelLstWordMatch( ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength, ET9U8 *pbFound, const ET9_FREQ_DESIGNATION bFreqIndicator) { ET9STATUS wStatus = ET9STATUS_NONE; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); *pbFound = 0; /* match the word against all taps */ if (pWord->Base.wWordLen >= pLingInfo->pLingCmnInfo->Private.wCurrMinSourceLength) { wStatus = __ET9AWSelLstWordMatch(pLingInfo, pWord, wIndex, (ET9U8)wLength, pbFound, bFreqIndicator); } return wStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * * * */ ET9STATUS ET9FARCALL _ET9AWSelLstWordSearch(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator) { ET9STATUS wStatus = ET9STATUS_NONE; ET9U8 bFound = 0; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); wStatus = _ET9AWSelLstWordMatch(pLingInfo, pWord, wIndex, wLength, &bFound, bFreqIndicator); if (!wStatus && bFound) { /* attempt to add, not interested in return value */ _ET9AWSelLstAdd(pLingInfo, pWord, wLength, bFreqIndicator); } return wStatus; } /*---------------------------------------------------------------------------*/ /** \ingroup et9asys * This function set that exact word sellist positioning parameters * * @param pLingInfo Pointer to alphabetic information structure. * @param eSetting ET9AEXACTINLIST setting desired. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWSetExactInList(ET9AWLingInfo * const pLingInfo, const ET9AEXACTINLIST eSetting) { ET9STATUS wStatus; wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (wStatus) { return wStatus; } wStatus = _ET9SettingsInhibited(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo); if (wStatus) { return wStatus; } { ET9UINT nChangeCount = 0; switch (eSetting) { case ET9AEXACTINLIST_OFF: if (pLingInfo->pLingCmnInfo->Private.bStateExactInList) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactInList = 0; } break; case ET9AEXACTINLIST_FIRST: if (!pLingInfo->pLingCmnInfo->Private.bStateExactInList) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactInList = 1; } if (pLingInfo->pLingCmnInfo->Private.bStateExactIsDefault) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactIsDefault = 0; } if (pLingInfo->pLingCmnInfo->Private.bStateExactLast) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactLast = 0; } break; case ET9AEXACTINLIST_LAST: if (!pLingInfo->pLingCmnInfo->Private.bStateExactInList) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactInList = 1; } if (pLingInfo->pLingCmnInfo->Private.bStateExactIsDefault) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactIsDefault = 0; } if (!pLingInfo->pLingCmnInfo->Private.bStateExactLast) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactLast = 1; } break; case ET9AEXACTINLIST_DEFAULT: if (!pLingInfo->pLingCmnInfo->Private.bStateExactInList) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactInList = 1; } if (!pLingInfo->pLingCmnInfo->Private.bStateExactIsDefault) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactIsDefault = 1; } if (pLingInfo->pLingCmnInfo->Private.bStateExactLast) { ++nChangeCount; pLingInfo->pLingCmnInfo->Private.bStateExactLast = 0; } break; default: return ET9STATUS_BAD_PARAM; } if (nChangeCount) { _ET9InvalidateSymbInfo(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo); } } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** \ingroup et9asys * This function gets the current exact word sellist positioning setting * * @param pLingInfo Pointer to alphabetic information structure. * @param peSetting Pointer to collect ET9AEXACTINLIST setting. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWGetExactInList(ET9AWLingInfo * const pLingInfo, ET9AEXACTINLIST * const peSetting) { ET9STATUS wStatus; if ((wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo)) != ET9STATUS_NONE) { return wStatus; } if (!peSetting) { return ET9STATUS_INVALID_MEMORY; } if (!ET9EXACTINLIST(pLingInfo->pLingCmnInfo)) { *peSetting = ET9AEXACTINLIST_OFF; } else if (ET9EXACTLAST(pLingInfo->pLingCmnInfo)) { *peSetting = ET9AEXACTINLIST_LAST; } else if (ET9EXACTISDEFAULT(pLingInfo->pLingCmnInfo)) { *peSetting = ET9AEXACTINLIST_DEFAULT; } else { *peSetting = ET9AEXACTINLIST_FIRST; } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** \ingroup et9asys * Set auto append in list. * This function set that auto append's should be included in lists. * * @param pLingInfo Pointer to alphabetic information structure. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWSetAutoAppendInList(ET9AWLingInfo * const pLingInfo) { ET9STATUS wStatus; wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (wStatus) { return wStatus; } wStatus = _ET9SettingsInhibited(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo); if (wStatus) { return wStatus; } if (!pLingInfo->pLingCmnInfo->Private.bStateAutoAppendInList) { pLingInfo->pLingCmnInfo->Private.bStateAutoAppendInList = 1; _ET9InvalidateSymbInfo(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo); } return wStatus; } /*---------------------------------------------------------------------------*/ /** \ingroup et9asys * Clear auto append in list. * This function clears that auto append's should be included in lists. * * @param pLingInfo Pointer to alphabetic information structure. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWClearAutoAppendInList(ET9AWLingInfo * const pLingInfo) { ET9STATUS wStatus; wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (wStatus) { return wStatus; } wStatus = _ET9SettingsInhibited(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo); if (wStatus) { return wStatus; } if (pLingInfo->pLingCmnInfo->Private.bStateAutoAppendInList) { pLingInfo->pLingCmnInfo->Private.bStateAutoAppendInList = 0; _ET9InvalidateSymbInfo(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo); } return wStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __ET9GetSpecialCharInfo(ET9AWLingInfo * const pLingInfo, const ET9U16 wStartPoint, ET9U16 * const pwPosition, ET9ASPECIAL * const peType) { ET9S16 swIndex; ET9SymbInfo *pSymbInfo; ET9UINT nTraceSegCount = 0; ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; ET9AssertLog(pWordSymbInfo != NULL); ET9AssertLog(pwPosition != NULL); ET9AssertLog(peType != NULL); WLOG2(fprintf(pLogFile2, "__ET9GetSpecialCharInfo, wStartPoint = %d, bNumSymbs = %d\n", wStartPoint, pWordSymbInfo->bNumSymbs);) *pwPosition = 0; *peType = NOSPECIAL; if (!pWordSymbInfo->bNumSymbs) { WLOG2(fprintf(pLogFile2, "..called with NO symbs\n");) return; } swIndex = (ET9S16)((ET9S16)pWordSymbInfo->bNumSymbs - 1); ET9AssertLog(swIndex < ET9MAXWORDSIZE); pSymbInfo = &pWordSymbInfo->SymbsInfo[swIndex]; for ( ; swIndex >= (ET9S16)wStartPoint; --swIndex, --pSymbInfo) { /* ignore invalid Insure++ warnings */ WLOG2(fprintf(pLogFile2, "..trying swIndex %d (position %d)\n", swIndex, (swIndex + 1 - wStartPoint));) if (pSymbInfo->bLocked) { WLOG2(fprintf(pLogFile2, "..found LOCKPOINT\n");) *peType = LOCKPOINT; break; } /* skipping traced symbs */ if (pSymbInfo->bTraceIndex) { if (nTraceSegCount || swIndex == (ET9S16)wStartPoint) { WLOG2(fprintf(pLogFile2, "..skipping traced symb (1)\n");) continue; } if (pSymbInfo->bTraceIndex == (pSymbInfo - 1)->bTraceIndex) { WLOG2(fprintf(pLogFile2, "..skipping traced symb (2)\n");) continue; } ++nTraceSegCount; WLOG2(fprintf(pLogFile2, "..going to examine one trace symb\n");) } /* if symb is last in the input and exact, do NOT make it special, handled by auto append exact */ if (swIndex + 1 == pWordSymbInfo->bNumSymbs && __IsExactSymb(pSymbInfo)) { WLOG2(fprintf(pLogFile2, "..skipping trailing EXPLICIT (from single symbol)\n");) continue; } /* if the character is "explicit"... */ if (pSymbInfo->bAmbigType == ET9EXACT && _ET9_IsPunctOrNumeric(pSymbInfo->DataPerBaseSym[0].sChar[0])) { WLOG2(fprintf(pLogFile2, "..found EXPLICIT (from exact)\n");) *peType = EXPLICIT; break; } if (pSymbInfo->bSymbType == ET9KTPUNCTUATION) { WLOG2(fprintf(pLogFile2, "..found PUNCT\n");) *peType = PUNCT; break; } if (pSymbInfo->bSymbType == ET9KTSMARTPUNCT) { WLOG2(fprintf(pLogFile2, "..found SMARTPUNCT\n");) *peType = SMARTPUNCT; break; } if (__IsExactSymb(pSymbInfo) && _ET9_IsPunctOrNumeric(pSymbInfo->DataPerBaseSym[0].sChar[0])) { WLOG2(fprintf(pLogFile2, "..found EXPLICIT (from single symbol)\n");) *peType = EXPLICIT; break; } } if (*peType != NOSPECIAL) { *pwPosition = swIndex + 1 - wStartPoint; WLOG2(fprintf(pLogFile2, "..SPECIAL, wPosition = %d (wStartPoint %d)\n", *pwPosition, wStartPoint);) } else { WLOG2(fprintf(pLogFile2, "..NOSPECIAL\n");) } return; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9BOOL ET9LOCALCALL __SuppressExact(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9UINT nCount; ET9SymbInfo *pSymbInfo; pSymbInfo = pWordSymbInfo->SymbsInfo; for (nCount = pWordSymbInfo->bNumSymbs; nCount; --nCount, ++pSymbInfo) { if (pSymbInfo->bTraceIndex) { return 1; } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9BOOL ET9LOCALCALL __ExactInList(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9U8 bNumSymbs = pWordSymbInfo->bNumSymbs; const ET9BOOL bLastInputIsMultitap = (ET9BOOL)(bNumSymbs && pWordSymbInfo->SymbsInfo[bNumSymbs - 1].eInputType == ET9MULTITAPKEY); if (__SuppressExact(pLingInfo)) { return 0; } if (pLingCmnInfo->Private.bHasFunctionKeySymbInExact) { return 0; } return (ET9BOOL)(ET9EXACTINLIST(pLingCmnInfo) || bLastInputIsMultitap || pWordSymbInfo->SymbsInfo[bNumSymbs - 1].bLocked); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9BOOL ET9LOCALCALL __ExactHasOverrideInhibitor(ET9AWLingCmnInfo * const pLingCmnInfo) { ET9WordSymbInfo const * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; WLOG2(fprintf(pLogFile2, "__ExactHasOverrideInhibitor\n");) /* check digits, puncts etc in the input */ { ET9UINT nAtCount = 0; ET9UINT nPunctCount = 0; ET9UINT nNumberCount = 0; ET9UINT nCount; ET9SymbInfo const *pSymbInfo; pSymbInfo = pWordSymbInfo->SymbsInfo; for (nCount = pWordSymbInfo->bNumSymbs; nCount; --nCount, ++pSymbInfo) { const ET9SYMB sSymb = pSymbInfo->DataPerBaseSym[0].sChar[0]; if (sSymb == '@') { ++nAtCount; } switch (pSymbInfo->bSymbType) { case ET9KTLETTER: break; case ET9KTNUMBER: ++nNumberCount; break; case ET9KTPUNCTUATION: ++nPunctCount; break; case ET9KTSMARTPUNCT: { const ET9SymbClass eClass = _ET9_GetSymbolClass(sSymb); if (eClass == ET9_NumbrSymbClass) { ++nNumberCount; } else { ++nPunctCount; } } break; default: { const ET9SymbClass eClass = _ET9_GetSymbolClass(sSymb); if (eClass == ET9_NumbrSymbClass) { ++nNumberCount; } else if (eClass == ET9_PunctSymbClass) { ++nPunctCount; } } break; } } WLOG2(fprintf(pLogFile2, " nAtCount %u, nNumberCount %u, nPunctCount %u\n", nAtCount, nNumberCount, nPunctCount);) if (nAtCount || nNumberCount || nPunctCount > 1) { WLOG2(fprintf(pLogFile2, " YES\n");) return 1; } } /* the rest is limited to low & medium */ switch (pLingCmnInfo->Private.eSelectionListCorrectionMode) { case ET9ASLCORRECTIONMODE_LOW: case ET9ASLCORRECTIONMODE_MEDIUM: break; default: WLOG2(fprintf(pLogFile2, " High -> NO\n");) return 0; } /* check completion */ { ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[1]]; /* not when completion (not term) */ if (!pWord->Base.bIsTerm) { WLOG2(fprintf(pLogFile2, " !Term -> YES\n");) return 1; } } /* done */ WLOG2(fprintf(pLogFile2, " NO\n");) return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9BOOL ET9LOCALCALL __HasUserShiftOverrideInhibit(ET9AWLingCmnInfo * const pLingCmnInfo) { ET9WordSymbInfo const * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; WLOG2(fprintf(pLogFile2, "__HasUserShiftOverrideInhibit\n");) if (pWordSymbInfo->bNumSymbs < 2) { WLOG2(fprintf(pLogFile2, " bNumSymbs < 2 -> NO\n");) return 0; } if (pWordSymbInfo->SymbsInfo[0].eShiftState != ET9SHIFT) { WLOG2(fprintf(pLogFile2, " eShiftState != ET9SHIFT -> NO\n");) return 0; } if (!pLingCmnInfo->Private.bCurrBuildHasShiftSignificance) { WLOG2(fprintf(pLogFile2, " !bCurrBuildHasShiftSignificance -> NO\n");) return 0; } { ET9U16 wContextLen = pLingCmnInfo->Private.pContextWords[0].wLen; if (wContextLen < ET9MAXWORDSIZE) { pLingCmnInfo->Private.pContextWords[0].sString[wContextLen++] = ' '; } if (_ET9IsAutoCapSituation(pWordSymbInfo, pLingCmnInfo->Private.pContextWords[0].sString, wContextLen)) { WLOG2(fprintf(pLogFile2, " IsAutoCapSituation -> NO\n");) return 0; } } { ET9WordSymbInfo const * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[1]]; /* spc can't win */ if (pWord->Body.bEditDistSpc) { WLOG2(fprintf(pLogFile2, " bEditDistSpc -> YES\n");) return 1; } /* upper wins */ if (_ET9SymIsUpper(pWord->Body.sPureFirstChar, pWordSymbInfo->Private.dwLocale)) { WLOG2(fprintf(pLogFile2, " upper(firstChar) -> NO\n");) return 0; } /* wrong length can't win */ if (pWord->Base.wWordLen != pWordSymbInfo->bNumSymbs) { WLOG2(fprintf(pLogFile2, " wWordLen != bNumSymbs -> YES\n");) return 1; } /* certain amount of regional correction can win */ if (pWord->Body.bEditDistStem <= __MaxStemDistanceFromInputLength(pWordSymbInfo->bNumSymbs)) { WLOG2(fprintf(pLogFile2, " bEditDistStem <= MaxStemDistance -> NO\n");) return 0; } /* the rest can't win */ WLOG2(fprintf(pLogFile2, " YES\n");) return 1; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9BOOL ET9LOCALCALL __HasStemDistanceOverrideInhibit(ET9AWLingCmnInfo * const pLingCmnInfo) { ET9WordSymbInfo const * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; switch (pLingCmnInfo->Private.eSelectionListCorrectionMode) { case ET9ASLCORRECTIONMODE_LOW: case ET9ASLCORRECTIONMODE_MEDIUM: break; default: return 0; } if (pWordSymbInfo->bNumSymbs >= 5) { ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[1]]; if (pWord->Body.bEditDistStem > pWordSymbInfo->bNumSymbs / 2) { return 1; } } if (pWordSymbInfo->bNumSymbs >= 1) { ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[1]]; if (!pWord->Body.bIsTop5) { if ((pWord->Body.bEditDistSpc - pWord->Body.bEditDistSpcTrp) > 1) { return 1; } if (pWord->Body.bEditDistStem > __MaxStemDistanceFromInputLength(pWordSymbInfo->bNumSymbs)) { return 1; } } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9BOOL ET9LOCALCALL __HasNumericStem(ET9AWLingCmnInfo * const pLingCmnInfo, ET9SYMB const * const psString, const ET9UINT nStrLen) { ET9WordSymbInfo const * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9INT snIndex; for (snIndex = pWordSymbInfo->bNumSymbs - 1; snIndex >= 0; --snIndex) { if (pWordSymbInfo->SymbsInfo[snIndex].eInputType != ET9EXPLICITSYM) { break; } } if (snIndex < 0) { return 0; } { ET9UINT nStemLen = (ET9UINT)(snIndex + 1); if (nStemLen > nStrLen) { return 0; } return _ET9_IsNumericString(psString, nStemLen); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9BOOL ET9LOCALCALL __HasInitialNonLetter(ET9SYMB const * const psString, const ET9UINT nStrLen) { if (nStrLen < 2) { return 0; } { const ET9SymbClass eClass = _ET9_GetSymbolClass(psString[0]); return (ET9BOOL)(eClass != ET9_AlphaSymbClass); } } /*---------------------------------------------------------------------------*/ /** * This function builds selection list for the current tap sequence. * If needed it will perform a number of list builds to catchup on the input changes. * * @param pLingInfo Pointer to alphabetic information structure. * @param pbTotalWords (out) number of candidate words found. * @param pbDefaultListIndex (out) suggested default candidate. * @param pwGestureValue (out) gesture return value. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWSelLstBuild(ET9AWLingInfo * const pLingInfo, ET9U8 * const pbTotalWords, ET9U8 * const pbDefaultListIndex, ET9U16 * const pwGestureValue) { ET9STATUS wStatus; __ProfileStart; WLOG2Shrink(pLogFile2); WLOG2(fprintf(pLogFile2, "\n************************* SELECTION LIST BUILD START *************************\n\n");) WLOG2(fprintf(pLogFile2, " sizeof(ET9AWLingCmnPrivate) = %6u\n", sizeof(ET9AWLingCmnPrivate));) WLOG2(fprintf(pLogFile2, " sizeof(ET9ASpc) = %6u\n", sizeof(ET9ASpc));) #ifdef _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH WLOG2(fprintf(pLogFile2, " sizeof(ET9ADdbCharHashInfo) = %6u\n", sizeof(ET9ADdbCharHashInfo));) #endif /* _ET9_ACTIVATE_ADDB_COMPARE_CHAR_HASH */ WLOG2(fprintf(pLogFile2, " sizeof(_ET9AW_WordCollection) = %6u\n", sizeof(_ET9AW_WordCollection));) WLOG2(fprintf(pLogFile2, "\n");) wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (wStatus) { return wStatus; } if (!pbTotalWords || !pbDefaultListIndex || !pwGestureValue) { return ET9STATUS_INVALID_MEMORY; } *pbTotalWords = 0; *pbDefaultListIndex = 0; *pwGestureValue = 0; #ifdef EVAL_BUILD { ET9U16 wUpdateCount = 0; if (pLingInfo->pLingCmnInfo->pRUDBInfo) { wUpdateCount = pLingInfo->pLingCmnInfo->pRUDBInfo->wUpdateCounter; } if (_ET9Eval_HasExpired(&pLingInfo->pLingCmnInfo->Base, wUpdateCount)) { return ET9STATUS_EVAL_BUILD_EXPIRED; } } #endif /*EVAL_BUILD*/ { #ifdef ET9_DEBUG const ET9U32 dwLDBStart = pLingInfo->pLingCmnInfo->dwLdbNum; const ET9U16 wLDBInitOK_start = pLingInfo->Private.wLDBInitOK; const ET9U32 dwSetActiveLanguageCount_start = pLingInfo->pLingCmnInfo->Private.dwSetActiveLanguageCount; #endif ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9U8 bNumSymbs = pWordSymbInfo->bNumSymbs; const ET9BOOL bHasRegionalInfo = _ET9HasRegionalInfo(pWordSymbInfo); ET9U8 i; ET9U8 bLastBuildLen; ET9U8 bValidLen; ET9U8 bLockLen; if (pWordSymbInfo->bNumSymbs > ET9MAXWORDSIZE) { _ET9AW_ResetWordList(pLingInfo); WLOG2(fprintf(pLogFile2, "too many symbs - aborting\n");) return ET9STATUS_ERROR; } __LogInputSymbs(pLingInfo, pWordSymbInfo->bNumSymbs, pLogFile2); __LogUDB(pLingInfo, pLogFile2); __LogCDB(pLingInfo, pLogFile2); if (pLingCmnInfo->Base.bSelListInvalidated) { pLingCmnInfo->Base.bSelListInvalidated = 0; WLOG2(fprintf(pLogFile2, "selection list invalidated - resetting\n");) __ResetAllCaptureInfo(pLingInfo); _ET9AW_ResetWordList(pLingInfo); pLingCmnInfo->Private.bLastBuildShrinking = 0; pLingCmnInfo->Private.bSpcDuringBuild = 0; pLingCmnInfo->Private.bExpandAsDuringBuild = 0; pLingCmnInfo->Private.bLastBuildLen = pWordSymbInfo->bNumSymbs; { ET9U8 bIndex; for (bIndex = 0; bIndex < pWordSymbInfo->bNumSymbs; ++bIndex) { if (pWordSymbInfo->Private.bRequiredLocate) { pLingCmnInfo->Base.bSymbInvalidated[bIndex] = 0; } } } if (pWordSymbInfo->Private.sRequiredWord.wLen && pWordSymbInfo->bNumSymbs) { ET9U16 wSymbolLen; ET9AWPrivWordInfo sWord; _ET9SimpleWordToPrivWord(&pWordSymbInfo->Private.sRequiredWord, &sWord); WLOG2Word(pLogFile2, "required word sets up defaults, word", &sWord); if (pWordSymbInfo->Private.sRequiredWord.wLen != pWordSymbInfo->bNumSymbs) { pLingCmnInfo->Private.bSpcDuringBuild = 1; } for (wSymbolLen = 1; wSymbolLen < pWordSymbInfo->bNumSymbs; ++wSymbolLen) { if (wSymbolLen <= pWordSymbInfo->Private.sRequiredWord.wLen) { sWord.Base.wWordCompLen = (ET9U16)(sWord.Base.wWordLen - wSymbolLen); } __CaptureDefault(pLingInfo, &sWord, (ET9U16)(wSymbolLen + 1), 0); } } } bLastBuildLen = pLingCmnInfo->Private.bLastBuildLen; if (pLingCmnInfo->Base.bSymbsInfoInvalidated) { WLOG2(fprintf(pLogFile2, "modified symbols since last build\n\n");) pLingCmnInfo->Base.bSymbsInfoInvalidated = 0; } else { WLOG2(fprintf(pLogFile2, "*** building without any symbols being modified since last build\n\n");) } #if 1 WLOG2(fprintf(pLogFile2, "pLingInfo x, pWordSymbInfo x, bNumSymbs = %d, bLastBuildLen = %d\n\n", bNumSymbs, bLastBuildLen);) #else WLOG2(fprintf(pLogFile2, "pLingInfo %p, pWordSymbInfo %p, bNumSymbs = %d, bLastBuildLen = %d\n\n", pLingInfo, pWordSymbInfo, bNumSymbs, bLastBuildLen);) #endif #ifdef ET9_ACTIVATE_SLST_STATS _ET9AWSpcClearStats(pLingInfo); #endif /* handle tracking */ if (!bLastBuildLen) { pLingCmnInfo->Private.bSpcDuringBuild = 0; pLingCmnInfo->Private.bExpandAsDuringBuild = 0; } if (ET9_SPC_IS_ACTIVE(pLingCmnInfo->Private.ASpc.eMode)) { pLingCmnInfo->Private.bSpcDuringBuild = 1; } if (ET9EXPANDAUTOSUB(pLingCmnInfo)) { pLingCmnInfo->Private.bExpandAsDuringBuild = 1; } WLOG2(fprintf(pLogFile2, "bSpcDuringBuild = %d, bExpandAsDuringBuild = %d\n\n", pLingCmnInfo->Private.bSpcDuringBuild, pLingCmnInfo->Private.bExpandAsDuringBuild);) /* handle language change (rebuild all if changed) */ if ((pLingCmnInfo->dwFirstLdbNum != pLingCmnInfo->Private.dwCurrBuildLangFirst) || (pLingCmnInfo->dwSecondLdbNum != pLingCmnInfo->Private.dwCurrBuildLangSecond)) { for (i = 0; i < bNumSymbs; ++i) { pLingCmnInfo->Base.bSymbInvalidated[i] = 1; pLingCmnInfo->Base.bLockInvalidated[i] = 0; /* or it will uninvalidate the symb */ } WLOG2(fprintf(pLogFile2, "*** language changed\n\n");) pLingCmnInfo->Private.dwCurrBuildLangFirst = pLingCmnInfo->dwFirstLdbNum; pLingCmnInfo->Private.dwCurrBuildLangSecond = pLingCmnInfo->dwSecondLdbNum; } /* handle lock invalidation */ bLockLen = 0; i = pWordSymbInfo->bNumSymbs; if (i) { ET9SymbInfo *pSymbInfo = &pWordSymbInfo->SymbsInfo[i - 1]; for (; i; --i, --pSymbInfo) { /* ignore invalid Insure++ warnings */ if (pSymbInfo->bLocked) { bLockLen = i; } } } for (i = 0; i < bNumSymbs; ++i) { if (pLingCmnInfo->Base.bLockInvalidated[i]) { pLingCmnInfo->Base.bLockInvalidated[i] = 0; if (i < bLockLen) { pLingCmnInfo->Base.bSymbInvalidated[i] = 0; WLOG2(fprintf(pLogFile2, "symbol locked @ %d, within lock => uninvalidate symbol\n\n", i+1);) } else { WLOG2(fprintf(pLogFile2, "symbol locked @ %d, outside lock => symbol invalidation state remains\n\n", i+1);) } } } /* calculate valid (non invalidated) len */ for (bValidLen = 0; bValidLen < bNumSymbs; ++bValidLen) { if (pLingCmnInfo->Base.bSymbInvalidated[bValidLen]) { break; } } /* check for function key symb in exact */ { ET9U16 wSymbIndex; ET9SimpleWord sSimpleWord; pLingCmnInfo->Private.bHasFunctionKeySymbInExact = 0; if (!ET9GetExactWord(pWordSymbInfo, &sSimpleWord, pLingInfo->Private.pConvertSymb, pLingInfo->Private.pConvertSymbInfo)) { for (wSymbIndex = 0; wSymbIndex < sSimpleWord.wLen; ++wSymbIndex) { if (ET9IsFunctionKeySymbol(sSimpleWord.sString[wSymbIndex])) { pLingCmnInfo->Private.bHasFunctionKeySymbInExact = 1; break; } } } } /* want to build (back) to valid len in case there were clears and adds in between */ if (bValidLen < bNumSymbs && bValidLen < bLastBuildLen) { if (!bValidLen) { WLOG2(fprintf(pLogFile2, "performing catchup clear all\n\n");) bLastBuildLen = 0; pLingCmnInfo->Private.bLastBuildLen = bLastBuildLen; pLingCmnInfo->Private.sBuildInfo.bCaptureInvalidated = 1; } else { WLOG2(fprintf(pLogFile2, "performing catchup clear build @ %d\n\n", bValidLen);) pWordSymbInfo->bNumSymbs = bValidLen; (void)__ET9AWDoSelLstBuild(pLingInfo, pbTotalWords, 0, pwGestureValue); __LogFullSelList(pLingInfo, "after catchup clear build", pLogFile2); bLastBuildLen = pWordSymbInfo->bNumSymbs; pLingCmnInfo->Private.bLastBuildLen = bLastBuildLen; } } /* want to build from last build length to current nNumSymbs in case there were multiple inputs in between */ if ((bLastBuildLen + 1) < bNumSymbs) { WLOG2(fprintf(pLogFile2, "performing catchup builds, %d -> %d\n\n", bLastBuildLen + 1, bNumSymbs - 1);) } for (i = bLastBuildLen + 1; i < bNumSymbs; ++i) { ET9BOOL bSuppressBuild; bSuppressBuild = 0; pWordSymbInfo->bNumSymbs = i; if (pWordSymbInfo->SymbsInfo[i - 1].bTraceIndex && pWordSymbInfo->SymbsInfo[i - 1].bTraceIndex == pWordSymbInfo->SymbsInfo[i].bTraceIndex) { #if 0 if (__IsExactSymb(&pWordSymbInfo->SymbsInfo[i])) { WLOG2(fprintf(pLogFile2, "calculating single catchup - has trace info with significant punctuation (%u symbs)\n", i);) } else #endif { WLOG2(fprintf(pLogFile2, "skipping single catchup - has trace info (%u symbs)\n", i);) bSuppressBuild = 1; } } else if (!pLingCmnInfo->Private.bStateIncrementalBuilds && bHasRegionalInfo) { WLOG2(fprintf(pLogFile2, "skipping single catchup - no incremental builds (%u symbs)\n", i);) bSuppressBuild = 1; } else if (pWordSymbInfo->SymbsInfo[i - 1].bAmbigType == ET9EXACT) { WLOG2(fprintf(pLogFile2, "skipping single catchup - is exact (%u symbs)\n", i);) bSuppressBuild = 1; } else if (pWordSymbInfo->SymbsInfo[i - 1].wInputIndex) { WLOG2(fprintf(pLogFile2, "skipping single catchup - has multi info (%u symbs)\n", i);) bSuppressBuild = 1; } else if (pLingCmnInfo->Base.bContentExplicified) { WLOG2(fprintf(pLogFile2, "skipping single catchup - has explicification (%u symbs)\n", i);) bSuppressBuild = 1; } wStatus = __ET9AWDoSelLstBuild(pLingInfo, pbTotalWords, bSuppressBuild, pwGestureValue); __LogFullSelList(pLingInfo, "after catchup build", pLogFile2); pLingCmnInfo->Private.bLastBuildLen = i; if (!bSuppressBuild && (wStatus || !*pbTotalWords)) { WLOG2(fprintf(pLogFile2, "Catchup build gave no result...\n\n");) } } pWordSymbInfo->bNumSymbs = bNumSymbs; pLingCmnInfo->Base.bContentExplicified = 0; wStatus = __ET9AWDoSelLstBuild(pLingInfo, pbTotalWords, 0, pwGestureValue); #if 0 /* should continue to the end */ if (*pwGestureValue) { return ET9STATUS_NONE; } #endif __LogFullSelList(pLingInfo, "after normal build", pLogFile2); pLingCmnInfo->Private.bLastBuildLen = bNumSymbs; /* following code applies whether previous build was successful or not */ /* set whether exact was in list for this build */ pLingCmnInfo->Private.sWordC.pCurrC->dwStateBits &= ~ET9SLEXACTINLIST; if (pWordSymbInfo->bNumSymbs && __ExactInList(pLingInfo)) { pLingCmnInfo->Private.sWordC.pCurrC->dwStateBits |= ET9SLEXACTINLIST; } if (pLingCmnInfo->Private.nDefaultIndex == (ET9UINT)-1) { pLingCmnInfo->Private.nDefaultIndex = ET9_NO_ACTIVE_INDEX; } else { ET9AssertLog(pLingCmnInfo->Private.nDefaultIndex < 0xFF); *pbDefaultListIndex = (ET9U8)pLingCmnInfo->Private.nDefaultIndex; } WLOG2(fprintf(pLogFile2, "nDefaultIndex = %u, nExactIndex %u\n", pLingCmnInfo->Private.nDefaultIndex, pLingCmnInfo->Private.nExactIndex);) if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords && !*pwGestureValue) { wStatus = ET9STATUS_NO_MATCHING_WORDS; } __ProfileEnd(tAW_SelLst_Build); /* before the recursive call */ /* handle input verification against the required word */ if (pWordSymbInfo->Private.bRequiredVerifyInput) { pWordSymbInfo->Private.bRequiredVerifyInput = 0; if (!pLingCmnInfo->Private.bRequiredFound) { ET9SimpleWord sWord = pWordSymbInfo->Private.sRequiredWord; WLOG2(fprintf(pLogFile2, "Required word not found on input verification - modifying input and rebuilding\n");) _ET9ExplicifyWord(pWordSymbInfo, &sWord); pWordSymbInfo->Private.sRequiredWord = sWord; pWordSymbInfo->Private.bRequiredLocate = 1; pWordSymbInfo->Private.bRequiredInhibitOverride = 0; pWordSymbInfo->Private.bRequiredInhibitCorrection = 0; wStatus = ET9AWSelLstBuild(pLingInfo, pbTotalWords, pbDefaultListIndex, pwGestureValue); } } #ifdef ET9_DEBUG WLOG2(fprintf(pLogFile2, "SetActiveLanguage count %u (%u)\n", (pLingInfo->pLingCmnInfo->Private.dwSetActiveLanguageCount - dwSetActiveLanguageCount_start), pLingInfo->pLingCmnInfo->Private.dwSetActiveLanguageCount);) ET9AssertLog(dwSetActiveLanguageCount_start == pLingInfo->pLingCmnInfo->Private.dwSetActiveLanguageCount || (dwSetActiveLanguageCount_start + 1 == pLingInfo->pLingCmnInfo->Private.dwSetActiveLanguageCount && (wLDBInitOK_start != ET9GOODSETUP || dwLDBStart != pLingInfo->pLingCmnInfo->dwFirstLdbNum)) || ET9AW_GetBilingualSupported(pLingInfo)); #endif WLOG2(fprintf(pLogFile2, "\n.......................... selection list build end ..........................\n\n");) return wStatus; } } /*---------------------------------------------------------------------------*/ /** * This function gets the inline word for the current input. * It's suggested to show this string inlined in the input buffer before the user potentially moves the active word in the selection list. * * @param pLingInfo Pointer to alphabetic information structure. * @param pWord Pointer to an adddress of a word. * @param pbCorrection Pointer to a flag that will be non zero if the string by default will be corrected, otherwise zero. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWSelLstGetInlineWord(ET9AWLingInfo * const pLingInfo, ET9SimpleWord * const pWord, ET9BOOL * const pbCorrection) { ET9STATUS wStatus; if ((wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo)) != ET9STATUS_NONE) { return wStatus; } if (!pWord || !pbCorrection) { return ET9STATUS_INVALID_MEMORY; } pWord->wLen = 0; pWord->wCompLen = 0; *pbCorrection = 0; { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9UINT nInlineIndex; if (pLingCmnInfo->Base.bSymbsInfoInvalidated) { return ET9STATUS_NEED_SELLIST_BUILD; } if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return ET9STATUS_NONE; } nInlineIndex = 0; if (pLingCmnInfo->Private.bRequiredFound) { nInlineIndex = pLingCmnInfo->Private.nDefaultIndex; } else { ET9AWWordInfo * const pWordZero = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[0]].Base; if (_ET9_IsNumericString(pWordZero->sWord, pWordZero->wWordLen)) { nInlineIndex = pLingCmnInfo->Private.nDefaultIndex; } } /* transfer word info */ { ET9AWPrivWordInfo * const pWordInline = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nInlineIndex]]; _ET9SymCopy(pWord->sString, pWordInline->Base.sWord, pWordInline->Base.wWordLen); pWord->wLen = pWordInline->Base.wWordLen; pWord->wCompLen = pWordInline->Base.wWordCompLen; } /* correction? */ if (nInlineIndex != pLingCmnInfo->Private.nDefaultIndex) { *pbCorrection = 1; } } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * This function gets an indexed word from the word list. * * @param pLingInfo Pointer to alphabetic information structure. * @param pWord Pointer to an adddress of a word. * @param bWordIndex Word index of base 0. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWSelLstGetWord(ET9AWLingInfo * const pLingInfo, ET9AWWordInfo ** const pWord, const ET9U8 bWordIndex) { ET9STATUS wStatus; if ((wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo)) != ET9STATUS_NONE) { return wStatus; } if (!pWord) { return ET9STATUS_INVALID_MEMORY; } *pWord = NULL; { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (pLingCmnInfo->Base.bSymbsInfoInvalidated) { return ET9STATUS_NEED_SELLIST_BUILD; } if (bWordIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return ET9STATUS_OUT_OF_RANGE; } *pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[bWordIndex]].Base; } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9STATUS ET9FARCALL _ET9AWSelLstStripActualTaps(ET9AWPrivWordInfo * const pWord) { ET9UINT nWordLen; ET9UINT nIncomingWordLen; ET9SYMB sSymbol; ET9UINT nNumDigt = 0; ET9UINT nNumPunct = 0; ET9UINT i; ET9UINT nLeadStrips = 0; ET9AssertLog(pWord != NULL); nWordLen = pWord->Base.wWordLen; nIncomingWordLen = nWordLen; while (nWordLen--) { sSymbol = pWord->Base.sWord[nWordLen]; /* Count punctuation and digits */ if (_ET9_IsNumeric(sSymbol)) { ++nNumDigt; } else if (_ET9_IsPunctChar(sSymbol)) { ++nNumPunct; } } nWordLen = pWord->Base.wWordLen; if (nNumPunct && ((nNumDigt + nNumPunct < nWordLen - nNumDigt - nNumPunct) || (nNumDigt + nNumPunct > EMOTICON_LENGTH_THRESHOLD))) { while (_ET9_IsPunctChar(pWord->Base.sWord[nLeadStrips]) && nWordLen--) { ++nLeadStrips; } for (i = nLeadStrips; i > 0 && i < pWord->Base.wWordLen; ++i) { pWord->Base.sWord[i - nLeadStrips] = pWord->Base.sWord[i]; } pWord->Base.wWordLen = (ET9U16) (pWord->Base.wWordLen - nLeadStrips); nWordLen = pWord->Base.wWordLen; while (nWordLen && _ET9_IsPunctChar(pWord->Base.sWord[nWordLen - 1])) { if (pWord->Base.wWordLen) { --pWord->Base.wWordLen; } if (pWord->Base.wWordCompLen) { --pWord->Base.wWordCompLen; } --nWordLen; } } if (pWord->Base.wWordLen == nIncomingWordLen) { return ET9STATUS_NO_OPERATION; } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWDoWordProcessing(ET9AWLingInfo * const pLingInfo, ET9SYMB * const pText, const ET9U16 wSize, const ET9U8 bWordSrc, const ET9U8 bLangIndex, const _ET9_Processing eProcessType, const ET9U32 dwWordIndex, const ET9BOOL bInhibitNewWord) { ET9STATUS wStatus; ET9U32 dwActiveLang; ET9AssertLog(pLingInfo); ET9AssertLog(pText); ET9AssertLog(eProcessType == _ET9_Processing_use || eProcessType == _ET9_Processing_space || eProcessType == _ET9_Processing_both); if (pLingInfo->Private.wInfoInitOK != ET9GOODSETUP) { return ET9STATUS_NO_INIT; } if (!wSize) { return ET9STATUS_NONE; } if (wSize > ET9MAXWORDSIZE) { return ET9STATUS_OUT_OF_RANGE; } WLOG2(fprintf(pLogFile2, "__ET9AWDoWordProcessing, bWordSrc = %u, bLangIndex = %u, eProcessType = %u, dwWordIndex = %u, bInhibitNewWord %u\n", bWordSrc, bLangIndex, eProcessType, dwWordIndex, bInhibitNewWord);) WLOG2String(pLogFile2, "__ET9AWDoWordProcessing", pText, wSize, 0, 0); ET9AWLdbGetActiveLanguage(pLingInfo, &dwActiveLang); /* _ET9AWSuppDBAddSelection only needs the length and character data */ { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9BOOL bDLMActive = (pLingCmnInfo->pDLMInfo && ET9DLMENABLED(pLingCmnInfo)) ? 1 : 0; ET9STATUS wStatusDLM; { const ET9BOOL bQuarantine = (pLingCmnInfo->Private.bQuarantineControlUserAction && !_ET9IsInhibitDelayedReorderAfterTrace(pLingCmnInfo->Base.pWordSymbInfo)) ? 1 : 0; const ET9BOOL bUpdateUse = (eProcessType == _ET9_Processing_use || eProcessType == _ET9_Processing_both) ? 1 : 0; const _ET9AW_DLM_WordQuality eQuality = (dwWordIndex) ? _ET9AW_DLM_WordQuality_UserNormal : (bQuarantine) ? _ET9AW_DLM_WordQuality_UserQuarantine : _ET9AW_DLM_WordQuality_UserNormal; wStatusDLM = _ET9AW_DLM_AddWord(pLingInfo, pText, wSize, bUpdateUse, eQuality, bInhibitNewWord); } /* this call updates the context, keep after everything that relys on the previous context being there */ /* it's an old rule to not use process length 1 words for RUDB/CDB, but that's handled inside this call */ wStatus = _ET9AWSuppDBAddSelection(pLingInfo, pText, wSize, bWordSrc, bLangIndex, eProcessType, bInhibitNewWord); if (bDLMActive) { wStatus = wStatusDLM; } } /* no DB? */ if (!pLingInfo->pLingCmnInfo->pRUDBInfo && !pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DB; } return wStatus; } #ifdef ET9_QA_ACCESS /*---------------------------------------------------------------------------*/ /* -IDR- * */ ET9STATUS ET9FARCALL _ET9_QA_AWDoWordProcessing(ET9AWLingInfo * const pLingInfo, ET9SYMB * const pText, const ET9U16 wSize, const ET9U8 bLangIndex, const ET9UINT nProcessType) { return __ET9AWDoWordProcessing(pLingInfo, pText, wSize, ET9WORDSRC_NONE, bLangIndex, (_ET9_Processing)nProcessType, 0, 0); } #endif /*---------------------------------------------------------------------------*/ /** * Select word. * This function allows the host to specify which entry has been selected. * This call will update usage. * * @param pLingInfo Pointer to alphabetic information structure. * @param bWordIndex Word index of base 0. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWSelLstSelWord(ET9AWLingInfo * const pLingInfo, const ET9U8 bWordIndex) { ET9STATUS wStatus; ET9BOOL bInhibitNewWord; ET9BOOL bProcessedWholeWord; ET9U32 dwActiveLang; ET9AWPrivWordInfo *pAmbigWord; ET9AWLingCmnInfo *pLingCmnInfo; if ((wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo)) != ET9STATUS_NONE) { return wStatus; } pLingCmnInfo = pLingInfo->pLingCmnInfo; if (pLingCmnInfo->Base.bSymbsInfoInvalidated) { return ET9STATUS_NEED_SELLIST_BUILD; } if (bWordIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return ET9STATUS_OUT_OF_RANGE; } ET9AWLdbGetActiveLanguage(pLingInfo, &dwActiveLang); pAmbigWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[bWordIndex]]; WLOG2Word(pLogFile2, "ET9AWSelLstSelWord, word", pAmbigWord); _ET9SaveWord(pLingCmnInfo->Base.pWordSymbInfo, pAmbigWord->Base.sWord, pAmbigWord->Base.wWordLen); bInhibitNewWord = (pLingCmnInfo->Base.pWordSymbInfo->Private.bHasShiftGesture || pLingCmnInfo->Base.pWordSymbInfo->Private.bHasAllCapsGesture); _ET9ProcessSelListQUDBEntries(pLingInfo, bWordIndex); #ifdef ET9_SPM_MODULE if (pLingCmnInfo->Private.__SPATH_Status.bHasSpmScores) { _ET9AWSP_SelectWord(pLingInfo, bWordIndex); } #endif _ET9AW_DLM_ProcessSelListEntries(pLingInfo, bWordIndex); /* NWP and non buildable exact gets use processing on entire word only, otherwise do pieces as well */ bProcessedWholeWord = 0; if (pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs && !(bWordIndex == pLingCmnInfo->Private.nExactIndex && !BUILDABLEEXACT(pLingCmnInfo->Private.sWordC.pCurrC->dwStateBits))) { /* do use processing on pieces */ /* this code needs to be redesigned, flush points are not a very good instrument to split words when using spc */ /* locks also makes things more complex and "unexpected" in combination with spc and resulting length diffs */ ET9INT snLenUsed; const ET9INT snWordLen = pAmbigWord->Base.wWordLen; ET9INT snFlushLen; ET9U16 wSymbolIndex; ET9STATUS wTmpStatus; /* loop over all possible flush points */ snLenUsed = 0; for (wSymbolIndex = 1; wSymbolIndex <= pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs; ++wSymbolIndex) { if (wSymbolIndex == pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { snFlushLen = snWordLen; } else { snFlushLen = __CaptureGetFlushStringLengthAtPoint(pLingInfo, wSymbolIndex); } if (snFlushLen && snFlushLen < snLenUsed) { WLOG2(fprintf(pLogFile2, "ET9AWSelLstSelWord, snFlushLen = %d, snLenUsed = %d\n", snFlushLen, snLenUsed);) } ET9AssertLog(!snFlushLen || snFlushLen >= snLenUsed); if (snFlushLen > snLenUsed) { ET9INT snOffset; const ET9INT snPartLen = __ET9Min((snFlushLen - snLenUsed), (snWordLen - snLenUsed)); WLOG2(fprintf(pLogFile2, "ET9AWSelLstSelWord, snPartLen = %d, snFlushLen = %d, snWordLen = %d, snLenUsed = %d\n", snPartLen, snFlushLen, snWordLen, snLenUsed);) ET9AssertLog(snPartLen >= 0); for (snOffset = 0; snOffset < snPartLen; ++snOffset) { const ET9U16 wStringLen = (ET9U16)(snPartLen - snOffset); ET9SYMB * const psString = &pAmbigWord->Base.sWord[snLenUsed + snOffset]; ET9BOOL bDoProcessing; ET9U8 bExact; ET9U8 bLowercase; ET9AWPrivWordInfo sWord; _InitPrivWordInfo(&sWord); sWord.Base.wWordLen = wStringLen; _ET9SymCopy(&sWord.Base.sWord, psString, wStringLen); /* since it's very hard to split the selected word up into proper "sub words" (how they were created), limit the selection to already known words rather than inventing too many new ones at this point */ if (wStringLen == pAmbigWord->Base.wWordLen) { bDoProcessing = 1; bProcessedWholeWord = 1; } else if (_ET9AWLdbFind(pLingInfo, pLingCmnInfo->dwFirstLdbNum, ET9MGD_FULL_WORD, &sWord, &bExact, &bLowercase, 0) == ET9STATUS_WORD_EXISTS) { bDoProcessing = 1; } else if (ET9AW_GetBilingualSupported(pLingInfo) && _ET9AWLdbFind(pLingInfo, pLingCmnInfo->dwSecondLdbNum, ET9MGD_FULL_WORD, &sWord, &bExact, &bLowercase, 0) == ET9STATUS_WORD_EXISTS) { bDoProcessing = 1; } #if 0 else if (_ET9AWMdbFind(pLingInfo, psString, wStringLen, 0) == ET9STATUS_WORD_EXISTS) { bDoProcessing = 1; } #endif else if (_ET9AWFindRUDBObject(pLingInfo, psString, wStringLen, ET9AWBOTH_LANGUAGES, 0, 0, 0)) { bDoProcessing = 1; } else if (_ET9AW_DLM_FindWord(pLingInfo, psString, wStringLen, 1, 0)) { bDoProcessing = 1; } else { bDoProcessing = 0; } if (bDoProcessing) { WLOG2String(pLogFile2, "ET9AWSelLstSelWord, process part", psString, wStringLen, 0, 0); wTmpStatus = __ET9AWDoWordProcessing(pLingInfo, psString, wStringLen, pAmbigWord->Body.bWordSrc, pAmbigWord->Base.bLangIndex, _ET9_Processing_use, pAmbigWord->Body.dwWordIndex, bInhibitNewWord); if (wTmpStatus && wTmpStatus != ET9STATUS_INVALID_TEXT) { wStatus = wTmpStatus; } } if (!_ET9_IsPunctChar(*psString)) { break; } } snLenUsed += snPartLen; } } } /* process the whole word if "pieces" above didn't cover it */ if (!bProcessedWholeWord) { WLOG2String(pLogFile2, "ET9AWSelLstSelWord, process all", pAmbigWord->Base.sWord, pAmbigWord->Base.wWordLen, 0, 0); wStatus = __ET9AWDoWordProcessing(pLingInfo, pAmbigWord->Base.sWord, pAmbigWord->Base.wWordLen, pAmbigWord->Body.bWordSrc, pAmbigWord->Base.bLangIndex, _ET9_Processing_use, pAmbigWord->Body.dwWordIndex, bInhibitNewWord); } /* don't propagate ET9STATUS_INVALID_TEXT error, thats our problem, if ET9STATUS_INVALID_TEXT occurs, the function is essentially a no-op. */ if (wStatus == ET9STATUS_INVALID_TEXT) { wStatus = ET9STATUS_NONE; } WLOG2(fprintf(pLogFile2, "ET9AWSelLstSelWord, old, bSwitchLanguage = %u, dwPreviousWordLanguage = %08X, dwPrevPrevWordLanguage = %08X\n", pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage, pLingCmnInfo->Private.dwPrevWordLanguage, pLingCmnInfo->Private.dwPrevPrevWordLanguage);) ET9Assert(ET9ACTIVELANGSWITCHENABLED(pLingCmnInfo) || !pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage); /* update previous word language */ if (pAmbigWord->Base.bLangIndex == ET9AWFIRST_LANGUAGE || pAmbigWord->Base.bLangIndex == ET9AWSECOND_LANGUAGE) { pLingCmnInfo->Private.dwPrevPrevWordLanguage = pLingCmnInfo->Private.dwPrevWordLanguage; } if (pAmbigWord->Base.bLangIndex == ET9AWFIRST_LANGUAGE) { pLingCmnInfo->Private.dwPrevWordLanguage = pLingCmnInfo->dwFirstLdbNum; } else if (pAmbigWord->Base.bLangIndex == ET9AWSECOND_LANGUAGE) { pLingCmnInfo->Private.dwPrevWordLanguage = pLingCmnInfo->dwSecondLdbNum; } else if (!pLingCmnInfo->Private.dwPrevWordLanguage) { pLingCmnInfo->Private.dwPrevWordLanguage = pLingCmnInfo->dwFirstLdbNum; } /* switch language? */ if (!ET9ACTIVELANGSWITCHENABLED(pLingCmnInfo) || !ET9AW_GetBilingualSupported(pLingInfo)) { pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage = 0; } else { if (pLingCmnInfo->Private.dwPrevPrevWordLanguage == pLingCmnInfo->Private.dwPrevWordLanguage) { if (pLingCmnInfo->Private.dwPrevWordLanguage == pLingCmnInfo->dwFirstLdbNum) { pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage = 0; } else { pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage = 1; } } } WLOG2(fprintf(pLogFile2, "ET9AWSelLstSelWord, new, bSwitchLanguage = %u, dwPreviousWordLanguage = %08X, dwPrevPrevWordLanguage = %08X\n", pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage, pLingCmnInfo->Private.dwPrevWordLanguage, pLingCmnInfo->Private.dwPrevPrevWordLanguage);) return wStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __ET9AWCheckForShortcutAssociation(ET9AWLingInfo * const pLingInfo, ET9SYMB * const pText, const ET9U16 aSize) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9UINT i; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pText != NULL); ET9AssertLog(aSize > 0); /* only handle if selection list still viable */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || pLingCmnInfo->Base.bSelListInvalidated) { return; } for (i = 0; i < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++i) { ET9AWPrivWordInfo const * const pEntry = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[i]]; /* if this matches the size of this entry's substitution length */ if (pEntry->Base.wSubstitutionLen == aSize) { const ET9U32 dwLdbNum = (pEntry->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; ET9SYMB const *psSub; ET9SYMB *psTarget; ET9UINT j; psSub = pEntry->Base.sSubstitution; psTarget = pText; for (j = 0; j < pEntry->Base.wSubstitutionLen; ++j) { if (_ET9SymToLower(*psSub++, dwLdbNum) != _ET9SymToLower(*psTarget++, dwLdbNum)) { break; } } /* if there's a match, add the shortcut to the CDB */ if (j == pEntry->Base.wSubstitutionLen) { _ET9AWCDBAddShortcut(pLingInfo, pEntry); break; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * */ #define _ET9_MAX_PHRASE_WORDS 1 /*---------------------------------------------------------------------------*/ /** * Note word done.
* This function allows the system to learn new words. * This call will pick up the correct context from the buffer and potentially learn one or more words. * * Sending in a single word (no white space etc) and the cursor pos last will do the same as * ET9AWNoteWordDone used to do. It will just learn the word and push it down to extend the known context. * * This function will NOT update usage, ET9AWSelLstSelWord does that. * * @param pLingInfo Pointer to alphabetic information structure. * @param psBuf Pointer to editor buffer (or part of it). * @param dwBufLen Length of the supplied buffer. * @param dwCursorPos Cursor position within the buffer. Zero means before the first character, one means after the first character and so on. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWNoteWordDone(ET9AWLingInfo * const pLingInfo, ET9SYMB * const psBuf, const ET9U32 dwBufLen, const ET9U32 dwCursorPos) { ET9STATUS eStatus; eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!psBuf) { return ET9STATUS_INVALID_MEMORY; } if (!dwBufLen) { return ET9STATUS_OUT_OF_RANGE; } if (dwCursorPos > dwBufLen) { return ET9STATUS_OUT_OF_RANGE; } if (!dwCursorPos) { return ET9STATUS_NONE; } #ifdef EVAL_BUILD _ET9Eval_UpdateUsage(&pLingInfo->pLingCmnInfo->Base); #endif WLOG2(;); WLOG2String(pLogFile2, "ET9AWNoteWordDone", psBuf, (ET9INT)dwBufLen, 0, 0); { ET9BOOL bInWord; ET9S32 sdwIndex; ET9U32 dwEndPos; ET9UINT nWordCount; ET9U32 dwWordStart[_ET9_MAX_PHRASE_WORDS]; ET9U32 dwWordEnd[_ET9_MAX_PHRASE_WORDS]; /* validate cursor pos */ { dwEndPos = dwCursorPos - 1; if (_ET9_GetSymbolClass(psBuf[dwEndPos]) == ET9_WhiteSymbClass) { } else if (dwEndPos + 1 < dwBufLen) { if (_ET9_GetSymbolClass(psBuf[dwEndPos + 1]) == ET9_WhiteSymbClass) { ++dwEndPos; } else { return ET9STATUS_BAD_PARAM; } } } /* identify words before the cursor */ bInWord = 0; nWordCount = 0; for (sdwIndex = dwEndPos; sdwIndex >= 0; --sdwIndex) { const ET9SymbClass eClass = _ET9_GetSymbolClass(psBuf[sdwIndex]); if (eClass == ET9_WhiteSymbClass || eClass == ET9_UnassSymbClass) { if (bInWord) { bInWord = 0; ++nWordCount; if (nWordCount >= _ET9_MAX_PHRASE_WORDS) { break; } } } else { if (!bInWord) { bInWord = 1; dwWordEnd[nWordCount] = sdwIndex; } dwWordStart[nWordCount] = sdwIndex; if (!sdwIndex) { ++nWordCount; break; } } } /* process found words */ if (nWordCount == 1 && dwWordStart[0] == 0 && dwWordEnd[0] == (dwBufLen - 1) && dwCursorPos == dwBufLen) { ET9SYMB * const psWord = psBuf; const ET9U16 wWordLen = (ET9U16)dwBufLen; ET9BOOL bHasShiftOrCapsGesture; const _ET9_Processing eProcessType = _ET9_IsLastSavedWord(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo, psWord, wWordLen, &bHasShiftOrCapsGesture) ? _ET9_Processing_space : _ET9_Processing_both; /* the whole buffer is a single word, special case */ WLOG2(fprintf(pLogFile2, "ET9AWNoteWordDone, processed as a single word\n");) __ET9AWCheckForShortcutAssociation(pLingInfo, psWord, wWordLen); eStatus = __ET9AWDoWordProcessing(pLingInfo, psWord, wWordLen, ET9WORDSRC_NONE, ET9AWUNKNOWN_LANGUAGE, eProcessType, 0, bHasShiftOrCapsGesture); } else { ET9STATUS eTmpStatus; ET9INT snIndex; for (snIndex = nWordCount - 1; snIndex >= 0; --snIndex) { ET9SYMB * const psWord = &psBuf[dwWordStart[snIndex]]; const ET9U16 wWordLen = (ET9U16)(dwWordEnd[snIndex] - dwWordStart[snIndex] + 1); ET9BOOL bHasShiftOrCapsGesture; const _ET9_Processing eProcessType = _ET9_IsLastSavedWord(pLingInfo->pLingCmnInfo->Base.pWordSymbInfo, psWord, wWordLen, &bHasShiftOrCapsGesture) ? _ET9_Processing_space : _ET9_Processing_both; WLOG2(fprintf(pLogFile2, "ET9AWNoteWordDone, processing word %d\n", -snIndex);) WLOG2String(pLogFile2, "context", psBuf, (ET9INT)dwWordStart[snIndex], 0, 0); eTmpStatus = ET9AWFillContextBuffer(pLingInfo, psBuf, dwWordStart[snIndex]); if (eTmpStatus) { eStatus = eTmpStatus; continue; } __ET9AWCheckForShortcutAssociation(pLingInfo, psWord, (ET9U16)(dwWordEnd[0] - dwWordStart[snIndex] + 1)); WLOG2String(pLogFile2, "word", psWord, wWordLen, 0, 0); eTmpStatus = __ET9AWDoWordProcessing(pLingInfo, psWord, wWordLen, ET9WORDSRC_NONE, ET9AWUNKNOWN_LANGUAGE, eProcessType, 0, bHasShiftOrCapsGesture); if (eTmpStatus) { eStatus = eTmpStatus; } } WLOG2String(pLogFile2, "final context", psBuf, (ET9INT)(dwEndPos + 1), 0, 0); eTmpStatus = ET9AWFillContextBuffer(pLingInfo, psBuf, dwEndPos + 1); if (eTmpStatus) { eStatus = eTmpStatus; } } } return eStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static void ET9LOCALCALL __ET9AWAddMagicStrToSel(ET9AWLingInfo * const pLingInfo, ET9SYMB * const pSrcString, const ET9U16 wStrSize, ET9U16 * const pwFreqIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo sLocalWord; ET9U16 wTmpLen; ET9U16 wSrcLen; ET9SYMB *pSrc; ET9SYMB *pDest; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pSrcString != NULL); if (!wStrSize) { return; } wTmpLen = pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord.Base.wWordLen; pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord.Base.wWordLen = 0; _InitPrivWordInfo(&sLocalWord); sLocalWord.Body.bWordSrc = ET9WORDSRC_MAGICSTRING; sLocalWord.Body.bLangIndexScoring = ET9AW_GetBilingualSupported(pLingInfo) ? ET9AWBOTH_LANGUAGES : ET9AWFIRST_LANGUAGE; pSrc = pSrcString; pDest = sLocalWord.Base.sWord; sLocalWord.Base.wWordLen = 0; for (wSrcLen = wStrSize; wSrcLen; --wSrcLen) { *pDest++ = *pSrc++; ++sLocalWord.Base.wWordLen; if (sLocalWord.Base.wWordLen == ET9MAXWORDSIZE) { sLocalWord.Body.xWordFreq = (ET9FREQPART)(1000 - (*pwFreqIndex)++); sLocalWord.Base.bLangIndex = 0xcc; _ET9AWSelLstAdd(pLingInfo, &sLocalWord, sLocalWord.Base.wWordLen, FREQ_NORMAL); sLocalWord.Base.wWordLen = 0; pDest = sLocalWord.Base.sWord; } } if (sLocalWord.Base.wWordLen) { /* make sure the word isn't shorter than the input */ while (sLocalWord.Base.wWordLen < ET9MAXLDBWORDSIZE) { *pDest++ = ' '; ++sLocalWord.Base.wWordLen; } sLocalWord.Body.xWordFreq = (ET9FREQPART)(1000 - (*pwFreqIndex)++); sLocalWord.Base.bLangIndex = 0xcc; _ET9AWSelLstAdd(pLingInfo, &sLocalWord, sLocalWord.Base.wWordLen, FREQ_NORMAL); } ET9AssertLog(!sLocalWord.Body.wEWordFreq && !sLocalWord.Body.wTWordFreq && !sLocalWord.Body.dwWordIndex); pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord.Base.wWordLen = wTmpLen; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ #define __FlushMagicString(wLength) \ if (psC - psStr + wLength > ET9MAXWORDSIZE) { \ __ET9AWAddMagicStrToSel(pLingInfo, psStr, (ET9U16)(psC - psStr), &wFreqIndex); \ psC = psStr; \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __ET9AWAddMagicStr(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U16 wFreqIndex = 0; ET9AssertLog(pLingInfo != NULL); /* disabling magic string in initial JXT9 release */ if ((pLingCmnInfo->dwLdbNum & ET9PLIDMASK) == ET9PLIDJapanese) { /* this is the correct LDB to use */ return; } /* add version info */ { ET9U16 wStrLen; ET9SYMB psStr[ET9MAXVERSIONSTR]; if (!ET9GetCodeVersion(psStr, ET9MAXVERSIONSTR, &wStrLen)) { __ET9AWAddMagicStrToSel(pLingInfo, psStr, wStrLen, &wFreqIndex); } if (!ET9AWLdbGetVersion(pLingInfo, psStr, ET9MAXVERSIONSTR, &wStrLen)) { __ET9AWAddMagicStrToSel(pLingInfo, psStr, wStrLen, &wFreqIndex); } __ET9AWAddMagicStrToSel(pLingInfo, pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->Private.szIDBVersion, pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->Private.wIDBVersionStrSize, &wFreqIndex); } /* add settings, debug etc info */ { ET9SYMB *psC; ET9SYMB psStr[ET9MAXWORDSIZE]; psC = psStr; *psC++ = 'L'; *psC++ = 'i'; *psC++ = 's'; *psC++ = 't'; *psC++ = ':'; *psC++ = (pLingCmnInfo->Private.eSelectionListMode == ET9ASLMODE_AUTO) ? 'A' : (pLingCmnInfo->Private.eSelectionListMode == ET9ASLMODE_CLASSIC) ? 'C' : (pLingCmnInfo->Private.eSelectionListMode == ET9ASLMODE_COMPLETIONSPROMOTED) ? 'P' : (pLingCmnInfo->Private.eSelectionListMode == ET9ASLMODE_MIXED) ? 'M' : '?'; *psC++ = (pLingCmnInfo->Private.eSelectionListCorrectionMode == ET9ASLCORRECTIONMODE_LOW) ? 'L' : (pLingCmnInfo->Private.eSelectionListCorrectionMode == ET9ASLCORRECTIONMODE_MEDIUM) ? 'M' : (pLingCmnInfo->Private.eSelectionListCorrectionMode == ET9ASLCORRECTIONMODE_HIGH) ? 'H' : '?'; *psC++ = '.'; *psC++ = (ET9SYMB)(pLingCmnInfo->Private.nListSize / 10 + '0'); *psC++ = (ET9SYMB)(pLingCmnInfo->Private.nListSize % 10 + '0'); *psC++ = ' '; __FlushMagicString(12); *psC++ = 'B'; *psC++ = 'i'; *psC++ = 'l'; *psC++ = 'i'; *psC++ = 'n'; *psC++ = 'g'; *psC++ = 'u'; *psC++ = 'a'; *psC++ = 'l'; *psC++ = ':'; *psC++ = ET9AW_GetBilingualSupported(pLingInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(6); *psC++ = 'N'; *psC++ = 'W'; *psC++ = 'P'; *psC++ = ':'; *psC++ = ET9NEXTWORDPREDICTION_MODE(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(9); *psC++ = 'E'; *psC++ = 'x'; *psC++ = 'a'; *psC++ = 'c'; *psC++ = 't'; *psC++ = ':'; *psC++ = ET9EXACTINLIST(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ET9EXACTLAST(pLingCmnInfo) ? 'L' : 'F'; *psC++ = ' '; __FlushMagicString(10); *psC++ = 'A'; *psC++ = 'A'; *psC++ = 'p'; *psC++ = 'p'; *psC++ = 'e'; *psC++ = 'n'; *psC++ = 'd'; *psC++ = ':'; *psC++ = ET9AUTOAPPENDINLIST(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(10); *psC++ = 'S'; *psC++ = 't'; *psC++ = 'e'; *psC++ = 'm'; *psC++ = 's'; *psC++ = ':'; *psC++ = ET9WORDSTEMS_MODE(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = (ET9SYMB)(pLingCmnInfo->Private.wWordStemsPoint / 10 + '0'); *psC++ = (ET9SYMB)(pLingCmnInfo->Private.wWordStemsPoint % 10 + '0'); *psC++ = ' '; __FlushMagicString(13); *psC++ = 'C'; *psC++ = 'o'; *psC++ = 'm'; *psC++ = 'p'; *psC++ = 's'; *psC++ = ':'; *psC++ = ET9WORDCOMPLETION_MODE(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = (ET9SYMB)(pLingCmnInfo->Private.wWordCompletionPoint / 10 + '0'); *psC++ = (ET9SYMB)(pLingCmnInfo->Private.wWordCompletionPoint % 10 + '0'); *psC++ = '.'; *psC++ = (ET9SYMB)(pLingCmnInfo->Private.nMaxCompletionCount / 10 + '0'); *psC++ = (ET9SYMB)(pLingCmnInfo->Private.nMaxCompletionCount % 10 + '0'); *psC++ = ' '; __FlushMagicString(8); *psC++ = 'N'; *psC++ = 'L'; *psC++ = 'o'; *psC++ = 'c'; *psC++ = 'k'; *psC++ = ':'; *psC++ = ET9NEXTLOCKING_MODE(pLingCmnInfo->Base.pWordSymbInfo->dwStateBits) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(8); *psC++ = 'P'; *psC++ = 'S'; *psC++ = 'o'; *psC++ = 'r'; *psC++ = 't'; *psC++ = ':'; *psC++ = ET9POSTSORTENABLED(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(7); *psC++ = 'Q'; *psC++ = 'U'; *psC++ = 'D'; *psC++ = 'B'; *psC++ = ':'; *psC++ = (ET9SYMB)(pLingCmnInfo->Private.bQuarantineControlUserAction + '0'); *psC++ = (ET9SYMB)(pLingCmnInfo->Private.bQuarantineControlScanAction + '0'); *psC++ = ' '; __FlushMagicString(11); *psC++ = 'S'; *psC++ = 'P'; *psC++ = 'C'; *psC++ = ':'; *psC++ = pLingCmnInfo->Private.ASpc.eMode == ET9ASPCMODE_OFF ? 'N' : pLingCmnInfo->Private.ASpc.eMode == ET9ASPCMODE_EXACT ? 'E' : 'R'; *psC++ = ET9_SPC_SEARCH_FILTER_IS_UNFILTERED(pLingCmnInfo->Private.ASpc.eSearchFilter) ? '-' : ET9_SPC_SEARCH_FILTER_IS_ONE(pLingCmnInfo->Private.ASpc.eSearchFilter) ? '1' : '2'; *psC++ = ET9_SPC_SEARCH_FILTER_IS_UNFILTERED(pLingCmnInfo->Private.ASpc.eSearchFilter) ? 'U' : ET9_SPC_SEARCH_FILTER_IS_EXACT(pLingCmnInfo->Private.ASpc.eSearchFilter) ? 'X' : 'R'; *psC++ = '.'; *psC++ = (ET9SYMB)(pLingCmnInfo->Private.ASpc.nMaxSpcTermCount / 10 + '0'); *psC++ = (ET9SYMB)(pLingCmnInfo->Private.ASpc.nMaxSpcTermCount % 10 + '0'); *psC++ = ' '; __FlushMagicString(6); *psC++ = 'U'; *psC++ = 'A'; *psC++ = 'S'; *psC++ = ':'; *psC++ = ET9USERDEFINEDAUTOSUBENABLED(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(6); *psC++ = 'L'; *psC++ = 'A'; *psC++ = 'S'; *psC++ = ':'; *psC++ = ET9LDBSUPPORTEDAUTOSUBENABLED(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(7); *psC++ = 'O'; *psC++ = 'T'; *psC++ = 'F'; *psC++ = 'M'; *psC++ = ':'; *psC++ = pLingInfo->Private.pConvertSymb != NULL ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(6); *psC++ = 'A'; *psC++ = 'L'; *psC++ = 'S'; *psC++ = ':'; *psC++ = ET9ACTIVELANGSWITCHENABLED(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(9); *psC++ = 'D'; *psC++ = 'B'; *psC++ = ':'; *psC++ = ET9LDBENABLED(pLingCmnInfo) ? 'L' : 'l'; *psC++ = ET9DLMENABLED(pLingCmnInfo) ? 'D' : 'd'; *psC++ = ET9CDBENABLED(pLingCmnInfo) ? 'C' : 'c'; *psC++ = ET9RUDBENABLED(pLingCmnInfo) ? 'R' : 'r'; *psC++ = ET9ASDBENABLED(pLingCmnInfo) ? 'A' : 'a'; *psC++ = ET9MDBENABLED(pLingCmnInfo) ? 'M' : 'm'; *psC++ = ' '; __FlushMagicString(5); *psC++ = 'C'; *psC++ = 'B'; *psC++ = 'P'; *psC++ = ':'; *psC++ = ET9CONTEXTBASEDPREDICTION(pLingCmnInfo) ? 'Y' : 'N'; *psC++ = ' '; __FlushMagicString(6); /* debug info */ *psC++ = 'D'; *psC++ = 'B'; *psC++ = 'G'; *psC++ = ':'; #if defined(_DEBUG) *psC++ = 'Y'; #elif defined(NDEBUG) *psC++ = 'N'; #else *psC++ = '?'; #endif *psC++ = ' '; __FlushMagicString(6); *psC++ = 'X'; *psC++ = 'D'; *psC++ = 'B'; *psC++ = 'G'; *psC++ = ':'; #ifdef ET9_DEBUG *psC++ = 'Y'; #else *psC++ = 'N'; #endif *psC++ = ' '; __FlushMagicString(7); /* std lib for memory operations etc info */ *psC++ = 'S'; *psC++ = 'T'; *psC++ = 'D'; *psC++ = 'L'; *psC++ = 'I'; *psC++ = 'B'; *psC++ = ':'; #ifdef ET9ACTIVATEMISCSTDCLIBUSE *psC++ = 'Y'; #else *psC++ = 'N'; #endif *psC++ = ' '; __FlushMagicString(9); /* direct LDB access info */ *psC++ = 'D'; *psC++ = 'L'; *psC++ = 'D'; *psC++ = 'B'; *psC++ = ':'; #ifdef ET9_DIRECT_LDB_ACCESS *psC++ = 'Y'; #else *psC++ = 'N'; #endif *psC++ = ' '; __FlushMagicString(7); /* Trace feature */ *psC++ = 'T'; *psC++ = 'R'; *psC++ = 'A'; *psC++ = 'C'; *psC++ = 'E'; *psC++ = ':'; #ifdef ET9_KDB_TRACE_MODULE { char const *pcChar; for (pcChar = ET9_TRACE_VER; *pcChar; ++pcChar) { *psC++ = (ET9SYMB)*pcChar; } } *psC++ = !pLingCmnInfo->Private.bStateSpmScoring ? ' ' : pLingCmnInfo->Private.__SPATH_Status.bKDBConfigOk ? '+' : '-'; #else *psC++ = 'N'; #endif *psC++ = ' '; __FlushMagicString(ET9MAXWORDSIZE); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static void ET9LOCALCALL __InitCaptureInfoLonger(ET9AWLingInfo * const pLingInfo, const ET9U16 wFromIndex, const ET9U16 wToIndex) { ET9U16 wIndex; ET9U8 *pbFlushPos; ET9U8 *pbFlushLen; ET9U16 *pwDefaultPos; ET9U8 *pbDefaultLen; ET9U8 *pbDefaultCompLen; ET9AWCaptureAction *pCaptureAction; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(wFromIndex <= wToIndex); ET9AssertLog(wToIndex < ET9MAXWORDSIZE); pbFlushPos = &pBuildInfo->pbFlushPos[wFromIndex]; pbFlushLen = &pBuildInfo->pbFlushLen[wFromIndex]; pwDefaultPos = &pBuildInfo->pwDefaultPos[wFromIndex]; pbDefaultLen = &pBuildInfo->pbDefaultLen[wFromIndex]; pbDefaultCompLen = &pBuildInfo->pbDefaultCompLen[wFromIndex]; pCaptureAction = &pBuildInfo->pCaptureActions[wFromIndex]; for (wIndex = wFromIndex; wIndex <= wToIndex; ++wIndex, ++pCaptureAction, ++pbFlushPos, ++pbFlushLen, ++pwDefaultPos, ++pbDefaultLen, ++pbDefaultCompLen) { pCaptureAction->bPop = 0; pCaptureAction->sbAddWordLen = 0; pCaptureAction->sbAddWordCompLen = 0; pCaptureAction->bAddSymbolLen = 0; *pbFlushPos = 0; *pbFlushLen = 0; *pwDefaultPos = UNDEFINED_STRING_INDEX; *pbDefaultLen = 0; *pbDefaultCompLen = 0; } /* remove old flush positions (from longer builds) */ pbFlushPos = &pBuildInfo->pbFlushPos[0]; for (wIndex = 0; wIndex <= wToIndex; ++wIndex, ++pbFlushPos) { if (*pbFlushPos > wFromIndex) { *pbFlushPos = 0; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __InitCaptureInfoShorter(ET9AWLingInfo * const pLingInfo, const ET9U16 wFromIndex, const ET9U16 wToIndex) { ET9U16 wIndex; ET9U8 *pbFlushPos; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(wFromIndex <= wToIndex); ET9AssertLog(wToIndex < ET9MAXWORDSIZE); /* remove old flush positions (from longer or equal builds) */ pbFlushPos = &pBuildInfo->pbFlushPos[0]; for (wIndex = 0; wIndex <= wToIndex; ++wIndex, ++pbFlushPos) { if (*pbFlushPos > wFromIndex) { *pbFlushPos = 0; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __ResetAllCaptureInfo(ET9AWLingInfo * const pLingInfo) { ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); WLOG2(fprintf(pLogFile2, "__ResetAllCaptureInfo, bCaptureInvalidated = %d\n", pBuildInfo->bCaptureInvalidated);) pBuildInfo->bCurrCapture = 0; _ET9ClearMem((ET9U8*)pBuildInfo->pCaptures, (ET9UINT)(ET9MAXBUILDCAPTURES * sizeof(ET9AWCaptureBuild))); if (pWordSymbInfo->bNumSymbs) { __InitCaptureInfoLonger(pLingInfo, 0, (ET9U16)(pWordSymbInfo->bNumSymbs - 1)); } pBuildInfo->bCaptureInvalidated = 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * / * * * * * */ static void ET9LOCALCALL __PreCaptureWord(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9AssertLog(pLingInfo != NULL); if (pWordSymbInfo->bNumSymbs > pLingCmnInfo->Private.bLastBuildLen) { __InitCaptureInfoLonger(pLingInfo, pLingCmnInfo->Private.bLastBuildLen, (ET9U16)(pWordSymbInfo->bNumSymbs - 1)); } else if (pWordSymbInfo->bNumSymbs) { __InitCaptureInfoShorter(pLingInfo, (ET9U16)(pWordSymbInfo->bNumSymbs - 1), (ET9U16)(pLingCmnInfo->Private.bLastBuildLen - 1)); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * */ static void ET9LOCALCALL __AssureLockedString(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex) { const ET9U16 wLockPoint = pLingInfo->pLingCmnInfo->Private.wCurrLockPoint; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); if (wLockPoint) { ET9U16 wCount = __ET9Min(pWord->Base.wWordLen, wLockPoint); ET9SYMB *pSymb = pWord->Base.sWord; ET9SymbInfo *pSymbInfo = &pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo[wIndex]; for (; wCount; --wCount, ++pSymb, ++pSymbInfo) { *pSymb = pSymbInfo->sLockedSymb; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9BOOL ET9LOCALCALL __CaptureIsFlushPoint(ET9AWLingInfo * const pLingInfo, const ET9U16 wSymbolPoint) { ET9U8 *pbFlushPos; ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWordSymbInfo != NULL); /* validate */ if (!pWordSymbInfo->bNumSymbs || !wSymbolPoint) { return 0; } /* check point */ pbFlushPos = &pBuildInfo->pbFlushPos[wSymbolPoint - 1]; if (*pbFlushPos && *pbFlushPos < pWordSymbInfo->bNumSymbs) { return 1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9U16 ET9LOCALCALL __CaptureGetFlushStringLengthAtPoint(ET9AWLingInfo * const pLingInfo, const ET9U16 wSymbolPoint) { ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; const ET9U16 wLockPoint = pLingInfo->pLingCmnInfo->Private.wCurrLockPoint; ET9U16 wStringLen; ET9U16 wSymbolLen; ET9AssertLog(pLingInfo != NULL); if (!__CaptureIsFlushPoint(pLingInfo, wSymbolPoint)) { return 0; } wSymbolLen = wSymbolPoint; wStringLen = pBuildInfo->pbFlushLen[wSymbolLen - 1]; if (wSymbolLen <= wLockPoint && wStringLen != wSymbolLen) { WLOG2(fprintf(pLogFile2, "__CaptureGetFlushStringLengthAtPoint, lock point overrides actual string length (%d <- %d)\n", wStringLen, wSymbolLen);) wStringLen = wSymbolLen; } return wStringLen; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9U16 ET9LOCALCALL __CaptureGetFlushPoint(ET9AWLingInfo * const pLingInfo, const ET9U8 bPeekOffset) { ET9U16 wPoint; ET9U8 *pbFlushPos; ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWordSymbInfo != NULL); /* validate */ if (!pWordSymbInfo->bNumSymbs) { return 0; } /* look for a flush point */ wPoint = pWordSymbInfo->bNumSymbs - 1 + bPeekOffset; pbFlushPos = &pBuildInfo->pbFlushPos[wPoint - 1]; for (; wPoint; --wPoint, --pbFlushPos) { if (*pbFlushPos && *pbFlushPos < pWordSymbInfo->bNumSymbs + bPeekOffset) { return wPoint; } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U16 ET9LOCALCALL __CaptureGetFlushSymbolLength(ET9AWLingInfo * const pLingInfo) { ET9U16 wPoint; ET9AssertLog(pLingInfo != NULL); wPoint = __CaptureGetFlushPoint(pLingInfo, 0); if (!wPoint) { return 0; } return wPoint; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9U16 ET9LOCALCALL __CaptureGetFlushStringLength(ET9AWLingInfo * const pLingInfo, const ET9BOOL bDisregardLock) { ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; const ET9U16 wLockPoint = pLingInfo->pLingCmnInfo->Private.wCurrLockPoint; ET9U16 wStringLen; ET9U16 wSymbolLen; ET9AssertLog(pLingInfo != NULL); wSymbolLen = __CaptureGetFlushSymbolLength(pLingInfo); if (!wSymbolLen) { return 0; } wStringLen = pBuildInfo->pbFlushLen[wSymbolLen - 1]; if (wSymbolLen <= wLockPoint && wStringLen != wSymbolLen) { if (bDisregardLock) { WLOG2(fprintf(pLogFile2, "__CaptureGetFlushStringLength, lock info is applicable, but disregarded\n");) } else { WLOG2(fprintf(pLogFile2, "__CaptureGetFlushStringLength, lock point overrides actual string length (%d <- %d)\n", wStringLen, wSymbolLen);) wStringLen = wSymbolLen; } } return wStringLen; } /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * */ static void ET9LOCALCALL __ApplySymbInfoShiftToWord(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo *pWord) { ET9U16 i; ET9U16 wLockPoint; ET9SymbInfo *pSymbInfo; ET9SYMB *pSymb; ET9U32 dwLdbNum; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); i = pWord->Base.wWordLen; if (i) { dwLdbNum = (pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum; pSymbInfo = &pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo[i-1]; pSymb = &pWord->Base.sWord[i-1]; wLockPoint = pLingInfo->pLingCmnInfo->Private.wCurrLockPoint; for (; i && i > wLockPoint; --i, --pSymbInfo, --pSymb) { if (pSymbInfo->eShiftState) { *pSymb = _ET9SymToUpper(*pSymb, dwLdbNum); ET9AssertLog(pSymbInfo->bForcedLowercase == 0); } else if (pSymbInfo->bForcedLowercase) { *pSymb = _ET9SymToLower(*pSymb, dwLdbNum); } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __CaptureGetFlush(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pFlush) { ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9U16 wFlushLen = __CaptureGetFlushStringLength(pLingInfo, 0); ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pFlush != NULL); WLOG2(fprintf(pLogFile2, "__CaptureGetFlush, wFlushLen = %d (string length)\n", wFlushLen);) _InitPrivWordInfo(pFlush); if (!wFlushLen) { WLOG2(fprintf(pLogFile2, "__CaptureGetFlush, no flush (empty)\n");) return; } /* get the flush info */ { ET9U16 wCount = wFlushLen; ET9SYMB *pSymb1 = pBuildInfo->psFlushedSymbs; ET9SYMB *pSymb2 = pFlush->Base.sWord; for (; wCount; --wCount) { *(pSymb2++) = *(pSymb1++); } } /* attributes */ pFlush->Base.wWordLen = wFlushLen; pFlush->Body.bWordSrc = ET9WORDSRC_CAPTURE; pFlush->Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; /* assure locked chars */ __AssureLockedString(pLingInfo, pFlush, 0); /* update syms based on shift/forced lowercasing info */ __ApplySymbInfoShiftToWord(pLingInfo, pFlush); /* done */ WLOG2Word(pLogFile2, "__CaptureGetFlush, flush from capture", pFlush); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __CaptureFlushPoint(ET9AWLingInfo * const pLingInfo, const ET9U16 wSymbolLen, ET9AWPrivWordInfo * const pWord) { ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); ET9AssertLog(wSymbolLen); ET9AssertLog(pWordSymbInfo->bNumSymbs >= wSymbolLen); ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); WLOG2(fprintf(pLogFile2, "__CaptureFlush, wSymbolLen = %d @ bNumSymbs = %d\n", wSymbolLen, pWordSymbInfo->bNumSymbs);) WLOG2Word(pLogFile2, "__CaptureFlush, word", pWord); WLOG2(fprintf(pLogFile2, "__CaptureFlush, current flush, point = %d, symbols = %d, length = %d\n", __CaptureGetFlushPoint(pLingInfo, 0), __CaptureGetFlushSymbolLength(pLingInfo), __CaptureGetFlushStringLength(pLingInfo, 0));) /* update flush info */ pBuildInfo->pbFlushPos[wSymbolLen-1] = pWordSymbInfo->bNumSymbs; pBuildInfo->pbFlushLen[wSymbolLen-1] = (ET9U8)pWord->Base.wWordLen; { ET9U16 wCount = pWord->Base.wWordLen; ET9SYMB *pSymb1 = pBuildInfo->psFlushedSymbs; ET9SYMB *pSymb2 = pWord->Base.sWord; for (; wCount; --wCount) { *(pSymb1++) = *(pSymb2++); } } /* log capture info */ WLOG2(fprintf(pLogFile2, "__CaptureFlush, peek point = %d\n", __CaptureGetFlushPoint(pLingInfo, 1));) WLOG2Captures(pLogFile2, "__CaptureFlush, after capture flush", pLingInfo); /* verify */ ET9AssertLog(__CaptureGetFlushPoint(pLingInfo, 1) == wSymbolLen); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __CaptureUpdateFlushInfo(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9AWBuildInfo * const pBuildInfo = &pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWordSymbInfo != NULL); ET9AssertLog(pWord != NULL); ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); WLOG2(fprintf(pLogFile2, "__CaptureUpdateFlushInfo, bNumSymbs = %d\n", pWordSymbInfo->bNumSymbs);) WLOG2Word(pLogFile2, "__CaptureUpdateFlushInfo, word", pWord); /* update flush diff info */ if (pWordSymbInfo->bNumSymbs && pWordSymbInfo->bNumSymbs > pLingCmnInfo->Private.bLastBuildLen) { ET9U16 wSymbolLen = pWordSymbInfo->bNumSymbs; WLOG2(fprintf(pLogFile2, "__CaptureUpdateFlushInfo, pbDefaultLen[%d], %d updated to %d\n", wSymbolLen-1, pBuildInfo->pbDefaultLen[wSymbolLen-1], pWord->Base.wWordLen);) WLOG2(fprintf(pLogFile2, "__CaptureUpdateFlushInfo, pbDefaultCompLen[%d], %d updated to %d\n", wSymbolLen-1, pBuildInfo->pbDefaultCompLen[wSymbolLen-1], pWord->Base.wWordCompLen);) if (pBuildInfo->pbDefaultLen[wSymbolLen-1] >= pWord->Base.wWordLen) { WLOG2(fprintf(pLogFile2, "__CaptureUpdateFlushInfo, updating\n");) pBuildInfo->pbFlushLen[wSymbolLen-1] = 0; pBuildInfo->pbDefaultLen[wSymbolLen-1] = (ET9U8)pWord->Base.wWordLen; pBuildInfo->pbDefaultCompLen[wSymbolLen-1] = (ET9U8)pWord->Base.wWordCompLen; } else { WLOG2(fprintf(pLogFile2, "__CaptureUpdateFlushInfo, supressing update to longer\n");) } if ((pWord->Base.wWordLen - pWord->Base.wWordCompLen) != wSymbolLen) { WLOG2Captures(pLogFile2, "__CaptureUpdateFlushInfo, after flush update", pLingInfo); } } else { WLOG2(fprintf(pLogFile2, "__CaptureUpdateFlushInfo, no default update, bNumSymbs = %d, bLastBuildLen = %d\n", pWordSymbInfo->bNumSymbs, pLingCmnInfo->Private.bLastBuildLen);) } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9U16 ET9LOCALCALL __CaptureGetDefaultStringLength(ET9AWLingInfo * const pLingInfo, const ET9U16 wSymbolLen) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWBuildInfo * const pBuildInfo = &pLingCmnInfo->Private.sBuildInfo; const ET9U16 wLockPoint = pLingCmnInfo->Private.wCurrLockPoint; const ET9U16 wStoreIndex = (ET9U16)(wSymbolLen - 1); ET9AssertLog(pLingInfo != NULL); ET9AssertLog(wSymbolLen <= ET9MAXWORDSIZE); if (!wSymbolLen || wSymbolLen > ET9MAXWORDSIZE) { WLOG2(fprintf(pLogFile2, "__CaptureGetDefaultStringLength, failure, returns zero\n");) return 0; } if (pBuildInfo->pwDefaultPos[wStoreIndex] == UNDEFINED_STRING_INDEX) { WLOG2(fprintf(pLogFile2, "__CaptureGetDefaultStringLength, missing, wSymbolLen = %d, length = %d\n", wSymbolLen, (wSymbolLen - 1));) return (ET9U16)(wSymbolLen - 1); } if (wSymbolLen - 1 <= wLockPoint || pBuildInfo->pbDefaultLen[wStoreIndex] <= wLockPoint) { WLOG2(fprintf(pLogFile2, "__CaptureGetDefaultStringLength, from lock, wSymbolLen = %d, length = %d\n", wSymbolLen, (wSymbolLen - 1));) return (ET9U16)(wSymbolLen - 1); } WLOG2(fprintf(pLogFile2, "__CaptureGetDefaultStringLength, from store, wSymbolLen = %d, length = %d\n", wSymbolLen, (pBuildInfo->pbDefaultLen[wSymbolLen - 1] - pBuildInfo->pbDefaultCompLen[wSymbolLen - 1]));) ET9AssertLog(pBuildInfo->pbDefaultLen[wSymbolLen - 1] >= pBuildInfo->pbDefaultCompLen[wSymbolLen - 1]); return (ET9U16)(pBuildInfo->pbDefaultLen[wSymbolLen - 1] - pBuildInfo->pbDefaultCompLen[wSymbolLen - 1]); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static ET9BOOL ET9LOCALCALL __CaptureGetDefault(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wSymbolLen) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWBuildInfo * const pBuildInfo = &pLingCmnInfo->Private.sBuildInfo; const ET9U16 wLockPoint = pLingCmnInfo->Private.wCurrLockPoint; const ET9U16 wStoreIndex = (ET9U16)(wSymbolLen - 1); WLOG2(fprintf(pLogFile2, "__CaptureGetDefault, wSymbolLen = %d\n", wSymbolLen);) _InitPrivWordInfo(pWord); if (!wSymbolLen || wSymbolLen > ET9MAXWORDSIZE || pBuildInfo->pwDefaultPos[wStoreIndex] == UNDEFINED_STRING_INDEX) { return 0; } if (wSymbolLen - 1 <= wLockPoint || pBuildInfo->pbDefaultLen[wStoreIndex] <= wLockPoint) { pWord->Base.wWordLen = (ET9U16)__ET9Min((wSymbolLen - 1), wLockPoint); __AssureLockedString(pLingInfo, pWord, 0); WLOG2Word(pLogFile2, "__CaptureDefault, word from lock", pWord); } else { pWord->Base.wWordLen = pBuildInfo->pbDefaultLen[wStoreIndex]; pWord->Base.wWordCompLen = pBuildInfo->pbDefaultCompLen[wStoreIndex]; _ET9SymCopy(pWord->Base.sWord, &pBuildInfo->psDefaultSymbs[pBuildInfo->pwDefaultPos[wStoreIndex]], pWord->Base.wWordLen); if (wLockPoint) { if (wSymbolLen - 1 <= wLockPoint && pWord->Base.wWordLen < wSymbolLen - 1) { WLOG2(fprintf(pLogFile2, "__CaptureGetDefault, too short for lock, extending\n");) pWord->Base.wWordLen = (ET9U16)(wSymbolLen - 1); pWord->Base.wWordCompLen = 0; } else if (wSymbolLen - 1 <= wLockPoint && (pWord->Base.wWordLen - pWord->Base.wWordCompLen) < wSymbolLen - 1) { WLOG2(fprintf(pLogFile2, "__CaptureGetDefault, stem too short for lock, extending\n");) pWord->Base.wWordCompLen = (ET9U16)(pWord->Base.wWordLen - (wSymbolLen - 1)); } __AssureLockedString(pLingInfo, pWord, 0); } WLOG2Word(pLogFile2, "__CaptureDefault, word from store", pWord); } /* language index */ pWord->Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; /* update syms based on shift/forced lowercasing info */ __ApplySymbInfoShiftToWord(pLingInfo, pWord); return 1; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static void ET9LOCALCALL __CaptureDefault(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wSymbolLen, const ET9U16 bLastBuildLen) { ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; const ET9U16 wStoreIndex = (ET9U16)(wSymbolLen - 1); ET9U16 wWordLen = pWord->Base.wWordLen; ET9U16 wWordCompLen = pWord->Base.wWordCompLen; WLOG2Word(pLogFile2, "__CaptureDefault, word", pWord); ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); if (!wSymbolLen || wSymbolLen > ET9MAXWORDSIZE) { return; } /* only store on increasing length or missing data */ if (bLastBuildLen >= wSymbolLen && pBuildInfo->pwDefaultPos[wStoreIndex] != UNDEFINED_STRING_INDEX) { WLOG2(fprintf(pLogFile2, "__CaptureDefault, shrinking and ok data, skipping\n");) return; } else { WLOG2(fprintf(pLogFile2, "__CaptureDefault, increasing or same build or missing data (bLastBuildLen = %d, wSymbolLen = %d, pos[%d] = %d)\n", bLastBuildLen, wSymbolLen, wStoreIndex, pBuildInfo->pwDefaultPos[wStoreIndex]);) /* try to maintain default (word) length though */ if (pBuildInfo->pbDefaultLen[wStoreIndex] && pBuildInfo->pbDefaultLen[wStoreIndex] >= pBuildInfo->pbDefaultCompLen[wStoreIndex]) { WLOG2(fprintf(pLogFile2, " trying to reuse old default len data\n");) if (wWordLen >= pBuildInfo->pbDefaultLen[wStoreIndex]) { wWordLen = pBuildInfo->pbDefaultLen[wStoreIndex]; wWordCompLen = pBuildInfo->pbDefaultCompLen[wStoreIndex]; } else { ET9U16 wStemLen = (ET9U16)(pBuildInfo->pbDefaultLen[wStoreIndex] - pBuildInfo->pbDefaultCompLen[wStoreIndex]); if (wWordLen >= wStemLen) { wWordCompLen = (ET9U16)(wWordLen - wStemLen); } else { wWordCompLen = 0; } } } else if (bLastBuildLen >= wSymbolLen && wWordLen >= wSymbolLen && wSymbolLen) { WLOG2(fprintf(pLogFile2, " no default data, shrinking, truncating to symbol len - 1 (%d)\n", (wSymbolLen - 1));) wWordLen = (ET9U16)(wSymbolLen - 1); wWordCompLen = 0; } } /* validate word length */ if (!wWordLen || wWordLen > ET9MAXWORDSIZE) { pBuildInfo->pwDefaultPos[wStoreIndex] = UNDEFINED_STRING_INDEX; pBuildInfo->pbDefaultLen[wStoreIndex] = 0; pBuildInfo->pbDefaultCompLen[wStoreIndex] = 0; return; } /* store */ { if (!wStoreIndex) { /* first entry, just start from the beginning */ pBuildInfo->pwDefaultPos[wStoreIndex] = 0; } else { /* additional entry, overlap previous or after previous */ ET9U16 wPos; ET9U16 wIndex; ET9U16 wPrevStoreIndex; ET9U16 wWriteStart = 0; ET9U16 wWriteLen = 0; /* look for previous */ for (wPrevStoreIndex = (ET9U16)(wStoreIndex - 1); ; --wPrevStoreIndex) { if (!wPrevStoreIndex || pBuildInfo->pwDefaultPos[wPrevStoreIndex] != UNDEFINED_STRING_INDEX) { break; } } /* something else with the same start pos and longer */ for (wIndex = 0; wIndex < wPrevStoreIndex; ++wIndex) { if (pBuildInfo->pwDefaultPos[wIndex] == pBuildInfo->pwDefaultPos[wPrevStoreIndex] && pBuildInfo->pbDefaultLen[wIndex] > pBuildInfo->pbDefaultLen[wPrevStoreIndex]) { wPrevStoreIndex = wIndex; } } /* see if things can be reused */ if (pBuildInfo->pwDefaultPos[wPrevStoreIndex] == UNDEFINED_STRING_INDEX) { wPos = 0; wWriteStart = wPos; wWriteLen = wWordLen; } else { /* check overlap */ ET9U16 wCmpLen; ET9SYMB *pStr1; ET9SYMB *pStr2; if (pBuildInfo->pbDefaultLen[wPrevStoreIndex] <= wWordLen) { wCmpLen = pBuildInfo->pbDefaultLen[wPrevStoreIndex]; } else { wCmpLen = wWordLen; } pStr1 = pWord->Base.sWord; pStr2 = &pBuildInfo->psDefaultSymbs[pBuildInfo->pwDefaultPos[wPrevStoreIndex]]; ++wCmpLen; while (--wCmpLen) { if (*pStr1++ != *pStr2++) { break; } } if (!wCmpLen) { /* overlap, reuse */ wPos = pBuildInfo->pwDefaultPos[wPrevStoreIndex]; if (wWordLen <= pBuildInfo->pbDefaultLen[wPrevStoreIndex]) { wWriteStart = wPos; wWriteLen = 0; } else { wWriteStart = wPos + pBuildInfo->pbDefaultLen[wPrevStoreIndex]; wWriteLen = (ET9U16)(wWordLen - pBuildInfo->pbDefaultLen[wPrevStoreIndex]); } } else { /* no overlap, move out */ wPos = pBuildInfo->pwDefaultPos[wPrevStoreIndex] + pBuildInfo->pbDefaultLen[wPrevStoreIndex]; wWriteStart = wPos; wWriteLen = wWordLen; } } /* make sure it fits in the buffer */ if (wPos + wWordLen > ET9DEFAULTSTORESIZE) { /* need to start over */ wPos = 0; wWriteStart = wPos; wWriteLen = wWordLen; } /* wipe out possibly over written entries */ if (wWriteLen) { ET9U16 wIndex; for (wIndex = 0; wIndex < wStoreIndex; ++wIndex) { if (pBuildInfo->pwDefaultPos[wIndex] >= wWriteStart && pBuildInfo->pwDefaultPos[wIndex] < wWriteStart + wWriteLen) { pBuildInfo->pwDefaultPos[wIndex] = UNDEFINED_STRING_INDEX; } } } /* found good spot */ pBuildInfo->pwDefaultPos[wStoreIndex] = wPos; } /* actually store stuff */ WLOG2(fprintf(pLogFile2, " actually storing, pbDefaultLen[%d] = %d, pbDefaultCompLen[%d] = %d\n", wStoreIndex, wWordLen, wStoreIndex, wWordCompLen);) pBuildInfo->pbDefaultLen[wStoreIndex] = (ET9U8)wWordLen; pBuildInfo->pbDefaultCompLen[wStoreIndex] = (ET9U8)wWordCompLen; _ET9SymCopy(&pBuildInfo->psDefaultSymbs[pBuildInfo->pwDefaultPos[wStoreIndex]], pWord->Base.sWord, wWordLen); ET9AssertLog(pBuildInfo->pwDefaultPos[wStoreIndex] < ET9DEFAULTSTORESIZE); ET9AssertLog(pBuildInfo->pwDefaultPos[wStoreIndex] + pBuildInfo->pbDefaultLen[wStoreIndex] <= ET9DEFAULTSTORESIZE); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ static ET9BOOL ET9LOCALCALL __CaptureGetWord(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wSymbolIndex, const ET9U16 wSymbolLength, const ET9U8 bStemsAllowed) { ET9U16 wCount; ET9SYMB *pSymb1; ET9SYMB *pSymb2; ET9U16 wFlushSymbolLen; ET9U16 wFlushStringLen; ET9U16 wOmitSymbolLength; ET9U16 wOmitStringLength; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AWCaptureBuild * const pCapture = &pBuildInfo->pCaptures[pBuildInfo->bCurrCapture]; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); WLOG2(fprintf(pLogFile2, "__CaptureGetWord, wSymbolIndex = %d, wSymbolLength = %d, StemsAllowed = %d\n", wSymbolIndex, wSymbolLength, bStemsAllowed);) /* init (result) */ _InitPrivWordInfo(pWord); /* something captured that can be used? */ wFlushSymbolLen = __CaptureGetFlushSymbolLength(pLingInfo); wFlushStringLen = __CaptureGetFlushStringLength(pLingInfo, 0); wOmitSymbolLength = wSymbolIndex - wFlushSymbolLen; wOmitStringLength = wOmitSymbolLength; WLOG2(fprintf(pLogFile2, "__CaptureGetWord, wFlushSymbolLen = %d, wFlushStringLen = %d, wOmitSymbolLength = %d, wOmitStringLength = %d\n", wFlushSymbolLen, wFlushStringLen, wOmitSymbolLength, wOmitStringLength);) if (!pCapture->bIsValid) { WLOG2(fprintf(pLogFile2, "__CaptureGetWord, nothing valid exists (fail)\n");) return 0; } if (!bStemsAllowed && pCapture->wWordCompLen) { WLOG2(fprintf(pLogFile2, "__CaptureGetWord, requested no stem but a stem is captured (fail)\n");) return 0; } if (wFlushStringLen != __CaptureGetFlushStringLength(pLingInfo, 1)) { WLOG2(fprintf(pLogFile2, "__CaptureGetWord, a lock affects the flush string length (fail)\n");) return 0; } if (wSymbolIndex < wFlushSymbolLen) { WLOG2(fprintf(pLogFile2, "__CaptureGetWord, bad symbol index, %d < %d (fail)\n", wSymbolIndex, wFlushSymbolLen);) return 0; } if (wFlushSymbolLen + wOmitSymbolLength + wSymbolLength != pCapture->wSymbolLen) { WLOG2(fprintf(pLogFile2, "__CaptureGetWord, bad capture symbol length, %d != %d (fail)\n", wFlushSymbolLen + wOmitSymbolLength + wSymbolLength, pCapture->wSymbolLen);) return 0; } if (pCapture->wWordLen == pCapture->wWordCompLen + wFlushStringLen + wOmitStringLength) { WLOG2(fprintf(pLogFile2, "__CaptureGetWord, bad capture string length, %d == %d (fail)\n", pCapture->wWordLen, pCapture->wWordCompLen + wFlushStringLen + wOmitStringLength);) return 0; } if (pCapture->wWordLen < pCapture->wWordCompLen + wFlushStringLen + wOmitStringLength) { WLOG2(fprintf(pLogFile2, "__CaptureGetWord, *VERY* bad capture string length, %d < %d (fail)\n", pCapture->wWordLen, pCapture->wWordCompLen + wFlushStringLen + wOmitStringLength);) return 0; } /* get the word from capture */ pSymb1 = &pCapture->sWord[wFlushStringLen + wOmitStringLength]; pSymb2 = pWord->Base.sWord; wCount = (ET9U16)(pCapture->wWordLen - pCapture->wWordCompLen - wFlushStringLen - wOmitStringLength); pWord->Base.wWordLen = wCount; for (; wCount; --wCount) { *(pSymb2++) = *(pSymb1++); } /* set attributes */ pWord->Body.xTapFreq = 1; pWord->Body.xWordFreq = 1; pWord->Body.bWordSrc = ET9WORDSRC_CAPTURE; WLOG2Word(pLogFile2, "__CaptureGetWord, word found", pWord); /* verify */ ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); #ifdef ET9_DEBUG { ET9INT snCount = pWord->Base.wWordLen; ET9SYMB *psSymb = pWord->Base.sWord; for (; snCount; --snCount, ++psSymb) { ET9AssertLog(*psSymb && *psSymb != 0xcccc); } } #endif /* success */ return 1; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static void ET9LOCALCALL __CaptureWord(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wSymbolLen) { ET9U8 bStartNew; ET9S16 swAddWordLen; ET9S16 swAddWordCompLen; ET9U16 wAddSymbolLen; ET9U16 wCount; ET9SYMB *pSymb1; ET9SYMB *pSymb2; ET9AWCaptureBuild *pCapture; ET9AWCaptureAction *pCaptureAction; ET9WordSymbInfo * const pWordSymbInfo = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo; ET9AWBuildInfo * const pBuildInfo = &pLingInfo->pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWordSymbInfo != NULL); ET9AssertLog(pWord != NULL); ET9AssertLog(wSymbolLen <= ET9MAXWORDSIZE); ET9AssertLog(pWord->Base.wWordLen <= ET9MAXWORDSIZE); WLOG2Word(pLogFile2, "__CaptureWord, word to capture", pWord); /* don't capture on zero symbols */ if (!wSymbolLen) { WLOG2(fprintf(pLogFile2, "__CaptureWord, discarding capture - zero symbol length\n");) return; } /* find place to capture in */ pCapture = &pBuildInfo->pCaptures[pBuildInfo->bCurrCapture]; bStartNew = 0; swAddWordLen = 0; swAddWordCompLen = 0; wAddSymbolLen = 0; if (!pCapture->bIsValid || !pCapture->wWordLen) { /* this is a valid spot (empty) */ bStartNew = 2; } else if (pCapture->wWordLen > pWord->Base.wWordLen) { /* need a new spot */ bStartNew = 1; } else if (pCapture->wWordLen <= pWord->Base.wWordLen) { /* check if it could be an extension*/ pSymb1 = pCapture->sWord; pSymb2 = pWord->Base.sWord; for (wCount = pCapture->wWordLen - pCapture->wWordCompLen; wCount; --wCount) { if (*(pSymb1++) != *(pSymb2++)) { break; } } if (wCount) { bStartNew = 1; } else { bStartNew = 0; swAddWordLen = (ET9S16)(pWord->Base.wWordLen - pCapture->wWordLen); swAddWordCompLen = (ET9S16)(pWord->Base.wWordCompLen - pCapture->wWordCompLen); wAddSymbolLen = (ET9U16)(wSymbolLen - pCapture->wSymbolLen); } } /* anthing new? */ if (!bStartNew && !swAddWordLen && !swAddWordCompLen && !wAddSymbolLen) { WLOG2Captures(pLogFile2, "__CaptureWord, nothing captured", pLingInfo); return; } /* move word into capture */ if (bStartNew) { if (bStartNew == 1) { pBuildInfo->bCurrCapture = (pBuildInfo->bCurrCapture + 1) % ET9MAXBUILDCAPTURES; if (pBuildInfo->pCaptures[pBuildInfo->bCurrCapture].bIsValid) { WLOG2(fprintf(pLogFile2, "__CaptureWord, *** OVERWRITING CAPTURE INFO ***\n");) } _ET9ClearMem((ET9U8*)&pBuildInfo->pCaptures[pBuildInfo->bCurrCapture], sizeof(ET9AWCaptureBuild)); } pCapture = &pBuildInfo->pCaptures[pBuildInfo->bCurrCapture]; swAddWordLen = pWord->Base.wWordLen; swAddWordCompLen = pWord->Base.wWordCompLen; wAddSymbolLen = wSymbolLen; } if (pWord->Base.wWordLen > pCapture->wWordLen - pCapture->wWordCompLen) { pSymb1 = &pCapture->sWord[pCapture->wWordLen - pCapture->wWordCompLen]; pSymb2 = &pWord->Base.sWord[pCapture->wWordLen - pCapture->wWordCompLen]; for (wCount = (pWord->Base.wWordLen - (pCapture->wWordLen - pCapture->wWordCompLen)); wCount; --wCount) { *(pSymb1++) = *(pSymb2++); } } pCapture->bIsValid = 1; pCapture->wWordLen = pWord->Base.wWordLen; pCapture->wWordCompLen = pWord->Base.wWordCompLen; pCapture->wSymbolLen = wSymbolLen; /* update action info */ pCaptureAction = &pBuildInfo->pCaptureActions[pWordSymbInfo->bNumSymbs-1]; if (bStartNew == 1) { pCaptureAction->bPop = 1; } else { pCaptureAction->bPop = 0; } pCaptureAction->sbAddWordLen = (ET9S8)swAddWordLen; pCaptureAction->sbAddWordCompLen = (ET9S8)swAddWordCompLen; pCaptureAction->bAddSymbolLen = (ET9U8)wAddSymbolLen; ET9AssertLog(pCaptureAction->sbAddWordLen == swAddWordLen); ET9AssertLog(pCaptureAction->sbAddWordCompLen == swAddWordCompLen); ET9AssertLog(pCaptureAction->bAddSymbolLen == wAddSymbolLen); /* log capture info */ WLOG2Captures(pLogFile2, "__CaptureWord, after capture word", pLingInfo); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __CaptureHandleAction(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9AWBuildInfo * const pBuildInfo = &pLingCmnInfo->Private.sBuildInfo; ET9AssertLog(pLingInfo != NULL); /* reset all capture info if this is the beginning of something new (or invalidated) */ if (!pLingCmnInfo->Private.bLastBuildLen || pBuildInfo->bCaptureInvalidated) { __ResetAllCaptureInfo(pLingInfo); return; } /* handle capture action */ if (pLingCmnInfo->Private.bLastBuildLen < pWordSymbInfo->bNumSymbs) { WLOG2(fprintf(pLogFile2, "__CaptureHandleAction, new symb, bNumSymbs = %d (no action)\n", pWordSymbInfo->bNumSymbs);) } else if (pLingCmnInfo->Private.bLastBuildLen > pWordSymbInfo->bNumSymbs) { ET9S16 swCurrSymb; for (swCurrSymb = (ET9S16)(pLingCmnInfo->Private.bLastBuildLen - 1); swCurrSymb >= (ET9S16)pWordSymbInfo->bNumSymbs; --swCurrSymb) { ET9AWCaptureBuild *pCapture = &pBuildInfo->pCaptures[pBuildInfo->bCurrCapture]; ET9AWCaptureAction *pCaptureAction = &pBuildInfo->pCaptureActions[swCurrSymb]; WLOG2(fprintf(pLogFile2, "__CaptureHandleAction, one symb less, bNumSymbs = %d, bCaptureAction = %d,%d,%d,%d, bCurrCapture = %d\n", swCurrSymb, pCaptureAction->bPop, pCaptureAction->sbAddWordLen, pCaptureAction->sbAddWordCompLen, pCaptureAction->bAddSymbolLen, pBuildInfo->bCurrCapture);) if (pCaptureAction->bPop) { ET9U8 *pbCurrCapture = &pBuildInfo->bCurrCapture; pCapture->bIsValid = 0; if (!*pbCurrCapture) { *pbCurrCapture = ET9MAXBUILDCAPTURES; } --(*pbCurrCapture); } else if (pCapture->bIsValid) { ET9AssertLog(pCapture->wWordLen >= pCaptureAction->sbAddWordLen); pCapture->wWordLen = pCapture->wWordLen - pCaptureAction->sbAddWordLen; pCapture->wWordCompLen = pCapture->wWordCompLen - pCaptureAction->sbAddWordCompLen; pCapture->wSymbolLen = pCapture->wSymbolLen - pCaptureAction->bAddSymbolLen; if (!pCapture->wWordLen) { pCapture->bIsValid = 0; } ET9AssertLog(pCapture->wWordLen > pCapture->wWordCompLen || !pCapture->wWordLen && !pCapture->wWordCompLen); } } } else { WLOG2(fprintf(pLogFile2, "__CaptureHandleAction, same length, bNumSymbs = %d (no action)\n", pWordSymbInfo->bNumSymbs);) } WLOG2Captures(pLogFile2, "__CaptureHandleAction, after capture action", pLingInfo); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __ShiftSubListUp(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nFirst, const ET9UINT nLast) { const ET9UINT nDown = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nFirst - 1]; ET9UINT nIndex; ET9AssertLog(pLingCmnInfo); ET9AssertLog(nFirst <= nLast); ET9AssertLog(nFirst > 0); ET9AssertLog(nLast < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); ET9AssertLog(pLingCmnInfo->Private.bDoneBuilding); WLOG2(fprintf(pLogFile2, "__ShiftSubListUp, nFirst %u, nLast %u\n", nFirst, nLast);) for (nIndex = nFirst - 1; nIndex < nLast; ++nIndex) { pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex] = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex + 1]; /* keep direct array access here*/ } pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLast] = nDown; /* exact index */ if (pLingCmnInfo->Private.nExactIndex == (nFirst - 1)) { pLingCmnInfo->Private.nExactIndex = nLast; } else if (pLingCmnInfo->Private.nExactIndex >= nFirst && pLingCmnInfo->Private.nExactIndex <= nLast) { --pLingCmnInfo->Private.nExactIndex; } ET9AssertLog(pLingCmnInfo->Private.nExactIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || ISEXACTSRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nExactIndex]].Body.bWordSrc)); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __ShiftSubListDown(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nFirst, const ET9UINT nLast) { const ET9UINT nUp = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLast + 1]; ET9UINT nIndex; ET9AssertLog(pLingCmnInfo); ET9AssertLog(nFirst <= nLast); ET9AssertLog(nLast + 1 < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); ET9AssertLog(pLingCmnInfo->Private.bDoneBuilding); WLOG2(fprintf(pLogFile2, "__ShiftSubListDown, nFirst %u, nLast %u\n", nFirst, nLast);) for (nIndex = nLast + 1; nIndex > nFirst; --nIndex) { pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex] = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex - 1]; /* keep direct array access here*/ } pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nFirst] = nUp; /* exact index */ if (pLingCmnInfo->Private.nExactIndex == (nLast + 1)) { pLingCmnInfo->Private.nExactIndex = nFirst; } else if (pLingCmnInfo->Private.nExactIndex >= nFirst && pLingCmnInfo->Private.nExactIndex <= nLast) { ++pLingCmnInfo->Private.nExactIndex; } ET9AssertLog(pLingCmnInfo->Private.nExactIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || ISEXACTSRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nExactIndex]].Body.bWordSrc)); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __RemoveWord(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nWordIndex) { ET9UINT nIndex; ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nWordIndex]]; ET9AssertLog(pLingCmnInfo); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords > 0); ET9AssertLog(nWordIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); ET9AssertLog(pLingCmnInfo->Private.bDoneBuilding); pWord->Body.bWordSrc = ET9WORDSRC_NONE; pWord->Base.sWord[0] = 0; pWord->Base.wWordLen = 0; pWord->Base.wWordCompLen = 0; pWord->Base.wSubstitutionLen = 0; for (nIndex = nWordIndex; nIndex + 1 < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex] = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex + 1]; /* keep direct array access here*/ } --pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; /* special case of list length 1 resets the default index */ if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords == 1) { pLingCmnInfo->Private.nDefaultIndex = 0; } /* exact index */ if (pLingCmnInfo->Private.nExactIndex == nWordIndex) { pLingCmnInfo->Private.nExactIndex = pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize; } else if (pLingCmnInfo->Private.nExactIndex > nWordIndex) { --pLingCmnInfo->Private.nExactIndex; } ET9AssertLog(pLingCmnInfo->Private.nExactIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || ISEXACTSRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nExactIndex]].Body.bWordSrc)); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9BOOL ET9LOCALCALL __DoGestureProcessing(ET9AWLingInfo * const pLingInfo, ET9U16 * pwGestureValue) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; ET9BOOL bGestureWins; ET9BOOL bIsGestureMax; ET9UINT nMaxScoreWordIndex; __ProfileStart; ET9AssertLog(pLingInfo); ET9AssertLog(pwGestureValue); *pwGestureValue = 0; if (!pLingCmnInfo->Private.bTraceBuild || !pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || !pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { return 0; } WLOG2(fprintf(pLogFile2, "__DoGestureProcessing\n");) bGestureWins = 0; bIsGestureMax = 0; nMaxScoreWordIndex = 0; { ET9UINT nIndex; ET9FREQPART xMaxScore = 0; const ET9BOOL bUseSpmScore = pLingCmnInfo->Private.__SPATH_Status.bHasSpmScores ? 1 : 0; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pWord = &pWordList[pnWordList[nIndex]]; const ET9FREQPART xWordScore = bUseSpmScore ? pWord->Body.xInputScoreSpm : pWord->Body.xInputScore; if (xWordScore > xMaxScore) { nMaxScoreWordIndex = pnWordList[nIndex]; xMaxScore = xWordScore; bIsGestureMax = (pWord->Body.bWordSrc == ET9WORDSRC_GDB) ? 1 : 0; } } } if (bIsGestureMax && !pWordList[nMaxScoreWordIndex].Body.bIsInternalGesture) { /* Handle winning custom gesture */ pLingCmnInfo->Private.nDefaultIndex = 0; pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords = 0; *pwGestureValue = pWordList[nMaxScoreWordIndex].Body.wGestureReturnVal; bGestureWins = 1; } __ProfileEnd(tAW_SelLst_GestureProcessing); return bGestureWins; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __ET9AWPerformPostSort(ET9AWLingInfo *pLingInfo, ET9WordSymbInfo * const pWordSymbInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo *pWord; ET9AWPrivWordInfo *pWord2; ET9UINT i; ET9UINT j; ET9UINT k; ET9UINT m; ET9UINT n; ET9U8 bCompTrackingStemDist = 0; ET9UINT nCompletionGroupCount; ET9UINT *pnWordList; ET9UINT *pnWordList2; ET9U32 dwLdbNum; ET9U32 dwLdbNum2; ET9AssertLog(pLingCmnInfo != NULL); ET9AssertLog(pWordSymbInfo != NULL); /* save the originally ordered selection list */ i = 0; while (i < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { pnWordList = &pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[i++]; pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[*pnWordList]; pWord->Body.bIsGroupBase = 1; pWord->Body.bGroupCount = 1; dwLdbNum = (pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum; if (!pWord->Base.bIsTerm && (pWord->Body.bWordSrc != ET9WORDSRC_CDB) && (pWord->Body.bWordSrc != ET9WORDSRC_RUDB)) { pnWordList2 = pnWordList; ++pnWordList2; nCompletionGroupCount = 1; bCompTrackingStemDist = pWord->Body.bEditDistStem; for (j = i; j < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++j, ++pnWordList2) { pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[*pnWordList2]; if (pWord2->Base.bIsTerm || (pWord2->Body.bWordSrc == ET9WORDSRC_CDB) || (pWord2->Body.bWordSrc == ET9WORDSRC_RUDB) || (bCompTrackingStemDist != pWord2->Body.bEditDistStem && pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode != ET9ASLMODE_CLASSIC)) { break; } else { ++nCompletionGroupCount; } } /* if not only completion in this group, sort 'em */ /* at this point: */ /* i = index pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[i] is beginning of completion group */ /* j = index of position AFTER completion group */ if (nCompletionGroupCount > 1) { /* k will be the insertion point/index for the next subgroup member */ k = i; /* want to continue until entire completion group exhausted */ while (k < j) { pnWordList2 = &pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[k]; m = k; /* now look for a completion matching the stem of the base word */ while (m < j) { pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[*pnWordList2]; dwLdbNum2 = (pWord2->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum; for (n = 0; n < pWordSymbInfo->bNumSymbs; ++n) { if (_ET9SymToLower(pWord->Base.sWord[n], dwLdbNum) != _ET9SymToLower(pWord2->Base.sWord[n], dwLdbNum2)) { break; } } /* if match found with existing group base word, add this word to that group */ if (n == pWordSymbInfo->bNumSymbs) { pWord2->Body.bIsGroupBase = 0; pWord2->Body.bGroupCount = 0; ET9AssertLog(pWord->Body.bGroupCount < 0xFF); ++pWord->Body.bGroupCount; /* roll entry m up to k position if not in current position */ if (m != k) { __ShiftSubListDown(pLingCmnInfo, k, (m - 1)); } /* move insertion position past previous found subgroup entry */ ++k; } /* move on to next entry in completion group */ ++m; ++pnWordList2; } /* while (m < j) */ /* after list traversed, entry at current insertion point will become new 'base' */ if (k < j) { pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[k]]; pWord->Body.bIsGroupBase = 1; pWord->Body.bGroupCount = 1; ++k; } } /* while (k < j) */ } /* if (bCompletionGroupCount > 1) */ /* move master index to position following previous group */ i = j; } /* if (!pWord->Base.bIsTerm */ } /* while (i < pLingCmnInfo->Private.bTotalWords) */ } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * / * * * * */ static void ET9LOCALCALL __UpdateSpcInfo(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength, const ET9U8 bSpcMode) { const ET9U8 bWordSrc = pWord->Body.bWordSrc; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AssertLog(pWord->Base.wWordLen == wLength || pLingCmnInfo->Private.bSpcDuringBuild || pWord->Body.bEditDistFree || pLingCmnInfo->Private.bTraceBuild || pLingCmnInfo->Private.bExpandAsDuringBuild); /* valid? */ if (!wLength) { WLOG2(fprintf(pLogFile2, "__UpdateSpcInfo, empty\n");) } /* make source not be LDB - upsets the edit distance calc in this case (asserts) */ pWord->Body.bWordSrc = ET9WORDSRC_NONE; /* calculate distance, if it fails, assign a big default */ if (_ET9AWCalcEditDistanceInit(pLingInfo, wIndex, wLength, bSpcMode, pLingCmnInfo->Private.ASpc.bSpcFeatures) || __CalcEditDistance(pLingInfo, pWord, wIndex, wLength)) { pWord->Body.xTapFreq = 1; pWord->Body.xWordFreq = 1; pWord->Body.dwWordIndex = 0; pWord->Body.bEditDistSpc = 50; pWord->Body.bEditDistStem = 50; pWord->Body.bEditDistFree = 50; } _ET9AWCalcEditDistanceDone(pLingInfo); if (pWord->Base.wWordCompLen > pWord->Base.wWordLen) { pWord->Base.wWordCompLen = 0; } /* restore word source */ pWord->Body.bWordSrc = bWordSrc; /* if spc never been on, make sure it doesn't come out this way... (stem dist is ok) */ if (!pLingCmnInfo->Private.bSpcDuringBuild) { pWord->Body.bEditDistSpc = 0; } /* verify */ __VerifyLockedSymbs(pLingInfo, pWord, wIndex, wLength); /* done */ WLOG2Word(pLogFile2, "__UpdateSpcInfo", pWord); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9BOOL ET9LOCALCALL __LeftHandWordFromPrevDefaultWord(ET9AWLingInfo * const pLingInfo, const ET9U8 bNumSymbs, ET9AWPrivWordInfo * const pLeftHandWord, ET9AWPrivWordInfo * const pPrevDefaultWord) { const ET9U16 wDefaultLen = __CaptureGetDefaultStringLength(pLingInfo, bNumSymbs); ET9AssertLog(pLeftHandWord != NULL); ET9AssertLog(pPrevDefaultWord != NULL); if (pPrevDefaultWord->Base.wWordLen < wDefaultLen) { return 0; } *pLeftHandWord = *pPrevDefaultWord; if (pLeftHandWord->Base.wWordLen > wDefaultLen) { const ET9U16 wLenDiff = pLeftHandWord->Base.wWordLen - wDefaultLen; if (wLenDiff >= pLeftHandWord->Base.wWordCompLen) { pLeftHandWord->Base.wWordCompLen = 0; } else { pLeftHandWord->Base.wWordCompLen = (ET9U16)(pLeftHandWord->Base.wWordCompLen - wLenDiff); } pLeftHandWord->Base.wWordLen = wDefaultLen; } pLeftHandWord->Body.bEditDistSpc = 0; pLeftHandWord->Body.bEditDistStem = 0; return 1; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DoBuildSubstitutionProcessing(ET9AWLingInfo * const pLingInfo) { ET9UINT nLook; ET9UINT nIndex; ET9AWPrivWordInfo *pWord1 = 0; /* keep compiler happy */ ET9AWPrivWordInfo *pWord2 = 0; /* keep compiler happy */ ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AssertLog(pLingInfo != NULL); /* keep these as separate issues (when applicable) for clarity (not a speed problem) */ WLOG2(fprintf(pLogFile2, "SP - start\n");) /* tag all shortcuts with substitutions that also are words in the list */ for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (!pWord1->Base.wSubstitutionLen) { continue; } /* look for a candidate that is the substitution */ for (nLook = 0; nLook < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nLook) { if (nLook == nIndex) { continue; } pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLook]]; if (pWord2->Body.bWordQuality < LIMITED_QUALITY) { continue; } if (pWord1->Base.wSubstitutionLen == pWord2->Base.wWordLen) { /* keep declarations here */ ET9U16 wCmpLen = pWord1->Base.wSubstitutionLen; ET9SYMB *pSrc = pWord1->Base.sSubstitution; ET9SYMB *pDest = pWord2->Base.sWord; ++wCmpLen; while (--wCmpLen) { if (*pSrc++ != *pDest++) { break; } } if (!wCmpLen) { WLOG2Word(pLogFile2, "SP - shortcut", pWord1); WLOG2Word(pLogFile2, "SP - word", pWord2); /* mark as a substitution duplicate depending on quality */ if (pWord1->Body.bWordQuality <= LIMITED_QUALITY) { WLOG2(fprintf(pLogFile2, "SP - limited quality shortcut, slating for dupe removal\n");) __WordOrder_UpdateOut(pLingCmnInfo, pWord1, 0, 1); pWord1->Body.bWordQuality = DUPE_QUALITY; __WordOrder_UpdateIn(pLingCmnInfo, pWord1, 0, 1); } else { WLOG2(fprintf(pLogFile2, "SP - quality shortcut, will keep its position\n");) } break; } } } } /* done */ WLOG2(fprintf(pLogFile2, "SP - done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DoDefaultSpaceCharProcessing(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9UINT nFirstIndex = pLingCmnInfo->Private.nDefaultIndex; ET9U8 bFound = 0; ET9UINT nPromoteIndex = nFirstIndex; if (GETRAWSRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nPromoteIndex]].Body.bWordSrc) == ET9WORDSRC_AUTOSPACE) { WLOG2(fprintf(pLogFile2, "PP - defaulting, already in top\n");) bFound = 1; } for (; !bFound && nPromoteIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nPromoteIndex) { ET9AWPrivWordInfo * const pThisWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nPromoteIndex]]; if (GETRAWSRC(pThisWord->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE) { WLOG2(fprintf(pLogFile2, "PP - defaulting, %d moves up to %d\n", nPromoteIndex, nFirstIndex);) __ShiftSubListDown(pLingCmnInfo, nFirstIndex, nPromoteIndex - 1); __LogPartialSelList(pLingInfo->pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "PP - after defaulting op", pLogFile2); bFound = 1; break; } } if (bFound) { ET9UINT nIndex; for (nIndex = nPromoteIndex + 1; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pThisWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (GETRAWSRC(pThisWord->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE) { WLOG2(fprintf(pLogFile2, "PP - defaulting, removing %d\n", nIndex);) __RemoveWord(pLingCmnInfo, nIndex); --nIndex; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DoBuildPreDefaultPostProcessing(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; /* anything to do? */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } /* keep these as separate issues (when applicable) for clarity (not a speed problem) */ /* words that start with initial non-letter (punctuation/symbols/numbers) should be shown in the list after any single character words (first key press) */ if (pWordSymbInfo->bNumSymbs == 1) { ET9UINT nLastLenOne = 0; ET9UINT nIndex; for (nIndex = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1; nIndex > 0; --nIndex) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; /* remember first len-one found (last one) */ if (pWord->Base.wWordLen == 1 && nLastLenOne < nIndex) { nLastLenOne = nIndex; } /* potentially move down word */ if (nLastLenOne > nIndex && __HasInitialNonLetter(pWord->Base.sWord, pWord->Base.wWordLen)) { WLOG2(fprintf(pLogFile2, "PDPP - moving down non-inital-letter word, %u -> %u\n", nIndex + 1, nLastLenOne);) __ShiftSubListUp(pLingCmnInfo, nIndex + 1, nLastLenOne); --nLastLenOne; /* pushed up one step by the shift */ /* no need to revisit index since the moved up word already is tested */ } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DoBuildPostProcessing(ET9AWLingInfo * const pLingInfo) { ET9UINT nLook; ET9UINT nIndex; ET9AWPrivWordInfo *pWord1 = 0; /* keep compiler happy */ ET9AWPrivWordInfo *pWord2 = 0; /* keep compiler happy */ ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; __ProfileStart; ET9AssertLog(pLingInfo != NULL); /* anything to do? */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } /* keep these as separate issues (when applicable) for clarity (not a speed problem) */ /* remove gray/kill words */ { const ET9BOOL bAllowBestHiddenTrace = pLingCmnInfo->Private.bTraceBuild; ET9UINT nCount; for (nCount = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; nCount; --nCount) { const ET9UINT nWordIndex = nCount - 1; ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nWordIndex]]; if (pWord->Body.bIsBlack) { if (ISEXACTSRC(pWord->Body.bWordSrc)) { WLOG2Word(pLogFile2, "PP - black exact loses source", pWord); pWord->Body.bWordSrc = EXACTOFFSET; pWord->Body.bWordQuality = SHAPED_QUALITY; } else { WLOG2Word(pLogFile2, "PP - removing black", pWord); __RemoveWord(pLingCmnInfo, nWordIndex); } } else if (pWord->Body.bIsGray && !ISEXACTSRC(pWord->Body.bWordSrc) && !ISEXACTISHSRC(pWord->Body.bWordSrc)) { WLOG2Word(pLogFile2, "PP - removing gray word", pWord); __RemoveWord(pLingCmnInfo, nWordIndex); } else if (pWord->Body.bIsHidden && !ISEXACTSRC(pWord->Body.bWordSrc) && !ISEXACTISHSRC(pWord->Body.bWordSrc)) { if (bAllowBestHiddenTrace && !nWordIndex) { WLOG2Word(pLogFile2, "PP - best hidden stays", pWord); } else { WLOG2Word(pLogFile2, "PP - removing hidden word", pWord); __RemoveWord(pLingCmnInfo, nWordIndex); } } else if (pWord->Body.bIsSegment && nWordIndex > 1) { WLOG2Word(pLogFile2, "PP - removing segment", pWord); __RemoveWord(pLingCmnInfo, nWordIndex); } else if (pWord->Body.wGestureReturnVal) { WLOG2Word(pLogFile2, "PP - removing custom gesture", pWord); __RemoveWord(pLingCmnInfo, nWordIndex); } } if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } } /* currently no post processing on NWP lists */ if (!pWordSymbInfo->bNumSymbs) { if (ET9POSTSORTENABLED(pLingCmnInfo)) { WLOG2(fprintf(pLogFile2, "PP - only 'post sort' action (NWP)\n");) __ET9AWPerformPostSort(pLingInfo, pWordSymbInfo); } else { WLOG2(fprintf(pLogFile2, "PP - no action (NWP)\n");) } return; } /* log unmodified list */ __LogPartialSelList(pLingInfo->pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "PP - before post processing", pLogFile2); /* promote term over completion * if the default word is a completion a term with equal or less correction will be brought up */ if (__IsUsingMixedStyle(pLingCmnInfo)) { const ET9UINT nFirstIndex = pLingCmnInfo->Private.nDefaultIndex; ET9AWPrivWordInfo * const pFirstWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nFirstIndex]]; if (pFirstWord->Base.wWordCompLen) { const ET9BOOL bTraceBuild = pLingCmnInfo->Private.bTraceBuild; ET9UINT nPromoteIndex; WLOG2(fprintf(pLogFile2, "PP - mixed with default completion - will try to promote term, bTraceBuild %u\n", bTraceBuild);) for (nPromoteIndex = nFirstIndex + 1; nPromoteIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nPromoteIndex) { ET9AWPrivWordInfo * const pThisWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nPromoteIndex]]; if (pThisWord->Base.wWordCompLen) { continue; } if (!ISREALSRC(pThisWord->Body.bWordSrc) || pThisWord->Body.bIsBlack) { continue; } if (bTraceBuild) { break; } if ((pThisWord->Body.bEditDistSpc > pFirstWord->Body.bEditDistSpc) || (pThisWord->Body.bEditDistSpc == pFirstWord->Body.bEditDistSpc && pThisWord->Body.bEditDistStem > pFirstWord->Body.bEditDistStem) || (pThisWord->Body.bEditDistStem == pFirstWord->Body.bEditDistStem && pThisWord->Body.bEditDistFree > pFirstWord->Body.bEditDistFree)) { continue; } break; } if (nPromoteIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { WLOG2(fprintf(pLogFile2, "PP - no term to promote\n");) } else { WLOG2(fprintf(pLogFile2, "PP - promoting term, %d moves up to %d\n", nPromoteIndex, nFirstIndex);) __ShiftSubListDown(pLingCmnInfo, nFirstIndex, nPromoteIndex - 1); __LogPartialSelList(pLingInfo->pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "PP - after promote term op", pLogFile2); } } } /* demote candidates * never demote index zero (really the exact entry as NONE source) * don't demote (non "demote src") spc unless what you pull up has less stem distance * "demote src" demote against <= stem distance * switch term/stem aborts (one chunk only - all list types) */ if (!__IsUsingMixedStyle(pLingCmnInfo)) { ET9UINT nFirst; /* keep here */ ET9UINT nLast; /* keep here */ ET9UINT nDemoteFirst; /* keep here */ ET9UINT nNonDemoteFirst; /* keep here */ ET9U8 bLengthIndex; /* keep here */ ET9BOOL bHasFlush; /* keep here */ /* first find the start of the chunk */ for (nFirst = 0; nFirst < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nFirst) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nFirst]]; if (ISEXACTSRC(pWord1->Body.bWordSrc) || GETBASESRC(pWord1->Body.bWordSrc) == ET9WORDSRC_NONE) { continue; } if (ISREALSRC(pWord1->Body.bWordSrc)) { break; } } /* then find the chunk end */ for (nLast = nFirst, nIndex = nFirst + 1; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pWord1->Base.bIsTerm != pWord2->Base.bIsTerm || !ISREALSRC(pWord2->Body.bWordSrc)) { break; } nLast = nIndex; } /* chunk must be at least size 2 to do anything */ if (nFirst < nLast) { WLOG2(fprintf(pLogFile2, "PP - found chunk for demote (%d -> %d)\n", nFirst, nLast);) __LogPartialSelList(pLingInfo->pLingCmnInfo, nFirst, nLast, "PP - chunk before demote", pLogFile2); /* Find if flush happened */ bHasFlush = 0; for (bLengthIndex = 0; bLengthIndex < pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs - 1; ++bLengthIndex) { if (pLingCmnInfo->Private.sBuildInfo.pbFlushLen[bLengthIndex]) { bHasFlush = 1; break; } } /* first demote the demote sources */ WLOG2(fprintf(pLogFile2, "PP - demote sources\n");) for (nDemoteFirst = nFirst; (nDemoteFirst - nFirst) < WORD_SOURCE_DEMOTE_FENCE && nDemoteFirst <= nLast; ++nDemoteFirst) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nDemoteFirst]]; /* must be a demotable source */ if (!ISDEMOTEWRD(pWord1)) { continue; } /* LAS only when not also a real word */ if ((GETRAWSRC(pWord1->Body.bWordSrc) == ET9WORDSRC_LAS_SHORTCUT && pWord1->Body.bWordQuality >= GENUINE_QUALITY) || GETRAWSRC(pWord1->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE) { continue; } /* look for something to promote */ for (nNonDemoteFirst = nDemoteFirst + 1; nNonDemoteFirst <= nLast; ++nNonDemoteFirst) { pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nNonDemoteFirst]]; if ((ISDEMOTEWRD(pWord2) && GETRAWSRC(pWord2->Body.bWordSrc) != ET9WORDSRC_AUTOSPACE) || pWord2->Body.bEditDistStem > pWord1->Body.bEditDistStem) { continue; } break; } if (nNonDemoteFirst > nLast) { WLOG2(fprintf(pLogFile2, "PP - nothing to promote (a)\n");) continue; } WLOG2(fprintf(pLogFile2, "PP - demoting (a), %d moves up to %d\n", nNonDemoteFirst, nDemoteFirst);) __ShiftSubListDown(pLingCmnInfo, nDemoteFirst, nNonDemoteFirst - 1); __LogPartialSelList(pLingInfo->pLingCmnInfo, nFirst, nLast, "PP - after demote op", pLogFile2); } /* then demote spc */ WLOG2(fprintf(pLogFile2, "PP - demote spc\n");) for (nDemoteFirst = nFirst; (nDemoteFirst - nFirst) < WORD_SOURCE_DEMOTE_FENCE && nDemoteFirst <= nLast; ++nDemoteFirst) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nDemoteFirst]]; if (!pWord1->Body.bHasPrimEditDist) { continue; } for (nNonDemoteFirst = nDemoteFirst + 1; nNonDemoteFirst <= nLast; ++nNonDemoteFirst) { pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nNonDemoteFirst]]; WLOG2B(fprintf(pLogFile2, " [%2u] ed %u, sd %u, src %u, bIsQuarantine %u, bIsWeak %u, demote %u, buildable %u\n", pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nNonDemoteFirst], pWord2->Body.bHasPrimEditDist, pWord2->Body.bEditDistStem, pWord2->Body.bWordSrc, pWord2->Body.bIsQuarantine, pWord2->Body.bIsWeak, ISDEMOTEWRD(pWord2), ISBUILDABLESRC(pWord2->Body.bWordSrc));) if (pWord2->Body.bHasPrimEditDist || pWord2->Body.bEditDistStem >= pWord1->Body.bEditDistStem || ISDEMOTEWRD(pWord2) || (!bHasFlush && !ISGENUINESRC(pWord2->Body.bWordSrc) && ISGENUINESRC(pWord1->Body.bWordSrc))) { continue; } break; } if (nNonDemoteFirst > nLast) { WLOG2(fprintf(pLogFile2, "PP - nothing to promote (b)\n");) continue; } WLOG2(fprintf(pLogFile2, "PP - demoting (b), %d moves up to %d\n", nNonDemoteFirst, nDemoteFirst);) __ShiftSubListDown(pLingCmnInfo, nDemoteFirst, nNonDemoteFirst - 1); __LogPartialSelList(pLingInfo->pLingCmnInfo, nFirst, nLast, "PP - after demote op", pLogFile2); __LogPartialSelList(pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "PP - after demote op", pLogFile2); } } } /* demote substitutions preceeding the substitution as candidate (really - pull up the spc) * if a substitution preceeds (any distance) an equal candidate then move index or move the candidate up (depending on default) * also remove substitution duplicates (two shortcuts having the same substitution) */ WLOG2(fprintf(pLogFile2, "PP - demote substitutions\n");) for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9BOOL bRepeatIndex = 0; pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (!pWord1->Base.wSubstitutionLen || pWord1->Body.bWordQuality <= DISPOSABLE_QUALITY) { continue; } WLOG2Word(pLogFile2, "PP - investigating (s vs w)", pWord1); /* look for a candidate that is the substitution */ for (nLook = 0; nLook < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nLook) { if (nLook == nIndex) { continue; } pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLook]]; if (pWord2->Body.bWordQuality < LIMITED_QUALITY) { continue; } /* check word1 subst against word2 word*/ if (pWord1->Base.wSubstitutionLen == pWord2->Base.wWordLen) { /* keep declarations here */ ET9U16 wCmpLen = pWord1->Base.wSubstitutionLen; ET9SYMB *pSrc = pWord1->Base.sSubstitution; ET9SYMB *pDest = pWord2->Base.sWord; ++wCmpLen; while (--wCmpLen) { if (*pSrc++ != *pDest++) { break; } } if (!wCmpLen) { WLOG2Word(pLogFile2, "PP - dupe with", pWord2); if (!nIndex && !pLingCmnInfo->Private.nDefaultIndex && (ISEXACTSRC(pWord1->Body.bWordSrc) || GETBASESRC(pWord1->Body.bWordSrc) == ET9WORDSRC_NONE)) { WLOG2(fprintf(pLogFile2, "PP - substitution preceeds candidate - moving default index up\n");) ++pLingCmnInfo->Private.nDefaultIndex; if (nLook > nIndex + 1) { WLOG2(fprintf(pLogFile2, "PP - substitution preceeds candidate - also moving down %d-%d\n", nIndex+1, nLook-1);) __ShiftSubListDown(pLingCmnInfo, nIndex + 1, nLook - 1); } } else if (nIndex > nLook) { WLOG2(fprintf(pLogFile2, "PP - candidate preceeds substitution - nothing moved\n");) } else { WLOG2(fprintf(pLogFile2, "PP - substitution preceeds candidate - moving down %d-%d\n", nIndex, nLook-1);) if (pWord2->Base.wSubstitutionLen) { bRepeatIndex = 1; } __ShiftSubListDown(pLingCmnInfo, nIndex, nLook - 1); if (pLingCmnInfo->Private.nDefaultIndex > nIndex && pLingCmnInfo->Private.nDefaultIndex <= nLook) { WLOG2(fprintf(pLogFile2, "PP - substitution preceeds candidate - also modifying default index %u -> %u\n", pLingCmnInfo->Private.nDefaultIndex, nIndex);) pLingCmnInfo->Private.nDefaultIndex = nIndex; } } if (pWord1->Body.bWordQuality <= LIMITED_QUALITY) { ET9AssertLog(!(ISEXACTSRC(pWord1->Body.bWordSrc) || GETBASESRC(pWord1->Body.bWordSrc) == ET9WORDSRC_NONE)); WLOG2(fprintf(pLogFile2, "PP - low quality shortcut - moving to stem pool (for removal)\n");) pWord1->Body.bWordSrc = ET9WORDSRC_STEMPOOL; pWord1->Body.bWordQuality = DISPOSABLE_QUALITY; } pWord1->Base.wSubstitutionLen = 0; __LogPartialSelList(pLingInfo->pLingCmnInfo, nIndex, nLook, "PP - after preecede op", pLogFile2); break; } } } /* for words 2 */ /* might have to repeat after being used here all dupe quality words are history */ if (bRepeatIndex) { WLOG2(fprintf(pLogFile2, "PP - about to repeat index (%d)\n", bRepeatIndex);) --nIndex; } else if (pWord1->Body.bWordQuality == DUPE_QUALITY) { ET9AssertLog(!(ISEXACTSRC(pWord1->Body.bWordSrc) || GETBASESRC(pWord1->Body.bWordSrc) == ET9WORDSRC_NONE)); WLOG2Word(pLogFile2, "PP - dupe quality shortcut - moving to stem pool (for removal)", pWord1); pWord1->Body.bWordSrc = ET9WORDSRC_STEMPOOL; pWord1->Body.bWordQuality = DISPOSABLE_QUALITY; } } /* for words 1 */ WLOG2(fprintf(pLogFile2, "PP - remove substitution dupes\n");) for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (!pWord1->Base.wSubstitutionLen || pWord1->Body.bWordQuality <= DISPOSABLE_QUALITY || pWord1->Body.bWordSrc == ET9WORDSRC_STEMPOOL) { continue; } WLOG2Word(pLogFile2, "PP - investigating (s vs s)", pWord1); /* look for a candidate that is the substitution */ for (nLook = 0; nLook < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nLook) { if (nLook == nIndex) { continue; } pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLook]]; if (pWord2->Body.bWordQuality < LIMITED_QUALITY) { continue; } /* check word1 subst against word2 subst (will never happen if the above happened) */ if (pWord1->Base.wSubstitutionLen == pWord2->Base.wSubstitutionLen) { /* keep declarations here */ ET9U16 wCmpLen = pWord1->Base.wSubstitutionLen; ET9SYMB *pSrc = pWord1->Base.sSubstitution; ET9SYMB *pDest = pWord2->Base.sSubstitution; ++wCmpLen; while (--wCmpLen) { if (*pSrc++ != *pDest++) { break; } } if (!wCmpLen) { WLOG2Word(pLogFile2, "PP - dupe with", pWord2); if (pWord2->Body.bWordQuality <= LIMITED_QUALITY) { ET9AssertLog(!(ISEXACTSRC(pWord2->Body.bWordSrc) || GETBASESRC(pWord2->Body.bWordSrc) == ET9WORDSRC_NONE)); WLOG2(fprintf(pLogFile2, "PP - low quality shortcut - moving 2nd to stem pool (for removal)\n");) pWord2->Body.bWordSrc = ET9WORDSRC_STEMPOOL; pWord2->Body.bWordQuality = DISPOSABLE_QUALITY; } pWord2->Base.wSubstitutionLen = 0; __LogPartialSelList(pLingInfo->pLingCmnInfo, nIndex, nLook, "PP - after subst dupe", pLogFile2); } } } /* for words 2 */ } /* for words 1 */ #if 0 /* avoid having a completion punct as default */ for (bIndex = pWordSymbInfo->Private.nDefaultIndex; bIndex > 0; --bIndex) { pWord1 = &pAlpSelList->pWordList[pAlpSelList->Private.sWordC.pCurrC->pnWordList[bIndex]]; if (GETBASESRC(pWord1->Body.bWordSrc) == ET9WORDSRC_COMPLETIONPUNCT) { --pWordSymbInfo->Private.nDefaultIndex; WLOG2(fprintf(pLogFile2, "PP - default is a completion punct, moving back one step (%d)\n", pWordSymbInfo->Private.nDefaultIndex);) } else { break; } } #endif /* clean up the stem pool - don't assume that it's consecutive this MUST be the last "operation" applied in the word list builder if duplicates are found the list will shrink, but since this is the last real operation on the list only the "sorting list" will be shrinking and thus possibly pointing to a word that is outside the new list size also no attempt is made to update any other internal info used when building the list stem pool entries that are spc will be removed no matter what (except when propagating distance) entries with disposable quality will be removed no matter what (e.g. orphan RDB entries) a "tricky" thing here is that when a word is deleted it affects all loops, indexes etc */ WLOG2(fprintf(pLogFile2, "PP - clean up the stem pool\n");) ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); { ET9UINT nFirst; /* keep here */ ET9UINT nLast; /* keep here */ for (nFirst = 0; nFirst < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nFirst) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nFirst]]; if (pWord1->Body.bWordSrc == ET9WORDSRC_STEMPOOL) { break; } } for (nLast = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1; nLast > nFirst; --nLast) { pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLast]]; if (pWord2->Body.bWordSrc == ET9WORDSRC_STEMPOOL) { break; } } if (nFirst <= nLast) { ET9UINT nPoolCount; ET9BOOL bDisposableOnly; WLOG2(fprintf(pLogFile2, "PP - found stem pool (%d-%d)\n", nFirst, nLast);) /* remove word stems and substitutions */ nPoolCount = 0; for (nIndex = nFirst; nIndex <= nLast; ++nIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pWord1->Body.bWordSrc == ET9WORDSRC_STEMPOOL) { if (pWord1->Base.wWordCompLen) { WLOG2Word(pLogFile2, "PP - removing completion", pWord1); } pWord1->Base.wWordLen = (ET9U16)(pWord1->Base.wWordLen - pWord1->Base.wWordCompLen); pWord1->Base.wWordCompLen = 0; pWord1->Base.wSubstitutionLen = 0; ++nPoolCount; } } /* check if disposable only, if so they all stay... (not dups) */ bDisposableOnly = 1; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pWord1->Body.bWordSrc != ET9WORDSRC_STEMPOOL || (!pWord1->Body.bHasPrimEditDist && pWord1->Body.bWordQuality > DISPOSABLE_QUALITY)) { bDisposableOnly = 0; break; } } WLOG2(fprintf(pLogFile2, "PP - disposable only = %d\n", bDisposableOnly);) /* remove resulting duplicates */ for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords && nPoolCount; ++nIndex) { WLOG2(fprintf(pLogFile2, "PP - checking word @ %02d (pool count = %2d)\n", nIndex, nPoolCount);) pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; for (nLook = nFirst; nLook <= nLast && nPoolCount; ++nLook) { ET9BOOL bRemove = 0; /* keep here */ pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLook]]; if (pWord2->Body.bWordSrc != ET9WORDSRC_STEMPOOL) { continue; } else if (pWord2->Body.bHasPrimEditDist) { bRemove = !bDisposableOnly; } else if (pWord2->Body.bWordQuality <= DISPOSABLE_QUALITY) { bRemove = !bDisposableOnly; } else if (nLook == nIndex) { continue; } if (!bRemove && nLook != nIndex && pWord1->Base.wWordLen == pWord2->Base.wWordLen) { /* keep declarations here */ ET9U16 wCmpLen = pWord1->Base.wWordLen; ET9SYMB *pSrc = pWord1->Base.sWord; ET9SYMB *pDest = pWord2->Base.sWord; ++wCmpLen; while (--wCmpLen) { if (*pSrc++ != *pDest++) { break; } } if (!wCmpLen) { bRemove = 1; } } if (bRemove) { WLOG2Word(pLogFile2, "PP - removing", pWord2); __RemoveWord(pLingCmnInfo, nLook); --nPoolCount; if (nIndex > nLook) { --nIndex; /* the list shrunk above this index, pWord1 is ok */ } --nLook; /* need to revisit this position */ --nLast; /* the stem pool has shrunk */ } } } WLOG2(fprintf(pLogFile2, "PP - %d word(s) remain in the stem pool\n", nPoolCount);) } } /* if the build is shrinking all substitutions should be taken out - do it last so that the list appears exactly the same, but without substitutions */ if (pLingCmnInfo->Private.bLastBuildShrinking) { for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pWord1->Base.wSubstitutionLen) { WLOG2Word(pLogFile2, "PP - shrinking build, removing substitution", pWord1); pWord1->Base.wSubstitutionLen = 0; } } } if (ET9POSTSORTENABLED(pLingCmnInfo)) { __ET9AWPerformPostSort(pLingInfo, pWordSymbInfo); } /* done */ __ProfileEnd(tAW_SelLst_DoBuildPostProcessing); WLOG2(fprintf(pLogFile2, "PP - done\n\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9BOOL ET9LOCALCALL __HasMatchingLanguage(ET9AWPrivWordInfo const * const pWord1, ET9AWPrivWordInfo const * const pWord2) { ET9AssertLog(pWord1); ET9AssertLog(pWord2); if (pWord1->Base.bLangIndex == pWord2->Base.bLangIndex) { return 1; } if (pWord1->Base.bLangIndex == ET9AWBOTH_LANGUAGES || pWord2->Base.bLangIndex == ET9AWBOTH_LANGUAGES) { return 1; } if (pWord1->Base.bLangIndex == ET9AWUNKNOWN_LANGUAGE || pWord2->Base.bLangIndex == ET9AWUNKNOWN_LANGUAGE) { return 1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __PromoteCompletion(ET9AWLingInfo * const pLingInfo, const ET9UINT nTargetPos) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9BOOL bLastInputIsTrace = (ET9BOOL)(pWordSymbInfo->bNumSymbs && pWordSymbInfo->SymbsInfo[pWordSymbInfo->bNumSymbs - 1].bTraceIndex); const ET9BOOL bLastInputIsSmartPunct = (ET9BOOL)(pWordSymbInfo->bNumSymbs && pWordSymbInfo->SymbsInfo[pWordSymbInfo->bNumSymbs - 1].bSymbType == ET9KTSMARTPUNCT); if (!(pLingCmnInfo->Private.bUsingALM || pLingCmnInfo->Private.bUsingDLM) || pLingCmnInfo->Private.eSelectionListCorrectionMode == ET9ASLCORRECTIONMODE_LOW || bLastInputIsTrace || bLastInputIsSmartPunct || pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= nTargetPos || _ET9HasDiscreteOnlyInfo(pWordSymbInfo)) { WLOG2(fprintf(pLogFile2, "FP %u - not moving up completion - wrong mode\n", nTargetPos);) return; } { ET9AWPrivWordInfo const * const pTargetWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nTargetPos - 1]]; if (pLingCmnInfo->Private.wCurrLockPoint || pTargetWord->Base.wWordCompLen || ISEXACTISHSRC(pTargetWord->Body.bWordSrc) || !_ET9_LanguageSpecific_ApplyCompletionRules(pLingInfo, pTargetWord)) { WLOG2(fprintf(pLogFile2, "FP %u - not moving up completion - untouchable word\n", nTargetPos);) return; } WLOG2(fprintf(pLogFile2, "FP %u - found non completion\n", nTargetPos);) { ET9UINT nIndex; for (nIndex = nTargetPos; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (!ISGENUINESRC(pIndexWord->Body.bWordSrc) || ISNONOVERRIDEWRD(pIndexWord) || pIndexWord->Body.bWordQuality != GENUINE_QUALITY || !(GETRAWSRC(pIndexWord->Body.bWordSrc) == ET9WORDSRC_LDB || pIndexWord->Body.dwWordIndex > 0) || /* subject to change with adaptive user dict */ !pIndexWord->Base.wWordCompLen || pIndexWord->Body.bIsAcronym || (!pIndexWord->Body.bIsTop5 && pTargetWord->Body.dwWordIndex < 100) || ((pIndexWord->Body.bEditDistStem > pTargetWord->Body.bEditDistStem) && pIndexWord->Body.bNLMOrder < 3) || !__HasMatchingLanguage(pTargetWord, pIndexWord)) { continue; } break; } if (nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { WLOG2(fprintf(pLogFile2, "FP %u - found promotable word @ %u, moving the promotable word up\n", nTargetPos, nIndex);) __ShiftSubListDown(pLingCmnInfo, (nTargetPos - 1), (nIndex - 1)); } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __PromoteSecondary(ET9AWLingInfo * const pLingInfo, const ET9UINT nTargetPos) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U8 bNonActiveIndex = pLingCmnInfo->Private.bNonActiveIndex; /* applicable? */ if (!ET9AW_GetBilingualSupported(pLingInfo) || !pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs || pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode != ET9ASLMODE_MIXED || pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= nTargetPos) { WLOG2(fprintf(pLogFile2, "FP %u - not moving up secondary - wrong mode\n", nTargetPos);) return; } /* already ok? */ { ET9UINT nIndex; for (nIndex = 0; nIndex < nTargetPos; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pIndexWord->Base.bLangIndex == bNonActiveIndex) { WLOG2(fprintf(pLogFile2, "FP %u - not moving up secondary - already ok\n", nTargetPos);) return; } } } /* move up "exactish" */ { ET9UINT nIndex; for (nIndex = nTargetPos; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pIndexWord->Base.bLangIndex != bNonActiveIndex) { continue; } if (__ET9AWIsLikeExactMatch(pLingInfo, pIndexWord, 1)) { break; } } if (nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { WLOG2(fprintf(pLogFile2, "FP %u - found promotable exactish @ %u, moving the promotable word up\n", nTargetPos, nIndex);) __ShiftSubListDown(pLingCmnInfo, (nTargetPos - 1), (nIndex - 1)); return; } } /* move up first found secondary */ { ET9UINT nIndex; for (nIndex = nTargetPos; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pIndexWord->Base.bLangIndex == bNonActiveIndex) { break; } } if (nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { WLOG2(fprintf(pLogFile2, "FP %u - found promotable word @ %u, moving the promotable word up\n", nTargetPos, nIndex);) __ShiftSubListDown(pLingCmnInfo, (nTargetPos - 1), (nIndex - 1)); return; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DemoteAcronym(ET9AWLingInfo * const pLingInfo, const ET9UINT nTargetPos) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo const * const pTargetWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nTargetPos]]; if (pLingCmnInfo->Base.pWordSymbInfo->Private.bHasAllCapsGesture) { WLOG2(fprintf(pLogFile2, "FP %u - Caps-Gesture - not handling acronyms\n", nTargetPos);) return; } if (pLingCmnInfo->Private.wCurrLockPoint || !pTargetWord->Body.bIsAcronym || pTargetWord->Body.bNLMOrder > 2 || ISEXACTSRC(pTargetWord->Body.bWordSrc) || !(pTargetWord->Body.bEditDistSpc || pTargetWord->Body.bEditDistStem || pTargetWord->Body.bEditDistFree) || !_ET9_LanguageSpecific_ApplyAcronymRules(pLingInfo, pTargetWord)) { WLOG2(fprintf(pLogFile2, "FP %u - didn't find suitable acronym\n", nTargetPos);) return; } WLOG2(fprintf(pLogFile2, "FP %u - found suitable acronym\n", nTargetPos);) { ET9UINT nIndex; for (nIndex = nTargetPos + 1; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (!ISGENUINESRC(pIndexWord->Body.bWordSrc) || ISNONOVERRIDEWRD(pIndexWord) || pIndexWord->Body.bWordQuality != GENUINE_QUALITY || pIndexWord->Body.bIsAcronym || pIndexWord->Body.bIsCapitalized) { continue; } break; } if (nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { WLOG2(fprintf(pLogFile2, "FP %u - found promotable word @ %u, moving the promotable word up\n", nTargetPos, nIndex);) __ShiftSubListDown(pLingCmnInfo, nTargetPos, (nIndex - 1)); } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __DemoteWeakDefault(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9UINT nTargetPos = pLingCmnInfo->Private.nDefaultIndex; ET9AWPrivWordInfo const * const pTargetWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nTargetPos]]; if (pLingCmnInfo->Private.wCurrLockPoint || !pTargetWord->Body.bIsWeak) { WLOG2(fprintf(pLogFile2, "WP %u - didn't find suitable weak\n", nTargetPos);) return; } WLOG2(fprintf(pLogFile2, "WP %u - found suitable weak\n", nTargetPos);) { ET9UINT nIndex; for (nIndex = nTargetPos + 1; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (ISNONOVERRIDEWRD(pIndexWord)) { continue; } break; } if (nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { if (!nTargetPos && ISEXACTSRC(pTargetWord->Body.bWordSrc)) { if (nIndex > 1) { WLOG2(fprintf(pLogFile2, "WP %u - found promotable word @ %u, but moving the promotable word up to 1 since exact is default\n", nTargetPos, nIndex);) __ShiftSubListDown(pLingCmnInfo, 1, (nIndex - 1)); } WLOG2(fprintf(pLogFile2, "WP %u - found promotable word @ %u, changing default from 0 to 1\n", nTargetPos, nIndex);) pLingCmnInfo->Private.nDefaultIndex = 1; } else { WLOG2(fprintf(pLogFile2, "WP %u - found promotable word @ %u, moving the promotable word up\n", nTargetPos, nIndex);) __ShiftSubListDown(pLingCmnInfo, nTargetPos, (nIndex - 1)); } } else if (nTargetPos) { ET9AWPrivWordInfo const * const pZeroWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[0]]; if (!pZeroWord->Body.bIsWeak) { WLOG2(fprintf(pLogFile2, "WP %u - no promotable word found, moving default to non weak 0 pos\n", nTargetPos);) pLingCmnInfo->Private.nDefaultIndex = 0; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9BOOL ET9LOCALCALL __IsCrowdingWord(ET9AWPrivWordInfo const * const pWord) { if (pWord->Body.dwWordIndex) { return 0; } if (pWord->Body.bIsWeak || pWord->Body.bIsHidden || pWord->Body.bIsQuarantine || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_CDB || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_MDB || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_QUDB || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_ASDB_SHORTCUT || (GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_RUDB && pWord->Body.bIsUDBWord) || (GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_DLM && !pWord->Body.bNLMOrder && !pWord->Body.dwWordIndex)) { return 1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __HandleTopCrowding(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9UINT nSegmentSize = 5; ET9UINT nIndex; ET9UINT nCrowdCount; if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= nSegmentSize) { return; } WLOG2(fprintf(pLogFile2, "__HandleTopCrowding\n");) nCrowdCount = 0; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { const ET9UINT nAllowedCrowdCount = (nIndex / nSegmentSize) + 1; ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (!__IsCrowdingWord(pIndexWord)) { continue; } WLOG2(fprintf(pLogFile2, " found crowding word @ %u\n", nIndex);) if (nCrowdCount < nAllowedCrowdCount) { ++nCrowdCount; WLOG2(fprintf(pLogFile2, " keeping word, nCrowdCount %u, nAllowedCrowdCount %u\n", nCrowdCount, nAllowedCrowdCount);) continue; } { ET9UINT nPromoteIndex; for (nPromoteIndex = nIndex + 1; nPromoteIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nPromoteIndex) { ET9AWPrivWordInfo const * const pPromoteIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nPromoteIndex]]; if (__IsCrowdingWord(pPromoteIndexWord) || ISPROTECTEDWRD(pPromoteIndexWord)) { continue; } break; } if (nPromoteIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { break; } WLOG2(fprintf(pLogFile2, " found non crowding word @ %u - moving it up\n", nPromoteIndex);) __ShiftSubListDown(pLingCmnInfo, nIndex, (nPromoteIndex - 1)); } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __AddSegmentedCandidate(ET9AWLingInfo * const pLingInfo, const ET9U8 bSpcMode, const ET9U16 wFlushPoint) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9SimpleWord sExactWordSimple; ET9AWPrivWordInfo sExactWord; const ET9UINT nExactIndex = pLingCmnInfo->Private.nExactIndex; __ProfileStart; ET9AssertLog(pLingInfo); ET9AssertLog(pLingCmnInfo); ET9AssertLog(pWordSymbInfo); /* anything to do? */ if (!ET9SPACESEGMENTATIONENABLED(pLingCmnInfo) || pWordSymbInfo->bNumSymbs < 3 || #if 0 (pWordSymbInfo->bNumSymbs > 13 && pLingInfo->pLingCmnInfo->Private.bStateNiceLatency) || #endif !pLingCmnInfo->Private.bHasRegionalInfo || pLingCmnInfo->Private.bTraceBuild || pLingCmnInfo->Private.wCurrLockPoint || _ET9_LanguageSpecific_IsNonSpaceLanguage(pLingCmnInfo) || ET9GetExactWord(pWordSymbInfo, &sExactWordSimple, pLingInfo->Private.pConvertSymb, pLingInfo->Private.pConvertSymbInfo)) { return; } /* set up globals */ pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; _InitPrivWordInfo(&pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord); /**/ _ET9SimpleWordToPrivWord(&sExactWordSimple, &sExactWord); if (wFlushPoint || !BUILDABLEEXACT(pLingCmnInfo->Private.sWordC.pCurrC->dwStateBits) || (nExactIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords && ISNONOVERRIDEWRD(&pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nExactIndex]]))) { ET9U8 bWordSrc; ET9U8 bMaxWordSrc = ET9WORDSRC_NONE; ET9U16 wSPt; ET9U16 wBestSPt = 0; ET9U16 wShiftPt = 0; ET9FREQPART xWordFreq; ET9FREQPART xMaxFreq = 0; ET9U8 bNLMOrder = 0; ET9U8 bMaxNLMOrder = 0; ET9BOOL bMaxExact = 0; ET9SimpleWord sBestLeftAmbigSegment; ET9SimpleWord sBestRightAmbigSegment; ET9SimpleWord sLeftWord = sExactWordSimple; const ET9U32 dwLdbNum = pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; _InitSimpleWord(&sBestLeftAmbigSegment); _InitSimpleWord(&sBestRightAmbigSegment); for (wSPt = 1; wSPt < sExactWordSimple.wLen; ++wSPt) { ET9SimpleWord sRightWord; ET9FREQPART xLMWordFreq = 0; ET9FREQPART xDLMWordFreq = 0; ET9U8 bLMOrder = 0; ET9U8 bDLMOrder = 0; ET9BOOL bIsDLMEntry; ET9BOOL bIsLMEntry; ET9U8 bLeftExact = 0; ET9U8 bRightExact = 0; ET9U8 bNumberLeftMatches; ET9U8 bNumberRightMatches; ET9U8 i; ET9U8 j; ET9U8 bLeftExactIndex; ET9U8 bRightExactIndex; ET9SimpleWord psLeftWord[ET9AW_MAX_AMBIG_SEGMENT_MATCHES]; ET9SimpleWord psRightWord[ET9AW_MAX_AMBIG_SEGMENT_MATCHES]; ET9U32 pdwIndexLeft[ET9AW_MAX_AMBIG_SEGMENT_MATCHES]; ET9U32 pdwIndexRight[ET9AW_MAX_AMBIG_SEGMENT_MATCHES]; ET9BOOL pbLeftMatchExact[ET9AW_MAX_AMBIG_SEGMENT_MATCHES] = {0}; ET9BOOL pbRightMatchExact[ET9AW_MAX_AMBIG_SEGMENT_MATCHES] = {0}; sLeftWord.wLen = wSPt; _ET9SymCopy(sRightWord.sString, &sExactWordSimple.sString[wSPt], sExactWordSimple.wLen - wSPt); sRightWord.wLen = sExactWordSimple.wLen - wSPt; /* Ambig segments? */ (void)_ET9AWLdbFindAmbigEntry(pLingInfo, dwLdbNum, sLeftWord.sString, sLeftWord.wLen, 0, (ET9SimpleWord *)psLeftWord, &bLeftExact, pdwIndexLeft, &bNumberLeftMatches, &bLeftExactIndex); if (!bNumberLeftMatches && !_ET9AW_DLM_FindWord(pLingInfo, sLeftWord.sString, sLeftWord.wLen, 0, 0)) { continue; } (void)_ET9AWLdbFindAmbigEntry(pLingInfo, dwLdbNum, sRightWord.sString, sRightWord.wLen, wSPt, (ET9SimpleWord *)psRightWord, &bRightExact, pdwIndexRight, &bNumberRightMatches, &bRightExactIndex); ET9Assert(bNumberLeftMatches <= ET9AW_MAX_AMBIG_SEGMENT_MATCHES && bNumberRightMatches <= ET9AW_MAX_AMBIG_SEGMENT_MATCHES); /* Space omission? */ if (bLeftExact && bRightExact) { if (!__IsInvalidSingleLetterWord(pLingInfo, dwLdbNum, &psLeftWord[bLeftExactIndex]) && !__IsInvalidSingleLetterWord(pLingInfo, dwLdbNum, &psRightWord[bRightExactIndex])) { bIsLMEntry = _ET9AWLdbIsLMEntriesByIndex(pLingInfo, dwLdbNum, pdwIndexLeft[bLeftExactIndex]-1, pdwIndexRight[bRightExactIndex]-1, &xLMWordFreq, &bLMOrder); bIsDLMEntry = _ET9AW_Is_DLM_Entries(pLingInfo, &psLeftWord[bLeftExactIndex], &psRightWord[bRightExactIndex], &xDLMWordFreq, &bDLMOrder, 0, NULL); if (bIsLMEntry || bIsDLMEntry) { if (ET9DLMENABLED(pLingCmnInfo) && ET9CONTEXTBASEDPREDICTION(pLingCmnInfo)) { xWordFreq = (ET9FREQPART)((pLingCmnInfo->Private.fInterpolFactLDB * xLMWordFreq) + (pLingCmnInfo->Private.fInterpolFactDLM * xDLMWordFreq)); } else if (bDLMOrder) { xWordFreq = xDLMWordFreq; } else { xWordFreq = xLMWordFreq; } bNLMOrder = bDLMOrder > bLMOrder ? bDLMOrder : bLMOrder; bWordSrc = bDLMOrder > bLMOrder ? ET9WORDSRC_DLM : ET9WORDSRC_LDB; if (xMaxFreq < xWordFreq || !bMaxExact) { sBestLeftAmbigSegment.wLen = sLeftWord.wLen; _ET9SymCopy(sBestLeftAmbigSegment.sString, psLeftWord[bLeftExactIndex].sString, sLeftWord.wLen); sBestRightAmbigSegment.wLen = sRightWord.wLen; _ET9SymCopy(sBestRightAmbigSegment.sString, psRightWord[bRightExactIndex].sString, sRightWord.wLen); xMaxFreq = xWordFreq; wBestSPt = wSPt; bMaxNLMOrder = bNLMOrder; bMaxWordSrc = bWordSrc; bMaxExact = 1; } } } if (!bMaxExact) { for (i = 0; i < bNumberLeftMatches; ++i) { ET9U16 wCount; ET9U16 const * psLeftSymb1 = sLeftWord.sString; ET9U16 const * psLeftSymb2 = psLeftWord[i].sString; if (i == bLeftExactIndex) { pbLeftMatchExact[i] = 1; continue; } for (wCount = sLeftWord.wLen; wCount && *psLeftSymb1 && ET9SymToLower(*psLeftSymb1, dwLdbNum) == ET9SymToLower(*psLeftSymb2, dwLdbNum); --wCount, ++psLeftSymb1, ++psLeftSymb2) { } if (!wCount) { pbLeftMatchExact[i] = 1; } } for (j = 0; j < bNumberRightMatches; ++j) { ET9U16 wCount; ET9U16 const * psRightSymb1 = sRightWord.sString; ET9U16 const * psRightSymb2 = psRightWord[j].sString; if (j == bRightExactIndex) { pbRightMatchExact[j] = 1; continue; } for (wCount = sRightWord.wLen; wCount && *psRightSymb1 && ET9SymToLower(*psRightSymb1, dwLdbNum) == ET9SymToLower(*psRightSymb2, dwLdbNum); --wCount, ++psRightSymb1, ++psRightSymb2) { } if (!wCount) { pbRightMatchExact[j] = 1; } } } } if (!bMaxExact) { /* Ambig matches */ for (i = 0; i < bNumberLeftMatches; ++i) { if (bLeftExact && bRightExact && !pbLeftMatchExact[i]) { continue; } if (__IsInvalidSingleLetterWord(pLingInfo, dwLdbNum, psLeftWord)) { continue; } for (j = 0; j < bNumberRightMatches; ++j) { if (bLeftExact && bRightExact && ((i == bLeftExactIndex && j == bRightExactIndex) || !pbRightMatchExact[j])) { continue; } if (__IsInvalidSingleLetterWord(pLingInfo, dwLdbNum, psRightWord)) { continue; } bIsLMEntry = _ET9AWLdbIsLMEntriesByIndex(pLingInfo, dwLdbNum, pdwIndexLeft[i]-1, pdwIndexRight[j]-1, &xLMWordFreq, &bLMOrder); bIsDLMEntry = _ET9AW_Is_DLM_Entries(pLingInfo, &psLeftWord[i], &psRightWord[j], &xDLMWordFreq, &bDLMOrder, 0, NULL); if (bIsLMEntry || bIsDLMEntry) { if (ET9DLMENABLED(pLingCmnInfo) && ET9CONTEXTBASEDPREDICTION(pLingCmnInfo)) { xWordFreq = (ET9FREQPART)((pLingCmnInfo->Private.fInterpolFactLDB * xLMWordFreq) + (pLingCmnInfo->Private.fInterpolFactDLM * xDLMWordFreq)); } else if (bDLMOrder) { xWordFreq = xDLMWordFreq; } else { xWordFreq = xLMWordFreq; } bNLMOrder = bDLMOrder > bLMOrder ? bDLMOrder : bLMOrder; bWordSrc = bDLMOrder > bLMOrder ? ET9WORDSRC_DLM : ET9WORDSRC_LDB; if (xMaxFreq < xWordFreq) { sBestLeftAmbigSegment.wLen = sLeftWord.wLen; _ET9SymCopy(sBestLeftAmbigSegment.sString, psLeftWord[i].sString, sLeftWord.wLen); sBestRightAmbigSegment.wLen = sRightWord.wLen; _ET9SymCopy(sBestRightAmbigSegment.sString, psRightWord[j].sString, sRightWord.wLen); xMaxFreq = xWordFreq; wBestSPt = wSPt; bMaxNLMOrder = bNLMOrder; bMaxWordSrc = bWordSrc; } } } } } { /* Check for exact in DLM */ ET9SimpleWord sAlternateWord; ET9BOOL bAlternate; sAlternateWord.wLen = 0; bIsDLMEntry = _ET9AW_Is_DLM_Entries(pLingInfo, &sLeftWord, &sRightWord, &xDLMWordFreq, &bDLMOrder, 1, &sAlternateWord); bAlternate = sAlternateWord.wLen ? 1 : 0; if (bIsDLMEntry) { if (bAlternate) { (void)_ET9AWLdbIsLMEntries(pLingInfo, &sLeftWord, &sAlternateWord, &xLMWordFreq, &bLMOrder); } else { (void)_ET9AWLdbIsLMEntries(pLingInfo, &sLeftWord, &sRightWord, &xLMWordFreq, &bLMOrder); } if (ET9DLMENABLED(pLingCmnInfo) && ET9CONTEXTBASEDPREDICTION(pLingCmnInfo)) { xWordFreq = (ET9FREQPART)((pLingCmnInfo->Private.fInterpolFactLDB * xLMWordFreq) + (pLingCmnInfo->Private.fInterpolFactDLM * xDLMWordFreq)); } else if (bDLMOrder) { xWordFreq = xDLMWordFreq; } else { xWordFreq = xLMWordFreq; } bNLMOrder = bDLMOrder > bLMOrder ? bDLMOrder : bLMOrder; bWordSrc = bDLMOrder > bLMOrder ? ET9WORDSRC_DLM : ET9WORDSRC_LDB; if (xMaxFreq < xWordFreq || !bMaxExact) { xMaxFreq = xWordFreq; wBestSPt = wSPt; bMaxNLMOrder = bNLMOrder; bMaxWordSrc = bWordSrc; sBestLeftAmbigSegment.wLen = sLeftWord.wLen; _ET9SymCopy(sBestLeftAmbigSegment.sString, sLeftWord.sString, sLeftWord.wLen); if (bAlternate) { sBestRightAmbigSegment.wLen = sAlternateWord.wLen; _ET9SymCopy(sBestRightAmbigSegment.sString, sAlternateWord.sString, sAlternateWord.wLen); } else { sBestRightAmbigSegment.wLen = sRightWord.wLen; _ET9SymCopy(sBestRightAmbigSegment.sString, sRightWord.sString, sRightWord.wLen); bMaxExact = 1; } } bLeftExact = 1; if (!bAlternate) { bRightExact = 1; } } /* Space approximation? */ if (wSPt < sExactWordSimple.wLen - 1 && _ET9HasSpaceInRegion(pWordSymbInfo, wSPt) && !bMaxExact) { xLMWordFreq = 0; xDLMWordFreq = 0; bLMOrder = 0; bDLMOrder = 0; sLeftWord.wLen = wSPt; _ET9SymCopy(sRightWord.sString, &sExactWordSimple.sString[wSPt + 1], sExactWordSimple.wLen - wSPt - 1); sRightWord.wLen = sExactWordSimple.wLen - wSPt - 1; if (!__IsInvalidSingleLetterWord(pLingInfo, dwLdbNum, &sLeftWord) && !__IsInvalidSingleLetterWord(pLingInfo, dwLdbNum, &sRightWord)) { bIsDLMEntry = _ET9AW_Is_DLM_Entries(pLingInfo, &sLeftWord, &sRightWord, &xDLMWordFreq, &bDLMOrder, 1, &sAlternateWord); bAlternate = sAlternateWord.wLen ? 1 : 0; if (bAlternate) { bIsLMEntry = _ET9AWLdbIsLMEntries(pLingInfo, &sLeftWord, &sAlternateWord, &xLMWordFreq, &bLMOrder); } else { bIsLMEntry = _ET9AWLdbIsLMEntries(pLingInfo, &sLeftWord, &sRightWord, &xLMWordFreq, &bLMOrder); } if (bIsDLMEntry || bIsLMEntry) { if (ET9DLMENABLED(pLingCmnInfo) && ET9CONTEXTBASEDPREDICTION(pLingCmnInfo)) { xWordFreq = (ET9FREQPART)((pLingCmnInfo->Private.fInterpolFactLDB * xLMWordFreq) + (pLingCmnInfo->Private.fInterpolFactDLM * xDLMWordFreq)); } else if (bDLMOrder) { xWordFreq = xDLMWordFreq; } else { xWordFreq = xLMWordFreq; } bNLMOrder = bDLMOrder > bLMOrder ? bDLMOrder : bLMOrder; bWordSrc = bDLMOrder > bLMOrder ? ET9WORDSRC_DLM : ET9WORDSRC_LDB; if (xMaxFreq < xWordFreq) { xMaxFreq = xWordFreq; wBestSPt = wSPt; bMaxNLMOrder = bNLMOrder; bMaxWordSrc = bWordSrc; sBestLeftAmbigSegment.wLen = sLeftWord.wLen; _ET9SymCopy(sBestLeftAmbigSegment.sString, sLeftWord.sString, sLeftWord.wLen); if (bAlternate) { sBestRightAmbigSegment.wLen = sAlternateWord.wLen; _ET9SymCopy(sBestRightAmbigSegment.sString, sAlternateWord.sString, sAlternateWord.wLen); } else { sBestRightAmbigSegment.wLen = sRightWord.wLen; _ET9SymCopy(sBestRightAmbigSegment.sString, sRightWord.sString, sRightWord.wLen); } } } } } } } if (xMaxFreq) { /* Add segmented candidate to list */ const ET9BOOL bDownShift = (ET9BOOL)(ET9DOWNSHIFTALLLDB(pLingCmnInfo) && pLingCmnInfo->Private.ALdb.header.bUpperCount && _ET9_LanguageSpecific_ApplyShifting(pLingInfo, NULL)); ET9AWPrivWordInfo sSegmentWord; _InitPrivWordInfo(&sSegmentWord); _ET9SymCopy(sSegmentWord.Base.sWord, sBestLeftAmbigSegment.sString, sBestLeftAmbigSegment.wLen); sSegmentWord.Base.sWord[wBestSPt] = ' '; _ET9SymCopy(&sSegmentWord.Base.sWord[wBestSPt + 1], sBestRightAmbigSegment.sString, sBestRightAmbigSegment.wLen); sSegmentWord.Base.wWordLen = sBestLeftAmbigSegment.wLen + sBestRightAmbigSegment.wLen + 1; wShiftPt = wBestSPt; sSegmentWord.Base.bIsTerm = 1; sSegmentWord.Body.bEditDistFree = 1; sSegmentWord.Body.bIsSegment = 1; sSegmentWord.Body.bLangIndexScoring = (dwLdbNum == pLingCmnInfo->dwSecondLdbNum) ? ET9AWSECOND_LANGUAGE : ET9AWFIRST_LANGUAGE; sSegmentWord.Body.bNLMOrder = bMaxNLMOrder; sSegmentWord.Body.bWordSrc = bMaxWordSrc; sSegmentWord.Body.xScaledTapFreq = sSegmentWord.Body.xTapFreq; sSegmentWord.Body.xWordFreq = xMaxFreq; WLOG2BWord(pLogFile2, "__AddSegmentedCandidate, attempting segmented word", &sSegmentWord); __UpdateSpcInfo(pLingInfo, &sSegmentWord, 0, pWordSymbInfo->bNumSymbs, bSpcMode); /* downshift word if option selected */ if (bDownShift) { ET9U16 wCount = sSegmentWord.Base.wWordLen; ET9SYMB *pSymb = sSegmentWord.Base.sWord; for (; wCount; --wCount, ++pSymb) { *pSymb = _ET9SymToLower(*pSymb, dwLdbNum); } } _ET9AWSelLstWordPreAdd(pLingInfo, &sSegmentWord, 0, (ET9U8) sSegmentWord.Base.wWordLen, (ET9U8) wShiftPt, FREQ_NORMAL); /* Attempt to add */ _ET9AWSelLstAdd(pLingInfo, &sSegmentWord, sSegmentWord.Base.wWordLen, FREQ_NORMAL); } } __ProfileEnd(tAW_SelLst_SpaceSegmentation); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9BOOL ET9LOCALCALL __AreConfusableWords(ET9AWLingInfo const * const pLingInfo, ET9AWPrivWordInfo const * const pWordU, ET9AWPrivWordInfo const * const pWordNU) { ET9AWLingCmnInfo const * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo const * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9BOOL bHasTraceInfo = pLingCmnInfo->Private.bTraceBuild; /* only if "mixed" */ if (!__IsUsingMixedStyle(pLingCmnInfo)) { return 0; } /* not if too short tap */ if (pWordSymbInfo->bNumSymbs < 2 && !bHasTraceInfo) { return 0; } /* not if the basic word-freq is higher */ if (pWordU->Body.xWordFreq > pWordNU->Body.xWordFreq) { return 0; } /* special rules for short lengths, disambiguation goes down badly with added words */ if (pWordSymbInfo->bNumSymbs >= 2 && pWordSymbInfo->bNumSymbs <= 3) { if (pWordNU->Base.wWordLen == pWordSymbInfo->bNumSymbs) { return 1; } } /* check for a confusable match */ { const ET9U32 dwCurrActiveLanguage = pLingCmnInfo->Private.dwCurrActiveLanguage; const ET9BOOL nFreeSyms = 1; const ET9BOOL nFreeCousins = 1; const ET9BOOL nFreeDoubleLetters = (bHasTraceInfo) ? 1 : 0; ET9UINT nLoopCount; ET9UINT nLen1; ET9UINT nLen2; ET9SYMB const * psSymb1; ET9SYMB const * psSymb2; nLen1 = pWordU->Base.wWordLen; nLen2 = pWordNU->Base.wWordLen; psSymb1 = pWordU->Base.sWord; psSymb2 = pWordNU->Base.sWord; for (nLoopCount = 0; nLoopCount < ET9MAXWORDSIZE; ++nLoopCount) { /* move comparison points forward */ while (nLen1 && ((nFreeSyms && _ET9_IsFree(*psSymb1)) || (nFreeDoubleLetters && nLen1 > 1 && _ET9SymToLower(psSymb1[0], dwCurrActiveLanguage) == _ET9SymToLower(psSymb1[1], dwCurrActiveLanguage)))) { --nLen1; ++psSymb1; } while (nLen2 && ((nFreeSyms && _ET9_IsFree(*psSymb2)) || (nFreeDoubleLetters && nLen2 > 1 && _ET9SymToLower(psSymb2[0], dwCurrActiveLanguage) == _ET9SymToLower(psSymb2[1], dwCurrActiveLanguage)))) { --nLen2; ++psSymb2; } /* if both exhausted then confusables */ if (!nLen1 && !nLen2) { return (nLoopCount) ? 1 : 0; } /* stop if one is exhausted */ if (!nLen1 || !nLen2) { break; } /* compare chars */ { ET9BOOL bMatch; if (*psSymb1 == *psSymb2) { bMatch = 1; } else if (_ET9SymToLower(*psSymb1, dwCurrActiveLanguage) == _ET9SymToLower(*psSymb2, dwCurrActiveLanguage)) { bMatch = 1; } else if (nFreeCousins) { ET9SYMB const * psAlt1; ET9SYMB const * psAlt2; psAlt1 = _ET9_GetAltCharsLower(*psSymb1); if (!psAlt1) { psAlt1 = _ET9_GetAltCharsUpper(*psSymb1); } psAlt2 = _ET9_GetAltCharsLower(*psSymb2); if (!psAlt2) { psAlt2 = _ET9_GetAltCharsUpper(*psSymb2); } bMatch = (psAlt1 && psAlt1 == psAlt2) ? 1 : 0; } else { bMatch = 0; } if (!bMatch) { break; } --nLen1; --nLen2; ++psSymb1; ++psSymb2; } } } /* done - not confusables */ return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DoBuildFinalProcessing(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; __ProfileStart; ET9AssertLog(pLingInfo != NULL); /* anything to do? */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } /* currently no final processing on NWP lists */ if (!pWordSymbInfo->bNumSymbs) { WLOG2(fprintf(pLogFile2, "FP - no action (NWP)\n");) return; } /* keep these as separate issues (when applicable) for clarity (not a speed problem) */ /* log unmodified list */ __LogPartialSelList(pLingInfo->pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "FP - before final processing", pLogFile2); /* if the current default word is capitalized (not shifted) and not locked and applicable language, then try to promote a lower case version of the word in front of it - skip this when there is lexical LM support (or currently ALM support) */ { ET9AWPrivWordInfo const * const pDefaultWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex]]; if (!pLingCmnInfo->Private.wCurrLockPoint && pDefaultWord->Body.bIsCapitalized && #if 1 !(pLingCmnInfo->Private.bUsingALM || pLingCmnInfo->Private.bUsingDLM) && #else !pDefaultWord->Body.bNLMOrder && #endif _ET9_LanguageSpecific_ApplyCapsRules(pLingInfo, pDefaultWord)) { ET9UINT nIndex; ET9SYMB sLowerWord[ET9MAXWORDSIZE]; WLOG2(fprintf(pLogFile2, "FP - found capitalized default\n");) { const ET9U32 dwLdbNum = (pDefaultWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; for (nIndex = 0; nIndex < pDefaultWord->Base.wWordLen; ++nIndex) { sLowerWord[nIndex] = _ET9SymToLower(pDefaultWord->Base.sWord[nIndex], dwLdbNum); } } for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (nIndex == pLingCmnInfo->Private.nDefaultIndex) { continue; } if (pIndexWord->Base.wWordLen == pDefaultWord->Base.wWordLen) { ET9UINT nCount; ET9SYMB const *pSymb1 = sLowerWord; ET9SYMB const *pSymb2 = pIndexWord->Base.sWord; for (nCount = pDefaultWord->Base.wWordLen; nCount; --nCount) { if (*pSymb1++ != *pSymb2++) { break; } } if (!nCount) { break; } } } if (nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (ISGENUINESRC(pIndexWord->Body.bWordSrc) && !ISNONOVERRIDEWRD(pIndexWord) && pIndexWord->Body.bWordQuality == GENUINE_QUALITY) { WLOG2(fprintf(pLogFile2, "FP - found genuine quality lower case version of capitalized word @ %u\n", nIndex);) if (nIndex < pLingCmnInfo->Private.nDefaultIndex) { WLOG2(fprintf(pLogFile2, "FP - lower case version already ahead of the default word, moving the default from %u to %u\n", pLingCmnInfo->Private.nDefaultIndex, nIndex);) pLingCmnInfo->Private.nDefaultIndex = nIndex; } else { WLOG2(fprintf(pLogFile2, "FP - lower case version after default word, moving the lower case word up\n");) __ShiftSubListDown(pLingCmnInfo, pLingCmnInfo->Private.nDefaultIndex, (nIndex - 1)); } } } } } /* if the current default word is a user word and a confusable with the best non-user word, then make the non-user word default instead */ { ET9AWPrivWordInfo const * const pDefaultWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex]]; if (!pDefaultWord->Body.dwWordIndex && !pDefaultWord->Body.bIsSegment && ((GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_RUDB && pDefaultWord->Body.bIsUDBWord) || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_CDB) || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_QUDB) || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_ASDB_SHORTCUT) || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_MDB) || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_DLM) || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_NONE)) && _ET9_LanguageSpecific_ApplyConfusablesRules(pLingInfo)) { ET9UINT nIndex; ET9Assert(GETRAWSRC(pDefaultWord->Body.bWordSrc) != ET9WORDSRC_LDB); ET9Assert(GETRAWSRC(pDefaultWord->Body.bWordSrc) != ET9WORDSRC_LAS_SHORTCUT); WLOG2(fprintf(pLogFile2, "FP - found default user word\n");) for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (nIndex == pLingCmnInfo->Private.nDefaultIndex) { continue; } if (pIndexWord->Body.dwWordIndex || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_LDB) || (GETRAWSRC(pDefaultWord->Body.bWordSrc) == ET9WORDSRC_LAS_SHORTCUT)) { break; } } if (nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; WLOG2(fprintf(pLogFile2, "FP - found non-user word @ %u\n", nIndex);) if (__AreConfusableWords(pLingInfo, pDefaultWord, pIndexWord)) { if (!pLingCmnInfo->Private.nDefaultIndex && ISEXACTSRC(pDefaultWord->Body.bWordSrc)) { WLOG2(fprintf(pLogFile2, "FP - user word first and exact\n");) if (nIndex > 1) { WLOG2(fprintf(pLogFile2, "FP - moving the non-user word from %u to 1\n", nIndex);) __ShiftSubListDown(pLingCmnInfo, 1, (nIndex - 1)); } WLOG2(fprintf(pLogFile2, "FP - changing the default to 1\n");) ++pLingCmnInfo->Private.nDefaultIndex; } else if (nIndex < pLingCmnInfo->Private.nDefaultIndex) { WLOG2(fprintf(pLogFile2, "FP - non-user word already ahead of the default word, moving the default from %u to %u\n", pLingCmnInfo->Private.nDefaultIndex, nIndex);) pLingCmnInfo->Private.nDefaultIndex = nIndex; } else { WLOG2(fprintf(pLogFile2, "FP - non-user word after default word, moving the non-user word up\n");) __ShiftSubListDown(pLingCmnInfo, pLingCmnInfo->Private.nDefaultIndex, (nIndex - 1)); } } } } } /* if the 1st word is acronymish (multi shifted) and not locked and applicable language and corrected, then try to promote a better word in front of it */ __DemoteAcronym(pLingInfo, 0); /* if the 2nd word is acronymish (multi shifted) and not locked and applicable language and corrected, then try to promote a better word in front of it */ __DemoteAcronym(pLingInfo, 1); #if 0 /* if the current default word is non override source, then try to promote a better word in front of it */ { ET9AWPrivWordInfo const * const pDefaultWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex]]; if (!pLingCmnInfo->Private.wCurrLockPoint && ISNONOVERRIDEWRD(pDefaultWord)) { WLOG2(fprintf(pLogFile2, "FP - found non-override-source default\n");) for (bIndex = 0; bIndex < pLingCmnInfo->Private.bTotalWords; ++bIndex) { ET9AWPrivWordInfo const * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[bIndex]]; if (bIndex == pLingCmnInfo->Private.nDefaultIndex) { continue; } if (!ISGENUINESRC(pIndexWord->Body.bWordSrc) || ISNONOVERRIDEWRD(pIndexWord)) { continue; } break; } if (bIndex < pLingCmnInfo->Private.bTotalWords) { WLOG2(fprintf(pLogFile2, "FP - found promotable word @ %u\n", bIndex);) if (bIndex < pLingCmnInfo->Private.nDefaultIndex) { WLOG2(fprintf(pLogFile2, "FP - promotable already ahead of the default word, moving the default from %u to %u\n", pLingCmnInfo->Private.nDefaultIndex, bIndex);) pLingCmnInfo->Private.nDefaultIndex = bIndex; } else { WLOG2(fprintf(pLogFile2, "FP - promotable after default word, moving the promotable word up\n");) __ShiftSubListDown(pLingCmnInfo, pLingCmnInfo->Private.nDefaultIndex, (bIndex - 1)); } } } } #endif /* if the current default word is uppercase (not shifted) and not locked and default downshifting is enabled */ if (ET9DOWNSHIFTDEFAULTENABLED(pLingCmnInfo) && !pLingCmnInfo->Private.wCurrLockPoint && !pWordSymbInfo->SymbsInfo[0].eShiftState && _ET9SymIsUpper(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex]].Base.sWord[0], pWordSymbInfo->Private.dwLocale) && GETBASESRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex]].Body.bWordSrc) != ET9WORDSRC_MAGICSTRING) { ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex]]; ET9AWPrivWordInfo *pLowerWord; ET9U8 bReadyToDownshift = 0; /* if the word is uppercase naturally need to check to see if lowercase version already exists in list if it does, don't do downshift logic (may want to revist this later to reposition the lowercase word to follow the uppercase version */ ET9SYMB sWord[ET9MAXWORDSIZE]; ET9UINT nIndex; { const ET9U32 dwLdbNum = (pWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; for (nIndex = 0; nIndex < pWord->Base.wWordLen; ++nIndex) { sWord[nIndex] = _ET9SymToLower(pWord->Base.sWord[nIndex], dwLdbNum); } } for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pLowerWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pLowerWord->Base.wWordLen == pWord->Base.wWordLen) { ET9U8 bIndex2; for (bIndex2 = 0; bIndex2 < pWord->Base.wWordLen; ++bIndex2) { if (pLowerWord->Base.sWord[bIndex2] != sWord[bIndex2]) { break; } } if (bIndex2 == pWord->Base.wWordLen) { break; } } } /* if all words processed with no matches to lowercase version of word, keep going */ if (nIndex == pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { WLOG2(fprintf(pLogFile2, "FP - adding lower case version of default word\n");) bReadyToDownshift = 1; } /* if lowercase index further down list than position after default */ else if (nIndex > (pLingCmnInfo->Private.nDefaultIndex + 1)) { WLOG2(fprintf(pLogFile2, "FP - moving up lower case version to after default word\n");) /* then roll the lowercase word up to position after default, and skip downshift logic */ if (pLingCmnInfo->Private.nDefaultIndex + 2 < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { __ShiftSubListDown(pLingCmnInfo, (pLingCmnInfo->Private.nDefaultIndex + 1), (nIndex - 1)); } } /* if still set to add lowercase version of default word... */ if (bReadyToDownshift) { /* if list full, remove low priority entry */ if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords == pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) { /* otherwise remove the last entry - skip protected */ for (nIndex = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1; nIndex; --nIndex) { if (!ISPROTECTEDWRD(&pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]])) { break; } } /* if all are protected, try to find a stem (likely full of stems) */ if (!nIndex) { for (nIndex = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1; nIndex; --nIndex) { if (GETBASESRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]].Body.bWordSrc) == ET9WORDSRC_STEM) { break; } } } /* otherwise just remove the last entry (should never happen unless the list is really small) */ if (!nIndex) { ET9AssertLog(0); nIndex = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1; } /* actual remove */ __RemoveWord(pLingCmnInfo, nIndex); ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); } /* find an available slot */ { ET9U8 bEntries[ET9MAXCOLLECTSIZE]; ET9UINT nIndex2; _ET9ClearMem(bEntries, ET9MAXCOLLECTSIZE); for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { bEntries[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]] = 1; } for (nIndex2 = 0; nIndex2 < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize; ++nIndex2) { if (!bEntries[nIndex2]) { nIndex = nIndex2; break; } } ET9AssertLog(nIndex2 < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); } /* fake a new entry at the end of the list by loading the free index and bumping the count */ pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords] = nIndex; ++pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; /* roll it to position following default word if default not the last entry in list*/ if (pLingCmnInfo->Private.nDefaultIndex + 2 < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { __ShiftSubListDown(pLingCmnInfo, (pLingCmnInfo->Private.nDefaultIndex + 1), pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 2); } /* now duplicate the default word, and downshift it */ pLowerWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex + 1]]; *pLowerWord = *pWord; if (pLowerWord->Body.bWordQuality == GENUINE_QUALITY) { pLowerWord->Body.bWordQuality = SHAPED_QUALITY; } { const ET9U32 dwLdbNum = (pLowerWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; for (nIndex = 0; nIndex < pLowerWord->Base.wWordLen; ++nIndex) { pLowerWord->Base.sWord[nIndex] = _ET9SymToLower(pLowerWord->Base.sWord[nIndex], dwLdbNum); } /* lowercase substitution too (NOT SURE IF THIS IS VALID?) */ if (pLowerWord->Base.wSubstitutionLen) { for (nIndex = 0; nIndex < pLowerWord->Base.wSubstitutionLen; ++nIndex) { pLowerWord->Base.sSubstitution[nIndex] = _ET9SymToLower(pLowerWord->Base.sSubstitution[nIndex], dwLdbNum); } } } } } /* if input len 1 and the current 2nd word is not a completion, then try to promote a completion in front of it */ if (pWordSymbInfo->bNumSymbs == 1) { __PromoteCompletion(pLingInfo, 2); } /* if the current 3rd word is not a completion, then try to promote a completion in front of it */ __PromoteCompletion(pLingInfo, 3); /* bring a secondary language word up to 4th */ __PromoteSecondary(pLingInfo, 4); /* assure that the top of the list isn't crowded by non LDB words */ __HandleTopCrowding(pLingInfo); /* last thing to do, assure that default isn't weak */ __DemoteWeakDefault(pLingInfo); /* done */ __LogPartialSelList(pLingInfo->pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "FP - after final processing", pLogFile2); __ProfileEnd(tAW_SelLst_DoBuildFinalProcessing); WLOG2(fprintf(pLogFile2, "FP - done\n\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DoRequiredWordProcessing(ET9AWLingInfo * const pLingInfo, const ET9U16 wFlushPoint) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9AssertLog(pLingInfo); /* anything to do? */ if (!pWordSymbInfo->Private.bRequiredLocate || !pWordSymbInfo->bNumSymbs || wFlushPoint || !pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } { ET9UINT nRequiredIndex; WLOG2(fprintf(pLogFile2, "RWP - Default index, searching for required word\n");) for (nRequiredIndex = 0; nRequiredIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nRequiredIndex) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nRequiredIndex]]; if (pWord->Base.wWordLen == pWordSymbInfo->Private.sRequiredWord.wLen) { ET9U16 wCmpLen; ET9SYMB *pStr1; ET9SYMB *pStr2; wCmpLen = pWord->Base.wWordLen; pStr1 = pWordSymbInfo->Private.sRequiredWord.sString; pStr2 = pWord->Base.sWord; ++wCmpLen; while (--wCmpLen) { if (*pStr1++ != *pStr2++) { break; } } if (wCmpLen) { continue; } WLOG2(fprintf(pLogFile2, "RWP - Default index, found required word @ %d\n", nRequiredIndex);) pLingCmnInfo->Private.bRequiredFound = 1; { ET9AWPrivWordInfo * const pWord0 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[0]]; const ET9UINT nTargetIndex = ISEXACTSRC(pWord0->Body.bWordSrc) ? 1 : 0; if (nRequiredIndex > nTargetIndex) { WLOG2(fprintf(pLogFile2, "RWP - moving up required word to %u\n", nTargetIndex);) __ShiftSubListDown(pLingCmnInfo, nTargetIndex, (nRequiredIndex - 1)); pLingCmnInfo->Private.nDefaultIndex = nTargetIndex; } else { pLingCmnInfo->Private.nDefaultIndex = nRequiredIndex; } } /* for this to work properly the word can't have a substitution */ if (pWord->Base.wSubstitutionLen) { WLOG2(fprintf(pLogFile2, "RWP - required word has substitution, removing it\n");) pWord->Base.wSubstitutionLen = 0; /* leave the source as is, doesn't matter and it's kind of unknown, the public one will be set ok */ } break; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __IsShrinkProtected(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo const * const pWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (ISPROTECTEDWRD(pWord) || ISREQUIREDSRC(pWord->Body.bWordSrc) || ISEXACTSRC(pWord->Body.bWordSrc)) { return 1; } if (pLingCmnInfo->Private.nExactIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords && pWord == pLingCmnInfo->Private.sWordC.pCurrC->pWordList) { return 1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __ShrinkSelListToTargetSize(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9BOOL bIsNWP = (ET9BOOL)!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs; const ET9UINT nTargetSize = bIsNWP ? pLingCmnInfo->Private.bTotalNWP : _ET9_LanguageSpecific_ApplyListShrinking(pLingInfo) ? pLingCmnInfo->Private.nListSize : __ET9Min(250, ET9MAXCOLLECTSIZE); const ET9UINT nKeepTopCount = 1; ET9UINT nProtectedCount; ET9UINT nAvailableNonProtectedCount; __ProfileStart; WLOG2(fprintf(pLogFile2, "__ShrinkSelListToTargetSize, target %u, current %u\n", nTargetSize, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords);) if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= nTargetSize) { return; } /* count the number of protected words */ { ET9UINT nIndex; nProtectedCount = 0; for (nIndex = nKeepTopCount; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords && (nProtectedCount + nKeepTopCount < nTargetSize); ++nIndex) { ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (__IsShrinkProtected(pLingInfo, pWord)) { ++nProtectedCount; } } } /* space to work with */ ET9AssertLog(nTargetSize >= nKeepTopCount + nProtectedCount); nAvailableNonProtectedCount = nTargetSize - nKeepTopCount - nProtectedCount; /* move words into place */ { ET9UINT nIndex; ET9UINT nCurrNewIndex; nCurrNewIndex = nKeepTopCount; for (nIndex = nCurrNewIndex; nCurrNewIndex < nTargetSize && nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo const * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; const ET9UINT nCurrNewIndexOld = nCurrNewIndex; if (__IsShrinkProtected(pLingInfo, pWord)) { pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nCurrNewIndex++] = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]; } else if (nAvailableNonProtectedCount) { --nAvailableNonProtectedCount; pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nCurrNewIndex++] = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]; } if (pLingCmnInfo->Private.nDefaultIndex == nIndex) { if (nCurrNewIndex > nCurrNewIndexOld) { pLingCmnInfo->Private.nDefaultIndex = nCurrNewIndex - 1; WLOG2(fprintf(pLogFile2, " nDefaultIndex changed to %u (following moved word)\n", pLingCmnInfo->Private.nDefaultIndex);) } } if (pLingCmnInfo->Private.nExactIndex == nIndex) { if (nCurrNewIndex > nCurrNewIndexOld) { pLingCmnInfo->Private.nExactIndex = nCurrNewIndex - 1; WLOG2(fprintf(pLogFile2, " nExactIndex changed to %u (following moved word)\n", pLingCmnInfo->Private.nExactIndex);) } } } ET9AssertLog(nCurrNewIndex == nTargetSize); /* assure that the exact is in */ if (pLingCmnInfo->Private.nExactIndex >= nIndex && pLingCmnInfo->Private.nExactIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { const ET9UINT nNewExactIndex = nCurrNewIndex - 1; pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nNewExactIndex] = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nExactIndex]; if (pLingCmnInfo->Private.nDefaultIndex == pLingCmnInfo->Private.nExactIndex) { pLingCmnInfo->Private.nDefaultIndex = nNewExactIndex; WLOG2(fprintf(pLogFile2, " nDefaultIndex changed to %u (following the exact)\n", pLingCmnInfo->Private.nDefaultIndex);) } pLingCmnInfo->Private.nExactIndex = nNewExactIndex; WLOG2(fprintf(pLogFile2, " nExactIndex changed to %u\n", pLingCmnInfo->Private.nExactIndex);) } /* assure that the default is in */ if (pLingCmnInfo->Private.nDefaultIndex >= nIndex && pLingCmnInfo->Private.nDefaultIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { if (nCurrNewIndex >= 2) { const ET9UINT nNewDefaultIndex = (pLingCmnInfo->Private.nExactIndex != nCurrNewIndex - 1) ? (nCurrNewIndex - 1) : (nCurrNewIndex - 2); pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nNewDefaultIndex] = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nDefaultIndex]; pLingCmnInfo->Private.nDefaultIndex = nNewDefaultIndex; WLOG2(fprintf(pLogFile2, " nDefaultIndex changed to %u\n", pLingCmnInfo->Private.nDefaultIndex);) } else { ET9Assert(nTargetSize == 1); pLingCmnInfo->Private.nDefaultIndex = 0; WLOG2(fprintf(pLogFile2, " nDefaultIndex changed to %u (target size 1)\n", pLingCmnInfo->Private.nDefaultIndex);) } } /* update total word count - done */ ET9AssertLog(pLingCmnInfo->Private.nExactIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || pLingCmnInfo->Private.nExactIndex < nCurrNewIndex); pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords = nCurrNewIndex; } /* log modified list */ __LogPartialSelList(pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "after shrink", pLogFile2); __ProfileEnd(tAW_SelLst_ShrinkSelListToTargetSize); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __ET9AWPriorityListFinalOrder(ET9AWLingCmnInfo * const pLingCmnInfo) { const ET9UINT nTotalWords = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ET9UINT const * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; ET9AWPrivWordInfo const * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; __ProfileStart; if (nTotalWords < 2) { return; } /* this is a fairly heavy (n^2/2) list re-ordering, try to optimize based on possible order anomalies */ if (!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { /* NWP list, no anomalies */ } else if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_MIXED) { /* mixed, issues around punct-source and completions (non-terms) */ ET9UINT nFirstNonTermIndex = 0xFFFF; ET9UINT nIndex; for (nIndex = 0; nIndex < nTotalWords; ++nIndex) { const ET9UINT nChkIndex = pnWordList[nIndex]; ET9AWPrivWordInfo const * const pWordChk = &pWordList[nChkIndex]; /* remember first non-term found */ if (!pWordChk->Base.bIsTerm && nFirstNonTermIndex > nIndex) { nFirstNonTermIndex = nIndex; } /* potentially move up puncts to just before the non-term (pushing it down) */ if (ISPUNCTSRC(pWordChk->Body.bWordSrc) && nFirstNonTermIndex < nIndex && __PriorityCompare(pLingCmnInfo, pWordChk, &pWordList[nFirstNonTermIndex], SS_Final) > 0) { __ShiftSubListDown(pLingCmnInfo, nFirstNonTermIndex, nIndex - 1); ++nFirstNonTermIndex; /* pushed down one step by the shift */ /* no need to revisit index since the moved down word already is tested */ } } } else { /* fallback, do it the hard way... */ ET9U8 pbWordVisited[ET9MAXCOLLECTSIZE]; ET9UINT nSentinelCount = 0; ET9UINT nIndex; _ET9ClearMem(pbWordVisited, sizeof(pbWordVisited)); for (nIndex = nTotalWords - 1; nIndex > 0; --nIndex) { const ET9UINT nChkIndex = pnWordList[nIndex]; ET9AWPrivWordInfo const * const pWordChk = &pWordList[nChkIndex]; ET9UINT nLook; ++nSentinelCount; if (pbWordVisited[nChkIndex]) { continue; } pbWordVisited[nChkIndex] = 1; for (nLook = 0; nLook < nIndex; ++nLook) { ET9AWPrivWordInfo const * const pWordLook = &pWordList[pnWordList[nLook]]; if (__PriorityCompare(pLingCmnInfo, pWordChk, pWordLook, SS_Final) > 0) { break; } } if (nLook >= nIndex) { continue; } WLOG2(fprintf(pLogFile2, "__ET9AWPriorityListFinalOrder, %u moves to %u\n", nIndex - 1, nLook);) __ShiftSubListDown(pLingCmnInfo, nLook, nIndex - 1); /* revisit */ ++nIndex; } ET9AssertLog(nSentinelCount <= 2 * pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); } __ProfileEnd(tAW_SelLst_PriorityListFinalOrder); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __TagExactLastAsRealExact(ET9AWLingCmnInfo * const pLingCmnInfo) \ { const ET9UINT nTotalWords = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ET9UINT const * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; if (!ET9EXACTLAST(pLingCmnInfo)) { return; } if (ISEXACTSRC(pWordList[0].Body.bWordSrc)) { return; } if (pWordList[0].Body.bWordSrc != ET9WORDSRC_EXACT_LAST) { return; } if (pLingCmnInfo->Private.nExactIndex < nTotalWords) { return; } { ET9UINT nIndex; for (nIndex = 0; nIndex < nTotalWords; ++nIndex) { if (!pnWordList[nIndex]) { pWordList[0].Body.bWordSrc |= EXACTOFFSET; pLingCmnInfo->Private.nExactIndex = nIndex; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * * * * * * * * * * * * * * */ static void ET9LOCALCALL __DoBilingualReorder(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; ET9UINT * const pnWordList = pLingCmnInfo->Private.sWordC.pCurrC->pnWordList; ET9AssertLog(pLingInfo); if (!ET9AW_GetBilingualSupported(pLingInfo)) { return; } if (!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { return; } WLOG2(fprintf(pLogFile2, "__DoBilingualReorder\n");) if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_MIXED) { /* handled in final processing for mixed, to assure what happens */ } else { const ET9U8 bActiveIndex = pLingCmnInfo->Private.bActiveIndex; const ET9U8 bNonActiveIndex = pLingCmnInfo->Private.bNonActiveIndex; ET9UINT i; ET9UINT j; ET9UINT nFenceCount; /* terms with distance 0 from active language (bound by primary fence count) */ i = pLingCmnInfo->Private.nDefaultIndex; j = i; while ((i < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1) && (j < pLingCmnInfo->Private.bPrimaryFence + pLingCmnInfo->Private.nDefaultIndex)) { ET9AWPrivWordInfo * const pCurrWord = &pWordList[pnWordList[i]]; if ((pCurrWord->Base.bLangIndex == bActiveIndex || pCurrWord->Base.bLangIndex == ET9AWBOTH_LANGUAGES) && ISREALSRC(pCurrWord->Body.bWordSrc) && !pCurrWord->Body.bEditDistStem && !pCurrWord->Base.wWordCompLen) { } else { break; } ++i; ++j; } /* terms with distance 0 from non-active language (bound by secondary fence count) */ nFenceCount = j; for (i = nFenceCount, j = 0; i < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1 && j < pLingCmnInfo->Private.bSecondaryFence; ++i) { ET9AWPrivWordInfo * const pCurrWord = &pWordList[pnWordList[i]]; if ((pCurrWord->Base.bLangIndex == bNonActiveIndex || pCurrWord->Base.bLangIndex == ET9AWBOTH_LANGUAGES) && ISREALSRC(pCurrWord->Body.bWordSrc) && !ISPUNCTSRC(pCurrWord->Body.bWordSrc) && !pCurrWord->Body.bEditDistStem && !pCurrWord->Base.wWordCompLen) { if (i != nFenceCount + j) { __ShiftSubListDown(pLingCmnInfo, nFenceCount + j, i - 1); } ++j; } } } WLOG2(fprintf(pLogFile2, "__DoBilingualReorder - done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * * * */ static ET9U8 ET9LOCALCALL __ET9BuildAroundSpecial(ET9AWLingInfo * const pLingInfo, const ET9U16 wStartPoint, const ET9U16 wSpecialPosition, const ET9ASPECIAL eSpecialType, ET9AWPrivWordInfo * const pLeftHandWord, ET9AWPrivWordInfo * const pDefaultWord, /* obsolete? Replaced by the flush */ const ET9ABUILAROUND eBuildaroundType, const ET9SYMB sEmbeddedChar, const ET9U8 bSpcMode) { ET9BOOL bFound; ET9BOOL bFoundFull; ET9U16 wLeftLength; ET9U16 wRightLength; ET9U16 wRightStartIndex; ET9SYMB sSymb; ET9U8 bTempLeftLangIndex; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9U16 wLeftHandLen = wSpecialPosition + wStartPoint - (eBuildaroundType == BA_LEADING ? 1 : 0); ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pLeftHandWord != NULL); ET9AssertLog(pDefaultWord != NULL); if (!_ET9_LanguageSpecific_ApplyBuildarounds(pLingInfo)) { WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, disabled, skipping\n");) return 0; } WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial\n StartPoint = %d\n SpecialPosition = %d\n SpecialType = %s\n BuildaroundType = %s\n sEmbeddedChar = %c (%x)\n", wStartPoint, wSpecialPosition, SPECIALTOSTRING(eSpecialType), BUILDAROUNDTOSTRING(eBuildaroundType), (sEmbeddedChar ? sEmbeddedChar : ' '), sEmbeddedChar);) WLOG2Word(pLogFile2, "..sLeftHandWord", pLeftHandWord); WLOG2Word(pLogFile2, "..DefaultWord", pDefaultWord); /* validate */ if (!pWordSymbInfo->bNumSymbs) { return 0; } if (pLingCmnInfo->Private.wCurrLockPoint == pWordSymbInfo->bNumSymbs) { WLOG2(fprintf(pLogFile2, "at lock point, won't perform build around\n");) return 0; } /* */ if (eBuildaroundType == BA_TRAILING || eBuildaroundType == BA_COMPOUND) { wLeftLength = wSpecialPosition; wRightLength = (ET9U16)(pWordSymbInfo->bNumSymbs - wSpecialPosition - wStartPoint); wRightStartIndex = (ET9U16)(wSpecialPosition + wStartPoint); WLOG2(fprintf(pLogFile2, "..trailing or compound, wLeftLength = %d, wRightLength = %d, wRightStartIndex = %d\n", wLeftLength, wRightLength, wRightStartIndex);) } else if (eBuildaroundType == BA_LEADING) { wLeftLength = (ET9U16)(wSpecialPosition - 1); wRightLength = (ET9U16)(pWordSymbInfo->bNumSymbs - wSpecialPosition + 1 - wStartPoint); wRightStartIndex = (ET9U16)(wSpecialPosition - 1 + wStartPoint); WLOG2(fprintf(pLogFile2, "..leading, wLeftLength = %d, wRightLength = %d, wRightStartIndex = %d\n", wLeftLength, wRightLength, wRightStartIndex);) } else { ET9AssertLog(eBuildaroundType == BA_EMBEDDED); wLeftLength = (ET9U16)(wSpecialPosition - 1); wRightLength = (ET9U16)(pWordSymbInfo->bNumSymbs - wSpecialPosition - wStartPoint); wRightStartIndex = (ET9U16)(wSpecialPosition + wStartPoint); WLOG2(fprintf(pLogFile2, "..embedded, wLeftLength = %d, wRightLength = %d, wRightStartIndex = %d\n", wLeftLength, wRightLength, wRightStartIndex);) } bTempLeftLangIndex = pLeftHandWord->Body.bLangIndexScoring; /* build left side */ _InitPrivWordInfo(pLeftHandWord); pLeftHandWord->Body.bLangIndexScoring = bTempLeftLangIndex; bFound = 1; bFoundFull = 0; if (wLeftLength) { ET9U8 bStemsAllowed = 0; /* allow getting top match stem words if this is an explicit at the end of a word, and the left */ if ((eBuildaroundType == BA_EMBEDDED) || (eBuildaroundType == BA_LEADING)) { bStemsAllowed = 1; } if (eSpecialType != LOCKPOINT) { bFound = __CaptureGetWord(pLingInfo, pLeftHandWord, wStartPoint, wLeftLength, bStemsAllowed); /* Was: if this is a leading or embedded special, and there is no top match found in the database, and exact is included in the list, use the default match as the left hand side. */ if (!bFound && (eBuildaroundType == BA_LEADING || eBuildaroundType == BA_EMBEDDED)) { ET9U16 wLen; WLOG2(fprintf(pLogFile2, "no capture found\n");) if ((wLeftLength == 1) && pWordSymbInfo->SymbsInfo[wStartPoint].bSymbType == ET9KTSMARTPUNCT) { WLOG2(fprintf(pLogFile2, "wLeftLength == 1 - special (embedded punct char)\n");) wLen = 1; sSymb = _ET9_GetEmbPunctChar(pLingInfo, pLingCmnInfo->Private.dwCurrActiveLanguage); WLOG2(fprintf(pLogFile2, "embedded symb %c (from GetEmbPunctChar lang %08x) - BAS\n", (char)sSymb, pLingCmnInfo->Private.dwCurrActiveLanguage);) _ET9SymCopy(pLeftHandWord->Base.sWord, &sSymb, 1); } else { const ET9U16 wCopyStartPoint = __CaptureGetFlushStringLength(pLingInfo, 0); ET9AssertLog(wStartPoint == __CaptureGetFlushPoint(pLingInfo, 0)); if (wStartPoint) { wLen = wLeftLength; } else { wLen = __CaptureGetDefaultStringLength(pLingInfo, wLeftLength + 1); } if (wLen + wCopyStartPoint > pDefaultWord->Base.wWordLen) { WLOG2(fprintf(pLogFile2, "default is too short - unusable\n");) wLen = 0; } else { WLOG2(fprintf(pLogFile2, "using %d symbs (%d chars) of default from pos %d (symb start %d)\n", wLeftLength, wLen, wCopyStartPoint, wStartPoint);) _ET9SymCopy(pLeftHandWord->Base.sWord, &pDefaultWord->Base.sWord[wCopyStartPoint], wLen); } } /* update if we ended up with something */ if (wLen) { /* don't need to set the rest of the elements of pLeftHandWord */ pLeftHandWord->Base.wWordLen = wLen; pLeftHandWord->Base.wWordCompLen = 0; WLOG2Word(pLogFile2, "__ET9BuildAroundSpecial, LeftHandWord (from prev default)", pLeftHandWord); bFound = 1; } else { bFound = 0; } } } else { ET9U16 wCount; ET9SYMB *psSymb = pLeftHandWord->Base.sWord; ET9SymbInfo *pSymbInfo = pWordSymbInfo->SymbsInfo; for (wCount = wStartPoint + wLeftLength; wCount; --wCount, ++psSymb, ++pSymbInfo) { *psSymb = pSymbInfo->sLockedSymb; } pLeftHandWord->Base.wWordLen = wStartPoint + wLeftLength; pLeftHandWord->Base.wWordCompLen = 0; /* all the elements of pLeftHandWord set */ WLOG2Word(pLogFile2, "__ET9BuildAroundSpecial, LeftHandWord (from lock)", pLeftHandWord); bFound = 1; bFoundFull = 1; /* no need to prepend */ } } /* at this point, if bFound, we have a top match term for symbs up to and including the special. lets build after the special. */ if (!bFound) { WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, no term found to build on (exit)\n");) return 0; } if (wRightLength) { ET9BOOL bLeftHandWordOk; WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, bFound & wRightLength, wRightLength = %d\n", wRightLength);) WLOG2Word(pLogFile2, "__ET9BuildAroundSpecial, LeftHandWord", pLeftHandWord); if (eBuildaroundType == BA_EMBEDDED) { pLeftHandWord->Base.sWord[pLeftHandWord->Base.wWordLen] = sEmbeddedChar; ++pLeftHandWord->Base.wWordLen; WLOG2Word(pLogFile2, "__ET9BuildAroundSpecial, LeftHandWord with embedded", pLeftHandWord); } /* build right side */ /* need to prepend lefthandword with flushed word */ if (wStartPoint && !bFoundFull) { bLeftHandWordOk = __ET9PrependWord(pLeftHandWord, pDefaultWord, __CaptureGetFlushStringLength(pLingInfo, 0)); WLOG2Word(pLogFile2, "__ET9BuildAroundSpecial, LeftHandWord prepended with default", pLeftHandWord); } else { bLeftHandWordOk = 1; } if (bLeftHandWordOk) { ET9U8 bLdbEntries; ET9U8 bSuppEntries; ET9U8 bSuppASEntries; bLdbEntries = 0; bSuppEntries = 0; bSuppASEntries = 0; pLingCmnInfo->Private.bStemsAllowed = __GetStemsAllowed(pLingCmnInfo, wRightLength); pLingCmnInfo->Private.wMaxWordLength = __GetMaxWordLength(pLingCmnInfo, wRightLength); __AssureLockedString(pLingInfo, pLeftHandWord, 0); __UpdateSpcInfo(pLingInfo, pLeftHandWord, 0, wLeftHandLen, bSpcMode); WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, performing _ET9AWSuppDBSelListBuild (buildaround non AS), index = %d, length = %d\n", wRightStartIndex, wRightLength);) _ET9AWSuppDBSelListBuild(pLingInfo, wRightStartIndex, wRightLength, &bSuppEntries, eBuildaroundType == BA_COMPOUND ? FREQ_BUILDCOMPOUND : FREQ_BUILDAROUND, ET9SUPPDB_NONAS_SOURCES, bSpcMode); WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, performing _ET9AWLdbWordsSearch (buildaround), index = %d, length = %d\n", wRightStartIndex, wRightLength);) _ET9AWLdbWordsSearch(pLingInfo, (pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum), wRightStartIndex, wRightLength, &bLdbEntries, eBuildaroundType == BA_COMPOUND ? FREQ_BUILDCOMPOUND : FREQ_BUILDAROUND, bSpcMode, 0); WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, performing _ET9AWSuppDBSelListBuild (buildaround AS), index = %d, length = %d\n", wRightStartIndex, wRightLength);) _ET9AWSuppDBSelListBuild(pLingInfo, wRightStartIndex, wRightLength, &bSuppASEntries, eBuildaroundType == BA_COMPOUND ? FREQ_BUILDCOMPOUND : FREQ_BUILDAROUND, ET9SUPPDB_AS_SOURCES, bSpcMode); WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, right side build results, bSuppEntries = %d, bSuppASEntries = %d, bLdbEntries = %d, bTotalWords = %d (exit)\n", (int)bSuppEntries, (int)bSuppASEntries, (int)bLdbEntries, (int)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords);) return (bSuppEntries || bSuppASEntries || bLdbEntries) ? 1 : 0; } else { WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, not lhw ok (exit)\n");) return 0; } } if (!wRightLength) { ET9BOOL bLeftHandWordOk; ET9AWPrivWordInfo TermEmbeddedWord; WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, bFound & !wRightLength\n");) if ((eBuildaroundType != BA_EMBEDDED) && eBuildaroundType != BA_COMPOUND) { WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, not embedded and not compound (exit)\n");) return 0; } if (wStartPoint && !bFoundFull) { bLeftHandWordOk = __ET9PrependWord(pLeftHandWord, pDefaultWord, __CaptureGetFlushStringLength(pLingInfo, 0)); WLOG2Word(pLogFile2, "__ET9BuildAroundSpecial, LeftHandWord prepended with default", pLeftHandWord); } else { bLeftHandWordOk = 1; } _InitPrivWordInfo(&TermEmbeddedWord); TermEmbeddedWord.Base.bIsTerm = 1; TermEmbeddedWord.Body.bWordSrc = ET9WORDSRC_LDB; if (eBuildaroundType == BA_EMBEDDED) { if (bLeftHandWordOk && pLeftHandWord->Base.wWordLen < ET9MAXWORDSIZE) { /* need to just add word with embedded character at the end */ pLeftHandWord->Base.sWord[pLeftHandWord->Base.wWordLen] = sEmbeddedChar; ++pLeftHandWord->Base.wWordLen; __UpdateSpcInfo(pLingInfo, pLeftHandWord, 0, wLeftHandLen, bSpcMode); WLOG2Word(pLogFile2, "__ET9BuildAroundSpecial, LeftHandWord with embedded", pLeftHandWord); pLingCmnInfo->Private.bStemsAllowed = 1; pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; __ET9AWSelLstInsert(pLingInfo, &TermEmbeddedWord, FREQ_BUILDAROUND, 0); WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, added term embedded word (exit)\n");) return 1; } } else { __UpdateSpcInfo(pLingInfo, pLeftHandWord, 0, wLeftHandLen, bSpcMode); pLingCmnInfo->Private.bStemsAllowed = 1; pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; __ET9AWSelLstInsert(pLingInfo, &TermEmbeddedWord, FREQ_BUILDCOMPOUND, 0); WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, added term lock (exit)\n");) return 1; } } WLOG2(fprintf(pLogFile2, "__ET9BuildAroundSpecial, nothing (exit)\n");) return 0; } /*---------------------------------------------------------------------------*/ typedef struct __InputPattern_s { ET9UINT nCountH; ET9UINT nCountV; ET9INT snDirInitH; ET9INT snDirCurrH; ET9INT snDirInitV; ET9INT snDirCurrV; } __InputPattern; /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static void ET9LOCALCALL __InitPattern(__InputPattern * const pPattern) { pPattern->nCountH = 0; pPattern->nCountV = 0; pPattern->snDirInitH = 0; pPattern->snDirCurrH = 0; pPattern->snDirInitV = 0; pPattern->snDirCurrV = 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __HandlePatternStep(__InputPattern * const pPattern, ET9TracePoint const * const pCurrPoint, ET9TracePoint const * const pPrevPoint) { if (!pPattern->snDirCurrH) { if (pCurrPoint->nX < pPrevPoint->nX) { pPattern->snDirInitH = -1; pPattern->snDirCurrH = -1; ++pPattern->nCountH; } else if (pCurrPoint->nX > pPrevPoint->nX) { pPattern->snDirInitH = +1; pPattern->snDirCurrH = +1; ++pPattern->nCountH; } } else if (pPattern->snDirCurrH < 0) { if (pCurrPoint->nX > pPrevPoint->nX) { pPattern->snDirCurrH = +1; ++pPattern->nCountH; } } else if (pPattern->snDirCurrH > 0) { if (pCurrPoint->nX < pPrevPoint->nX) { pPattern->snDirCurrH = -1; ++pPattern->nCountH; } } if (!pPattern->snDirCurrV) { if (pCurrPoint->nY < pPrevPoint->nY) { pPattern->snDirInitV = -1; pPattern->snDirCurrV = -1; ++pPattern->nCountV; } else if (pCurrPoint->nY > pPrevPoint->nY) { pPattern->snDirInitV = +1; pPattern->snDirCurrV = +1; ++pPattern->nCountV; } } else if (pPattern->snDirCurrV < 0) { if (pCurrPoint->nY > pPrevPoint->nY) { pPattern->snDirCurrV = +1; ++pPattern->nCountV; } } else if (pPattern->snDirCurrV > 0) { if (pCurrPoint->nY < pPrevPoint->nY) { pPattern->snDirCurrV = -1; ++pPattern->nCountV; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __AdjustForInputPattern(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; /* anything to do? */ if (!pWordSymbInfo->bNumSymbs) { return; } if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } if (!_ET9HasTraceOnlyInfo(pWordSymbInfo)) { return; } WLOG2B(fprintf(pLogFile2, "__AdjustForInputPattern\n");) /* investigate trace */ { ET9BOOL bInputUsingFree = 0; ET9FLOAT fTargetTraceLen = 0.0f; __InputPattern sTracePattern; __InitPattern(&sTracePattern); { ET9UINT nCount; ET9SymbInfo const *pSymbInfo; ET9TracePoint sPrevPoint = { 0 }; /* silly compiler warning */ ET9BOOL bIsFirst = 1; pSymbInfo = pWordSymbInfo->SymbsInfo; for (nCount = pWordSymbInfo->bNumSymbs; nCount; --nCount, ++pSymbInfo) { fTargetTraceLen += pSymbInfo->fTraceLen; if (pSymbInfo->bTraceProbability) { ET9TracePoint sPoint; _ET9KeyLookupKind eLookupKind; if (!_ET9_GetKeyPositionForSymbol(pWordSymbInfo, pSymbInfo->DataPerBaseSym[0].sChar[0], &sPoint, &eLookupKind)) { WLOG2B(fprintf(pLogFile2, " trace point %4u %4u kind %u\n", sPoint.nX, sPoint.nY, eLookupKind);) if (eLookupKind == ET9KeyLookupKind_Free) { bInputUsingFree = 1; } if (bIsFirst) { bIsFirst = 0; sPrevPoint = sPoint; continue; } __HandlePatternStep(&sTracePattern, &sPoint, &sPrevPoint); sPrevPoint = sPoint; } else { WLOG2B(fprintf(pLogFile2, " outside character value range, aborting\n");) return; } } } WLOG2B(fprintf(pLogFile2, " T trace bInputUsingFree %u\n", bInputUsingFree);) WLOG2B(fprintf(pLogFile2, " T tracelen %8.1f\n", fTargetTraceLen);) WLOG2B(fprintf(pLogFile2, " T trace nCountH %2d, snDirInitH %d\n", sTracePattern.nCountH, sTracePattern.snDirInitH);) WLOG2B(fprintf(pLogFile2, " T trace nCountV %2d, snDirInitV %d\n", sTracePattern.nCountV, sTracePattern.snDirInitV);) } /* investigate words */ { ET9UINT nOutsideRangeCount = 0; ET9UINT nIndex; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9UINT nCount; ET9SYMB *psSymb; ET9BOOL bIsFirst = 1; ET9FLOAT fTraceLen = 0.0f; ET9TracePoint sPrevPoint = { 0 }; /* silly compiler warning */ __InputPattern sWordPattern; #ifdef ET9_DEBUGLOG2B WLOG2String(pLogFile2, " ----- word", pIndexWord->Base.sWord, pIndexWord->Base.wWordLen, 0, 1); WLOG2B(fprintf(pLogFile2, "\n");) #endif if (!pIndexWord->Base.bIsTerm) { continue; } __InitPattern(&sWordPattern); psSymb = pIndexWord->Base.sWord; for (nCount = (ET9UINT)(pIndexWord->Base.wWordLen - pIndexWord->Base.wWordCompLen); nCount; --nCount, ++psSymb) { ET9TracePoint sPoint; _ET9KeyLookupKind eLookupKind; const ET9STATUS eStatus = _ET9_GetKeyPositionForSymbol(pWordSymbInfo, *psSymb, &sPoint, &eLookupKind); if (!eStatus) { WLOG2B(fprintf(pLogFile2, " word point %4u %4u kind %u\n", sPoint.nX, sPoint.nY, eLookupKind);) if (eLookupKind == ET9KeyLookupKind_Free && !bInputUsingFree) { WLOG2(fprintf(pLogFile2, " free and not tracing free - skipping\n");) continue; } if (bIsFirst) { bIsFirst = 0; sPrevPoint = sPoint; continue; } __HandlePatternStep(&sWordPattern, &sPoint, &sPrevPoint); fTraceLen += _ET9sqrt_f(((ET9FLOAT)sPoint.nX - (ET9FLOAT)sPrevPoint.nX) * ((ET9FLOAT)sPoint.nX - (ET9FLOAT)sPrevPoint.nX) + ((ET9FLOAT)sPoint.nY - (ET9FLOAT)sPrevPoint.nY) * ((ET9FLOAT)sPoint.nY - (ET9FLOAT)sPrevPoint.nY)); sPrevPoint = sPoint; } else { WLOG2(fprintf(pLogFile2, " char not found in KDB: %c\n", (unsigned char)*psSymb);) if (eStatus == ET9STATUS_OUT_OF_RANGE) { WLOG2(fprintf(pLogFile2, " outside character value range\n");) if (++nOutsideRangeCount > 10) { WLOG2(fprintf(pLogFile2, " aborting\n");) return; } } } } { const ET9FLOAT fLenDiff = (fTargetTraceLen - fTraceLen); const ET9FLOAT fLenDiffPercent = (fLenDiff / fTargetTraceLen * 100.0f); WLOG2B(fprintf(pLogFile2, " tracelen %8.1f diff %8.1f diff %6.2f%%\n", fTraceLen, fLenDiff, fLenDiffPercent);) if (fLenDiffPercent >= _ET9_MIXED_WEIGHT_TraceLenLo_trace && fLenDiffPercent <= _ET9_MIXED_WEIGHT_TraceLenHi_trace) { WLOG2B(fprintf(pLogFile2, " matching length\n");) #if 0 pIndexWord->Body.xTapFreq *= 10; #endif pIndexWord->Body.snCorrectionCost -= _ET9_MIXED_WEIGHT_TraceLen_trace; } } WLOG2B(fprintf(pLogFile2, " nCountH %2d, snDirInitH %d\n", sWordPattern.nCountH, sWordPattern.snDirInitH);) WLOG2B(fprintf(pLogFile2, " nCountV %2d, snDirInitV %d\n", sWordPattern.nCountV, sWordPattern.snDirInitV);) if (sWordPattern.nCountV == sTracePattern.nCountV && sWordPattern.snDirInitV == sTracePattern.snDirInitV) { WLOG2B(fprintf(pLogFile2, " matching V patern\n");) #if 0 pIndexWord->Body.xTapFreq *= 10; pIndexWord->Body.snCorrectionCost -= _ET9_MIXED_WEIGHT_Pattern_trace; #endif } if (sWordPattern.nCountH == sTracePattern.nCountH && sWordPattern.snDirInitH == sTracePattern.snDirInitH) { WLOG2B(fprintf(pLogFile2, " matching H patern\n");) pIndexWord->Body.xTapFreq *= 10; pIndexWord->Body.snCorrectionCost -= _ET9_MIXED_WEIGHT_Pattern_trace; } } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __ScaleTapFreqs(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; #ifdef ET9_USE_FLOAT_FREQS const ET9FREQPART xMaxValue = 1000.0f; const ET9FREQPART xMinFreq = 1e-32f; #else const ET9FREQPART xMaxValue = 10; const ET9FREQPART xMinFreq = 1; #endif ET9AssertLog(pLingCmnInfo != NULL); /* anything to do? */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } WLOG2(fprintf(pLogFile2, "__ScaleTapFreqs\n");) /* NWP? */ if (!pWordSymbInfo->bNumSymbs) { ET9UINT nIndex; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; const ET9FREQPART xFreq = (ET9FREQPART)1; WLOG2(fprintf(pLogFile2, " %5u: NWP - %7.2f (%9.2f)\n", nIndex, (float)xFreq, (float)pIndexWord->Body.xScaledTapFreq);) pIndexWord->Body.xScaledTapFreq = xFreq; } return; } /* scale */ { ET9UINT nIndex; ET9FREQPART xMaxFreq; xMaxFreq = 0; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9FREQPART xFreq = pIndexWord->Body.xScaledTapFreq; if (xFreq < xMinFreq) { xFreq = xMinFreq; } if (xMaxFreq < xFreq) { xMaxFreq = xFreq; } if (pIndexWord->Body.xScaledTapFreq != xFreq) { WLOG2(fprintf(pLogFile2, " %5u: limits - %9.2f (%9.2f)\n", nIndex, (float)xFreq, (float)pIndexWord->Body.xScaledTapFreq);) } pIndexWord->Body.xScaledTapFreq = xFreq; } for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9FREQPART xFreq = (ET9FREQPART)((pIndexWord->Body.xScaledTapFreq * xMaxValue) / xMaxFreq); if (xFreq < xMinFreq) { xFreq = xMinFreq; } WLOG2(fprintf(pLogFile2, " %5u: scaled - %7.2f (%9.2f)\n", nIndex, (float)xFreq, (float)pIndexWord->Body.xScaledTapFreq);) pIndexWord->Body.xScaledTapFreq = xFreq; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __CalculateInputScores(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; #ifdef ET9_USE_FLOAT_FREQS const ET9FREQPART xMaxScoreValue = 1000000000.0f; const ET9FREQPART xMinScoreValue = 1e-32f; #else const ET9FREQPART xMaxScoreValue = 65535; const ET9FREQPART xMinScoreValue = 1; #endif __ScaleTapFreqs(pLingInfo); __AdjustForInputPattern(pLingInfo); /* anything to do? */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return; } WLOG2(fprintf(pLogFile2, "__CalculateInputScores\n");) { const ET9BOOL bTraceBuild = pLingCmnInfo->Private.bTraceBuild; const ET9FLOAT fTapFreqExp = (bTraceBuild) ? _ET9_MIXED_WEIGHT_TapFreqExp_trace : _ET9_MIXED_WEIGHT_TapFreqExp_tap; const ET9FLOAT fCostFactor = (bTraceBuild) ? _ET9_MIXED_WEIGHT_CostFactor_trace : _ET9_MIXED_WEIGHT_CostFactor_tap; const ET9FLOAT fPowBase = (bTraceBuild) ? _ET9_MIXED_WEIGHT_PowBase_trace : _ET9_MIXED_WEIGHT_PowBase_tap; ET9UINT nIndex; ET9FREQPART xMaxScore = 0; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9FREQPART xInputScore; #ifdef ET9_USE_FLOAT_FREQS { const ET9FLOAT fMaxCorrectionExp = 32.0f; const ET9FLOAT fMinCorrectionExp = -32.0f; const ET9FLOAT fFinalTapFreq = _ET9pow_f(pWord->Body.xScaledTapFreq, fTapFreqExp); ET9FLOAT fCorrectionExp = (24.0f - pWord->Body.snCorrectionCost * fCostFactor); ET9FLOAT fCorrectionFactor; if (fCorrectionExp > 2 * fMaxCorrectionExp) { fCorrectionFactor = _ET9pow_f(fPowBase, 2 * fMaxCorrectionExp); xInputScore = (ET9FREQPART)(fCorrectionFactor * fFinalTapFreq); } else if (fCorrectionExp < 2 * fMinCorrectionExp) { xInputScore = xMinScoreValue; } else { fCorrectionFactor = _ET9pow_f(fPowBase, fCorrectionExp); xInputScore = (ET9FREQPART)(fCorrectionFactor * fFinalTapFreq); } } #else /* ET9_USE_FLOAT_FREQS */ { ET9INT snCorrectionExp = (ET9INT)(16 - pWord->Body.snCorrectionCost + pWord->Body.xScaledTapFreq); ET9FREQPART xCorrectionFactor; snCorrectionExp /= 2; if (snCorrectionExp < 0) { snCorrectionExp = 0; } else if (snCorrectionExp > 15) { snCorrectionExp = 15; } xCorrectionFactor = (ET9FREQPART)(1 << snCorrectionExp); xInputScore = (ET9FREQPART)xCorrectionFactor; } #endif /* ET9_USE_FLOAT_FREQS */ if (!nIndex) { xMaxScore = xInputScore; } else { if (xMaxScore < xInputScore) { xMaxScore = xInputScore; } } WLOG2(fprintf(pLogFile2, " %5u: initial - %14.2f\n", nIndex, (float)xInputScore);) pWord->Body.xInputScore = xInputScore; } if (xMaxScore == 0) { xMaxScore = 1; } for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pIndexWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9FREQPART xScore = (ET9FREQPART)((pIndexWord->Body.xInputScore * xMaxScoreValue) / xMaxScore); if (xScore < xMinScoreValue) { xScore = xMinScoreValue; } WLOG2(fprintf(pLogFile2, " %5u: scaled - %14.2f (%14.2f)\n", nIndex, (float)xScore, (float)pIndexWord->Body.xInputScore);) pIndexWord->Body.xInputScore = xScore; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __CalculateInputScoresSpm(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; pLingCmnInfo->Private.__SPATH_Status.bHasSpmScores = 0; pLingCmnInfo->Private.__SPATH_Status.nSpmModifiedWordCount = 0; pLingCmnInfo->Private.__SPATH_Status.nSpmClearedCompletionCount = 0; pLingCmnInfo->Private.__SPATH_Status.bHasShiftGesture = 0; pLingCmnInfo->Private.__SPATH_Status.bHasAllCapsGesture = 0; pLingCmnInfo->Base.pWordSymbInfo->Private.bHasShiftGesture = 0; pLingCmnInfo->Base.pWordSymbInfo->Private.bHasAllCapsGesture = 0; if (!pLingCmnInfo->Private.bStateSpmScoring || !pLingCmnInfo->Private.bTraceBuild) { WLOG2(fprintf(pLogFile2, "__CalculateInputScoresSpm, skip, not active or not trace, bStateSpmScoring %u, bTraceBuild %u\n", pLingCmnInfo->Private.bStateSpmScoring, pLingCmnInfo->Private.bTraceBuild);) return; } #ifdef ET9_SPM_MODULE #ifdef ET9_DEBUG { _ET9AW_WordCollection *pWordC_copy = &pLingCmnInfo->Private.sWordC_copy; *pWordC_copy = pLingCmnInfo->Private.sWordC; #endif { ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9U8 bTraceIndex = pWordSymbInfo->SymbsInfo[0].bTraceIndex; ET9UINT nCount; ET9SymbInfo const *pSymbInfo; __ProfileStart; if (!pWordSymbInfo->bNumSymbs) { return; } if (pWordSymbInfo->Private.sCurrTrace.bPointOverflow) { WLOG2(fprintf(pLogFile2, "__CalculateInputScoresSpm, skip, path overflow\n");) return; } pSymbInfo = pWordSymbInfo->SymbsInfo; for (nCount = pWordSymbInfo->bNumSymbs; nCount; --nCount, ++pSymbInfo) { if (!pSymbInfo->bTraceIndex || pSymbInfo->bTraceIndex != bTraceIndex) { return; } } WLOG2(fprintf(pLogFile2, "__CalculateInputScoresSpm, performing SPM scoring\n");) pLingCmnInfo->Private.__SPATH_Status.bHasSpmScores = _ET9AWSP_ScoreWords(pLingInfo); if (pLingCmnInfo->Private.__SPATH_Status.bHasSpmScores) { pLingCmnInfo->Base.pWordSymbInfo->Private.bHasShiftGesture = pLingCmnInfo->Private.__SPATH_Status.bHasShiftGesture; pLingCmnInfo->Base.pWordSymbInfo->Private.bHasAllCapsGesture = pLingCmnInfo->Private.__SPATH_Status.bHasAllCapsGesture; WLOG2(fprintf(pLogFile2, "__CalculateInputScoresSpm, bHasShiftGesture %u, bHasAllCapsGesture %u\n", pLingCmnInfo->Base.pWordSymbInfo->Private.bHasShiftGesture, pLingCmnInfo->Base.pWordSymbInfo->Private.bHasAllCapsGesture);) } __ProfileEnd(tAW_SelLst_SpmScoring); } #ifdef ET9_DEBUG if (!pLingCmnInfo->Private.__SPATH_Status.nSpmModifiedWordCount && !pLingCmnInfo->Private.__SPATH_Status.nSpmClearedCompletionCount) { ET9UINT nWordIndex; for (nWordIndex = 0; nWordIndex < ET9MAXCOLLECTSIZE; ++nWordIndex) { ET9UINT nCharIndex; ET9AssertLog(pWordC_copy->sPool.pWordList[nWordIndex].Base.wWordLen == pLingCmnInfo->Private.sWordC.sPool.pWordList[nWordIndex].Base.wWordLen); for (nCharIndex = 0; nCharIndex < pWordC_copy->sPool.pWordList[nWordIndex].Base.wWordLen; ++nCharIndex) { ET9AssertLog(pWordC_copy->sPool.pWordList[nWordIndex].Base.sWord[nCharIndex] == pLingCmnInfo->Private.sWordC.sPool.pWordList[nWordIndex].Base.sWord[nCharIndex]); } ET9AssertLog(_ET9ByteCheckSum((ET9U8*)&pWordC_copy->sPool.pWordList[nWordIndex].Base, (ET9U32)sizeof(pWordC_copy->sPool.pWordList[nWordIndex].Base)) == _ET9ByteCheckSum((ET9U8*)&pLingCmnInfo->Private.sWordC.sPool.pWordList[nWordIndex].Base, (ET9U32)sizeof(pWordC_copy->sPool.pWordList[nWordIndex].Base))); } } } #endif #ifdef ET9_DEBUG if (pLingCmnInfo->Private.__SPATH_Status.bHasSpmScores) { ET9UINT nIndex; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9AssertLog(pWord->Body.xInputScoreSpm > 0); } } #endif #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __DedupeAfterSpmShifting(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (!pLingCmnInfo->Private.__SPATH_Status.nSpmModifiedWordCount) { return; } #ifdef ET9_SPM_MODULE WLOG2(fprintf(pLogFile2, "__DedupeAfterSpmShifting, %u modified words\n", pLingCmnInfo->Private.__SPATH_Status.nSpmModifiedWordCount);) WLOG2(fprintf(pLogFile2, "__DedupeAfterSpmShifting, %u modified words (cleared comletions)\n", pLingCmnInfo->Private.__SPATH_Status.nSpmClearedCompletionCount);) { ET9UINT nIndex; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; __UpdateStringHash(pWord); } /* only testing hash values, different strings with same hash will be removed... */ /* not testing against substitutions - at least not now */ for (nIndex = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1; nIndex > 0; --nIndex) { const ET9U32 dwThisStringHash = pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]].Body.dwStringHash; ET9UINT nLook; for (nLook = 0; nLook < nIndex; ++nLook) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLook]]; if (pWord->Body.dwStringHash == dwThisStringHash) { __RemoveWord(pLingCmnInfo, nIndex); break; } } } } #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __CalculateInterpolationFactors(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9FLOAT fDLMConfidence = _ET9AW_DLM_GetConfidence(pLingInfo); pLingCmnInfo->Private.fInterpolFactDLM = fDLMConfidence * 1.0f; pLingCmnInfo->Private.fInterpolFactLDB = (1.0f - pLingCmnInfo->Private.fInterpolFactDLM); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __PreCalculateWSI(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9BOOL bLocked = 0; ET9INT snSymbIndex; for (snSymbIndex = pWordSymbInfo->bNumSymbs - 1; snSymbIndex >= 0; --snSymbIndex) { ET9SymbInfo * const pSymbInfo = &pWordSymbInfo->SymbsInfo[snSymbIndex]; bLocked |= pSymbInfo->bLocked; pSymbInfo->bCurrBuildLocked = bLocked; if (snSymbIndex && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[snSymbIndex - 1] && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[snSymbIndex - 1] <= pWordSymbInfo->bNumSymbs) { pSymbInfo->bCurrBuildAfterFlush = 1; } else { pSymbInfo->bCurrBuildAfterFlush = 0; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ #define __RestoreSelectionListOverrideValues(pLingCmnInfo) \ { \ pLingCmnInfo->Private.ASpc.bSpcFeatures = bSpcFeaturesSave; \ pLingCmnInfo->Private.ASpc.nMaxSpcTermCount = nMaxSpcTermCountSave; \ pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount = nMaxSpcCmplCountSave; \ pLingCmnInfo->Private.nMaxCompletionCount = nMaxCompletionCountSave; \ pLingCmnInfo->Private.wWordCompletionPoint = wWordCompletionPointSave; \ pLingCmnInfo->Private.bStateWordCompletion = bStateWordCompletionSave; \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWDoSelLstBuild(ET9AWLingInfo * const pLingInfo, ET9U8 * const pbTotalWords, const ET9BOOL bSuppressBuild, ET9U16 * const pwGestureValue) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9AWPrivWordInfo * const pLeftHandWord = &pLingCmnInfo->Private.sWordC.pCurrC->sLeftHandWord; ET9AWPrivWordInfo * const pPrevDefaultWord = &pLingCmnInfo->Private.sPrevDefaultWord; ET9UINT i; ET9U16 * const pwLockPoint = &pLingCmnInfo->Private.wCurrLockPoint; ET9U16 wFlushPoint; ET9UINT nFullWordMatches; ET9U16 wSpecialPosition; ET9ASPECIAL eSpecialType; ET9BOOL bExactLock; ET9BOOL bExactWon; ET9BOOL bHasTermPunct; ET9BOOL bHasTermSmartPunct; ET9BOOL bContinue; ET9BOOL bDefaultOkay; ET9BOOL bFound; ET9SymbInfo *pSymbInfo; ET9AWPrivWordInfo *pWord; ET9AWPrivWordInfo *pAmbigWord; ET9AWPrivWordInfo *pExactWord; ET9SimpleWord sSimpleWord; ET9U8 bSecondLangTermPunctFirst = 0; ET9U16 wSavedLockPoint = 0; const ET9U8 bNumSymbs = pWordSymbInfo->bNumSymbs; const ET9U8 bLastBuildLen = pLingCmnInfo->Private.bLastBuildLen; const ET9BOOL bLastInputIsTrace = (ET9BOOL)(bNumSymbs && pWordSymbInfo->SymbsInfo[bNumSymbs - 1].bTraceIndex); const ET9BOOL bLastInputIsMultitap = (ET9BOOL)(bNumSymbs && pWordSymbInfo->SymbsInfo[bNumSymbs - 1].eInputType == ET9MULTITAPKEY); const ET9BOOL bLastInputIsSmartPunct = (ET9BOOL)(bNumSymbs && pWordSymbInfo->SymbsInfo[bNumSymbs - 1].bSymbType == ET9KTSMARTPUNCT); const ET9BOOL bExactInList = __ExactInList(pLingInfo); const ET9BOOL bHasTraceInfo = _ET9HasTraceInfo(pWordSymbInfo); const ET9BOOL bHasRegionalInfo = _ET9HasRegionalInfo(pWordSymbInfo); const ET9BOOL bHasDiscreteOnlyInfo = _ET9HasDiscreteOnlyInfo(pWordSymbInfo); const ET9BOOL bHasAllShiftedInfo = _ET9HasAllShiftedInfo(pWordSymbInfo); const ET9BOOL bUsingLM = _ET9AW_IsUsingLM(pLingInfo); const ET9BOOL bUsingALM = bUsingLM && _ET9AW_IsUsingALM(pLingInfo); const ET9BOOL bUsingDLM = _ET9AW_DLM_HasALMConfidence(pLingInfo); const ET9BOOL bUsingMGD = _ET9AW_IsPrimaryMGD(pLingInfo); const ET9BOOL bHasOverrides = (ET9BOOL)(bHasTraceInfo || ((bUsingALM || bUsingDLM) && !pLingCmnInfo->Private.bStateContextBasedPredictionManualMode)); const ET9U8 bSpcMode = (ET9U8)(pWordSymbInfo->Private.bRequiredInhibitCorrection ? ET9ASPCMODE_OFF : bHasOverrides ? ET9ASPCMODE_REGIONAL : pLingCmnInfo->Private.ASpc.eMode); const ET9BOOL bStateWordCompletionSave = pLingCmnInfo->Private.bStateWordCompletion; /* restore on return */ const ET9U8 bSpcFeaturesSave = pLingCmnInfo->Private.ASpc.bSpcFeatures; /* restore on return */ const ET9UINT nMaxSpcTermCountSave = pLingCmnInfo->Private.ASpc.nMaxSpcTermCount; /* restore on return */ const ET9UINT nMaxSpcCmplCountSave = pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount; /* restore on return */ const ET9UINT nMaxCompletionCountSave = pLingCmnInfo->Private.nMaxCompletionCount; /* restore on return */ const ET9U16 wWordCompletionPointSave = pLingCmnInfo->Private.wWordCompletionPoint; /* restore on return */ __ProfileStart; ET9AssertLog(pLingInfo); ET9AssertLog(pbTotalWords); ET9AssertLog(pwGestureValue); ET9AssertLog(bSpcMode <= ET9ASPCMODE_REGIONAL); /* init */ __PreCalculateWSI(pLingInfo); __WordOrder_Initialize(pLingCmnInfo); /* get the current language */ ET9AWLdbGetActiveLanguage(pLingInfo, &pLingCmnInfo->Private.dwCurrActiveLanguage); /* interpolation factors */ __CalculateInterpolationFactors(pLingInfo); /* log build state */ WLOG2(fprintf(pLogFile2, "__ET9AWDoSelLstBuild, START *****\n");) WLOG2(fprintf(pLogFile2, "bManualMode = %c, bHasOverrides %c\n", pLingCmnInfo->Private.bStateContextBasedPredictionManualMode ? 'Y' : 'N', bHasOverrides ? 'Y' : 'N');) WLOG2(fprintf(pLogFile2, "bNumSymbs = %d, bLastBuildLen = %d, dwFirstLdbNum = %08X, dwSecondLdbNum = %08X, dwCurrActiveLanguage %08X, dwPrevWordLanguage = %08X, LDB-enabled %u\n", bNumSymbs, bLastBuildLen, pLingCmnInfo->dwFirstLdbNum, pLingCmnInfo->dwSecondLdbNum, pLingCmnInfo->Private.dwCurrActiveLanguage, pLingCmnInfo->Private.dwPrevWordLanguage, ET9LDBENABLED(pLingCmnInfo));) WLOG2(fprintf(pLogFile2, "exactInList = %c, exactLast = %c, exactDefault = %c\n", (ET9EXACTINLIST(pLingCmnInfo) ? 'Y' : 'N'), (ET9EXACTLAST(pLingCmnInfo) ? 'Y' : 'N'), (ET9EXACTISDEFAULT(pLingCmnInfo) ? 'Y' : 'N'));) WLOG2(fprintf(pLogFile2, "maxCompletionCount = %d, maxSpcTermCount = %d, maxSpcCmplCount = %d\n", pLingCmnInfo->Private.nMaxCompletionCount, pLingCmnInfo->Private.ASpc.nMaxSpcTermCount, pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount);) WLOG2(fprintf(pLogFile2, "wordStemsActive = %c, wWordStemsPoint = %d\n", (ET9WORDSTEMS_MODE(pLingCmnInfo) ? 'Y' : 'N'), pLingCmnInfo->Private.wWordStemsPoint);) WLOG2(fprintf(pLogFile2, "wordCompletionActive = %c, wWordCompletionPoint = %d\n", (ET9WORDCOMPLETION_MODE(pLingCmnInfo) ? 'Y' : 'N'), pLingCmnInfo->Private.wWordCompletionPoint);) WLOG2(fprintf(pLogFile2, "SPC eMode = %s (%s), eSearchFilter = %s\n", SPCMODETOSTRING(pLingCmnInfo->Private.ASpc.eMode), SPCMODETOSTRING(bSpcMode), SPCFILTERTOSTRING(pLingCmnInfo->Private.ASpc.eSearchFilter));) WLOG2(fprintf(pLogFile2, "eAltMode = %s\n", ALTMODETOSTRING(pLingCmnInfo->Private.eAltMode));) WLOG2(fprintf(pLogFile2, "eSelectionListMode = %s\n", SELLSTMODETOSTRING(pLingCmnInfo->Private.eSelectionListMode));) WLOG2(fprintf(pLogFile2, "eSelectionListCorrectionMode = %s\n", SELLSTCORRMODETOSTRING(pLingCmnInfo->Private.eSelectionListCorrectionMode));) WLOG2(fprintf(pLogFile2, "OTFM = %c\n", (!pLingInfo->Private.pConvertSymb ? 'N' : 'Y'));) WLOG2(fprintf(pLogFile2, "postSort = %c\n", ET9POSTSORTENABLED(pLingCmnInfo) ? 'Y' : 'N');) WLOG2(fprintf(pLogFile2, "bHasTraceInfo = %c, bHasRegionalInfo = %c, bUsingLM = %c, bUsingALM = %c, bUsingDLM = %c\n", (bHasTraceInfo ? 'Y' : 'N'), (bHasRegionalInfo ? 'Y' : 'N'), (bUsingLM ? 'Y' : 'N'), (bUsingALM ? 'Y' : 'N'), (bUsingDLM ? 'Y' : 'N'));) WLOG2(fprintf(pLogFile2, "bRequiredLocate = %c\n", pWordSymbInfo->Private.bRequiredLocate ? 'Y' : 'N');) WLOG2(fprintf(pLogFile2, "bSuppressBuild = %c\n", bSuppressBuild ? 'Y' : 'N');) /* handle tracking */ pLingCmnInfo->Private.bDoneBuilding = 1; if (bSpcMode) { pLingCmnInfo->Private.bSpcDuringBuild = 1; /* since there is a local override above */ } /* set current selection list mode */ if (pLingCmnInfo->Private.eSelectionListMode == ET9ASLMODE_AUTO) { if (bHasTraceInfo) { if (bHasRegionalInfo) { pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_MIXED; } else { pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_MIXED; } } else if (bUsingALM || bUsingDLM) { pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_MIXED; } else { switch (pLingCmnInfo->Private.eSelectionListCorrectionMode) { case ET9ASLCORRECTIONMODE_LOW: pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_COMPLETIONSPROMOTED; break; case ET9ASLCORRECTIONMODE_HIGH: case ET9ASLCORRECTIONMODE_MEDIUM: if (bHasRegionalInfo) { switch (pLingCmnInfo->Private.eAltMode) { case ET9AW_AltMode_1: pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_MIXED; break; case ET9AW_AltMode_2: pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_CLASSIC; break; default: ET9AssertLog(0); pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_MIXED; break; } } else { pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_CLASSIC; } break; default: ET9AssertLog(0); pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_CLASSIC; break; } } } else { if (bHasTraceInfo) { pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_MIXED; } else { pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = pLingCmnInfo->Private.eSelectionListMode; } } WLOG2(fprintf(pLogFile2, "base mode, eCurrSelectionListMode = %s\n", SELLSTMODETOSTRING(pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode));) /* potentially override current selection list mode */ if (!bNumSymbs) { WLOG2(fprintf(pLogFile2, "NWP -> ET9ASLMODE_MIXED (really NWP)\n");) pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_MIXED; } else if (bHasTraceInfo) { WLOG2(fprintf(pLogFile2, "input has trace info -> prevent further overrides\n");) } else if (bLastInputIsSmartPunct) { WLOG2(fprintf(pLogFile2, "last input SMARTPUNCT -> ET9ASLMODE_CLASSIC\n");) pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_CLASSIC; } else if (bLastInputIsMultitap) { WLOG2(fprintf(pLogFile2, "last input multitap -> ET9ASLMODE_COMPLETIONSPROMOTED\n");) pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_COMPLETIONSPROMOTED; } else if (pLingCmnInfo->dwSecondLdbNum && ((pLingCmnInfo->dwSecondLdbNum & ET9PLIDMASK) != ET9PLIDNone)) { if (bUsingALM || bUsingDLM) { WLOG2(fprintf(pLogFile2, "bilingual -> NOT switching to ET9ASLMODE_COMPLETIONSPROMOTED (ALM/DLM)\n");) } else { WLOG2(fprintf(pLogFile2, "bilingual -> ET9ASLMODE_COMPLETIONSPROMOTED\n");) pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_COMPLETIONSPROMOTED; } } if (bNumSymbs == 1 && pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_MIXED) { if ((bUsingALM || bUsingDLM) && !bLastInputIsMultitap && bLastInputIsSmartPunct) { WLOG2(fprintf(pLogFile2, "input length one - but ALM/DLM, NOT changing ET9ASLMODE_MIXED to ET9ASLMODE_CLASSIC\n");) } else { WLOG2(fprintf(pLogFile2, "input length one, ET9ASLMODE_MIXED -> ET9ASLMODE_CLASSIC\n");) pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_CLASSIC; } } /* setup collections and size - after deciding selection list mode */ { ET9UINT nTargetCollectSize = 0; /* zero means max */ if (bLastInputIsMultitap) { nTargetCollectSize = __ET9Min(64, ET9MAXCOLLECTSIZE); } #if 1 /* could be removed when performance improves */ else if (!bHasTraceInfo) { nTargetCollectSize = __ET9Min(64, ET9MAXCOLLECTSIZE); } #endif else if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode != ET9ASLMODE_MIXED) { nTargetCollectSize = __ET9Min(64, ET9MAXCOLLECTSIZE); } _ET9AW_InitWordCollections(pLingInfo, nTargetCollectSize, _ET9AW_IsUsingMGD(pLingInfo)); WLOG2(fprintf(pLogFile2, "Word collection set sizes: current %u aux %u (max %u)\n", pLingCmnInfo->Private.sWordC.pCollections[0].nMaxCollectSize, pLingCmnInfo->Private.sWordC.pCollections[1].nMaxCollectSize, ET9MAXCOLLECTSIZE);) } /* save some useful values for use by other functions */ pLingCmnInfo->Private.bUsingLM = bUsingLM; pLingCmnInfo->Private.bUsingALM = bUsingALM; pLingCmnInfo->Private.bUsingDLM = bUsingDLM; pLingCmnInfo->Private.bUsingMGD = bUsingMGD; pLingCmnInfo->Private.bHasRegionalInfo = bHasRegionalInfo; pLingCmnInfo->Private.bHasDiscreteOnlyInfo = bHasDiscreteOnlyInfo; pLingCmnInfo->Private.bTraceBuild = bHasTraceInfo; pLingCmnInfo->Private.bHasAllShiftedInfo = bHasAllShiftedInfo; pLingCmnInfo->Private.bActiveIndex = (pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage) ? ET9AWSECOND_LANGUAGE : ET9AWFIRST_LANGUAGE; pLingCmnInfo->Private.bNonActiveIndex = (pLingCmnInfo->Private.bActiveIndex == ET9AWFIRST_LANGUAGE) ? ET9AWSECOND_LANGUAGE : ET9AWFIRST_LANGUAGE; /* potentially override some system settings */ { /* flex features */ if (bHasTraceInfo) { pLingCmnInfo->Private.ASpc.bSpcFeatures = ET9_FLEX_FEATURE_ALL_MASK; } else if (bUsingMGD) { /* use flex */ if (!pLingCmnInfo->Private.ASpc.bSpcFeatures) { pLingCmnInfo->Private.ASpc.bSpcFeatures = ET9_FLEX_FEATURE_ACTIVATE_MASK | ET9_FLEX_FEATURE_FREE_PUNCT_MASK | ET9_FLEX_FEATURE_OPTIONAL_MASK; } } else if (bSpcMode && bNumSymbs == 2 && _ET9_LanguageSpecific_ApplySpcLenTwo(pLingInfo)) { /* use flex to get limited spell correction at length 2 for tap */ if (!pLingCmnInfo->Private.ASpc.bSpcFeatures) { pLingCmnInfo->Private.ASpc.bSpcFeatures = ET9_FLEX_FEATURE_ACTIVATE_MASK | ET9_FLEX_FEATURE_FREE_PUNCT_MASK; } } /* spell correction properties */ if (bHasTraceInfo) { const ET9UINT nMinSpcTermCount = pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize; const ET9UINT nMinSpcCmplCount = pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize; if (pLingCmnInfo->Private.ASpc.nMaxSpcTermCount < nMinSpcTermCount) { pLingCmnInfo->Private.ASpc.nMaxSpcTermCount = nMinSpcTermCount; } if (pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount < nMinSpcCmplCount) { pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount = nMinSpcCmplCount; } } else if (pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_MIXED) { const ET9UINT nMinSpcTermCount = pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize; const ET9UINT nMinSpcCmplCount = pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize; if (pLingCmnInfo->Private.ASpc.nMaxSpcTermCount < nMinSpcTermCount) { pLingCmnInfo->Private.ASpc.nMaxSpcTermCount = nMinSpcTermCount; } if (pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount < nMinSpcCmplCount) { pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount = nMinSpcCmplCount; } } /* word completion properties */ if (!bLastInputIsMultitap) { const ET9BOOL bStateWordCompletion = (bHasOverrides) ? 1 : 0; const ET9UINT nMinCompletionCount = (bUsingALM || bUsingDLM || pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode == ET9ASLMODE_MIXED) ? pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize : (bHasTraceInfo ? 5 : 1); const ET9U16 wMaxWordCompletionPoint = (!bHasOverrides) ? ET9MAXWORDSIZE : (bUsingALM || bUsingDLM) ? 1 : (bHasTraceInfo) ? 2 : ET9MAXWORDSIZE; if (pLingCmnInfo->Private.wWordCompletionPoint > wMaxWordCompletionPoint) { pLingCmnInfo->Private.wWordCompletionPoint = wMaxWordCompletionPoint; } if (pLingCmnInfo->Private.nMaxCompletionCount < nMinCompletionCount) { pLingCmnInfo->Private.nMaxCompletionCount = nMinCompletionCount; } if (!pLingCmnInfo->Private.bStateWordCompletion) { pLingCmnInfo->Private.bStateWordCompletion = bStateWordCompletion; } } WLOG2(fprintf(pLogFile2, "word completion properties, bStateWordCompletion %u, wWordCompletionPoint %u, nMaxCompletionCount %u\n", pLingCmnInfo->Private.bStateWordCompletion, pLingCmnInfo->Private.wWordCompletionPoint, pLingCmnInfo->Private.nMaxCompletionCount);) } /* verify efficient setup */ ET9AssertLog(pLingCmnInfo->Private.ASpc.nMaxSpcTermCount == pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize || pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize <= 64); ET9AssertLog(pLingCmnInfo->Private.ASpc.nMaxSpcCmplCount == pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize || pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize <= 64); ET9AssertLog(pLingCmnInfo->Private.nMaxCompletionCount == pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize || pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize <= 64); /* if looking for the required word, make sure the last known shift state matches */ if (pWordSymbInfo->Private.bRequiredLocate && bNumSymbs) { if (pWordSymbInfo->Private.eLastShiftState != pWordSymbInfo->Private.eRequiredLastShiftState) { WLOG2(fprintf(pLogFile2, "required's last shift state applied (old = %d)\n", pWordSymbInfo->Private.eLastShiftState);) } pWordSymbInfo->Private.eLastShiftState = pWordSymbInfo->Private.eRequiredLastShiftState; } WLOG2(fprintf(pLogFile2, "bLastShiftState = %d\n", pWordSymbInfo->Private.eLastShiftState);) WLOG2(fprintf(pLogFile2, "\n");) /* init result values */ *pbTotalWords = 0; *pwGestureValue = 0; pLingCmnInfo->Private.bRequiredFound = 0; pLingCmnInfo->Private.sWordC.pCurrC->bHasRealWord = 0; pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts = 0; pLingCmnInfo->Private.bSpcComplDuringSingleBuild = 0; pLingCmnInfo->Private.bCurrBuildHasShiftSignificance = 0; /* init latency counters */ /* init left hand word */ _InitPrivWordInfo(pLeftHandWord); /* verify some essentials */ if (!bNumSymbs && !ET9NEXTWORDPREDICTION_MODE(pLingCmnInfo)) { WLOG2(fprintf(pLogFile2, "no symbs and no prediction - aborting\n");) _ET9AW_ResetWordList(pLingInfo); __RestoreSelectionListOverrideValues(pLingCmnInfo); return ET9STATUS_NONE; } if (bNumSymbs > ET9MAXWORDSIZE) { WLOG2(fprintf(pLogFile2, "too many symbs - aborting\n");) _ET9AW_ResetWordList(pLingInfo); __RestoreSelectionListOverrideValues(pLingCmnInfo); return ET9STATUS_ERROR; } /* verify that a reselected word is compatible with current setting for spell correction */ if (pWordSymbInfo->Private.bRequiredVerifyInput && bNumSymbs) { if (pWordSymbInfo->Private.sRequiredWord.wLen < bNumSymbs && !ET9_SPC_IS_ACTIVE(bSpcMode)) { WLOG2(fprintf(pLogFile2, "aborting reselect build, spc mode not compatible (off)\n");) __RestoreSelectionListOverrideValues(pLingCmnInfo); return ET9STATUS_NONE; } } /* reset word list? */ if (!bLastBuildLen) { _ET9AW_ResetWordList(pLingInfo); } /* ok to start building */ pLingCmnInfo->Private.bDoneBuilding = 0; /* a list build restores default post shift mode */ pWordSymbInfo->Private.eCurrPostShiftMode = ET9POSTSHIFTMODE_DEFAULT; /* set build direction */ if (!bNumSymbs) { pLingCmnInfo->Private.bLastBuildShrinking = 0; WLOG2(fprintf(pLogFile2, "bLastBuildShrinking = %d (empty input)\n", pLingCmnInfo->Private.bLastBuildShrinking);) } else if (bNumSymbs != bLastBuildLen) { pLingCmnInfo->Private.bLastBuildShrinking = (ET9BOOL)(bLastBuildLen > bNumSymbs); WLOG2(fprintf(pLogFile2, "bLastBuildShrinking = %d (%u > %u)\n", pLingCmnInfo->Private.bLastBuildShrinking, bLastBuildLen, bNumSymbs);) } else { WLOG2(fprintf(pLogFile2, "bLastBuildShrinking = %d (not modified)\n", pLingCmnInfo->Private.bLastBuildShrinking);) } /* handle (turn off) symbol invalidation flag */ ET9AssertLog( /* empty */ !bNumSymbs || /* same length one is ok any way */ bNumSymbs == 1 && bLastBuildLen == 1 || /* growing and invalidated (or locked) */ bNumSymbs > bLastBuildLen && (pLingCmnInfo->Base.bSymbInvalidated[bNumSymbs-1] || 1 /* complicated to keep track of when a locked removed the invalidation */) || /* shrinking or same and NOT invalidated */ bNumSymbs <= bLastBuildLen && !pLingCmnInfo->Base.bSymbInvalidated[bNumSymbs-1] ); if (bNumSymbs) { pLingCmnInfo->Base.bSymbInvalidated[bNumSymbs-1] = 0; } /* find the latest flush point, and lockpoint */ /* if clearing syms, save lockpoint for comparison */ if (bNumSymbs < bLastBuildLen && *pwLockPoint) { wSavedLockPoint = *pwLockPoint; } *pwLockPoint = 0; bExactLock = 0; { ET9U8 bIndex; bIndex = bNumSymbs; if (bIndex) { for (pSymbInfo = &pWordSymbInfo->SymbsInfo[bIndex - 1]; bIndex; --bIndex, --pSymbInfo) { /* ignore invalid Insure++ warnings */ if (pSymbInfo->bLocked) { *pwLockPoint = bIndex; break; } } /* if the lock point (latest only) is exact lock, then "lock" the exact (default index to the exact) */ if (*pwLockPoint && pSymbInfo->bLocked == ET9EXACTLOCK) { bExactLock = 1; } } } /* if clearing sym caused loss of lockpoint (or we hit a flush point ) */ /* then we want to deactivate downshifting that was in force for previous righthand word */ if (!bNumSymbs || (bNumSymbs < bLastBuildLen && ((*pwLockPoint != wSavedLockPoint) || pLingCmnInfo->Private.sBuildInfo.pbFlushPos[bNumSymbs-1]))) { pWordSymbInfo->Private.bCompoundingDownshift = 0; } wFlushPoint = __CaptureGetFlushPoint(pLingInfo, 0); WLOG2(fprintf(pLogFile2, "FlushPoint = %d, LockPoint = %d, ExactLock = %d\n", wFlushPoint, *pwLockPoint, bExactLock);) /* Record the default object. This is designed to catch the previously highlighted word. If there was lock, that will be the default. */ bDefaultOkay = 0; _InitPrivWordInfo(pPrevDefaultWord); if (bNumSymbs == 1 && !pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { /* make the default exist in this first build of first symb (empty) */ bDefaultOkay = 1; WLOG2Word(pLogFile2, "default empty on first build of first symb", pPrevDefaultWord); } else if (bNumSymbs && pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { /* first check to see of there was a lock, effectively overriding any default index. */ if ((bNumSymbs && pWordSymbInfo->SymbsInfo[bNumSymbs - 1].bLocked) || (bNumSymbs > 1 && pWordSymbInfo->SymbsInfo[bNumSymbs - 2].bLocked)) { ET9U16 wCount; ET9SYMB *psSymb = pPrevDefaultWord->Base.sWord; ET9SymbInfo *pSymbInfo = pWordSymbInfo->SymbsInfo; for (wCount = (ET9U16)(bNumSymbs - 1); wCount; --wCount, ++psSymb, ++pSymbInfo) { *psSymb = pSymbInfo->sLockedSymb; } pPrevDefaultWord->Base.wWordLen = (ET9U16)(bNumSymbs - 1); pPrevDefaultWord->Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; __CaptureDefault(pLingInfo, pPrevDefaultWord, bNumSymbs, 0); /* should we assign a word source here? */ bDefaultOkay = 1; WLOG2Word(pLogFile2, "default from lock", pPrevDefaultWord); } else if (bLastBuildLen >= bNumSymbs && __CaptureGetDefault(pLingInfo, pPrevDefaultWord, bNumSymbs)) { bDefaultOkay = 1; WLOG2Word(pLogFile2, "default from capture", pPrevDefaultWord); } else { if (bLastBuildLen >= bNumSymbs) { pWord = NULL; } else if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords && pLingCmnInfo->Private.nDefaultIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { ET9UINT nIndex = pLingCmnInfo->Private.nDefaultIndex; for (pWord = NULL; !pWord && nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; if (pWord->Body.bIsSegment || GETRAWSRC(pWord->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE) { pWord = NULL; } } } else { pWord = NULL; } if (pWord && pWord->Body.bWordSrc != ET9WORDSRC_MAGICSTRING) { *pPrevDefaultWord = *pWord; bDefaultOkay = 1; ET9AssertLog(pPrevDefaultWord->Base.wWordLen); pPrevDefaultWord->Body.bLangIndexScoring = pWord->Body.bLangIndexScoring; WLOG2(fprintf(pLogFile2, "default index (%d)\n", pLingCmnInfo->Private.nDefaultIndex);) WLOG2Word(pLogFile2, "default from default index", pPrevDefaultWord); } else { _InitPrivWordInfo(pPrevDefaultWord); if (!ET9GetExactWord(pWordSymbInfo, &sSimpleWord, pLingInfo->Private.pConvertSymb, pLingInfo->Private.pConvertSymbInfo)) { _ET9SimpleWordToPrivWord(&sSimpleWord, pPrevDefaultWord); if (pPrevDefaultWord->Base.wWordLen > bNumSymbs - 1) { pPrevDefaultWord->Base.wWordLen = (ET9U16)(bNumSymbs - 1); } else if (pPrevDefaultWord->Base.wWordLen) { --pPrevDefaultWord->Base.wWordLen; } pPrevDefaultWord->Base.wWordCompLen = 0; bDefaultOkay = 1; WLOG2Word(pLogFile2, "default from exact", pPrevDefaultWord); } else { WLOG2(fprintf(pLogFile2, "failed to get exact for default use\n");) /* no support for default by question marks... */ } } /* make sure the flush part is intact */ if (bDefaultOkay) { ET9AWPrivWordInfo sFlush; __CaptureGetFlush(pLingInfo, &sFlush); if (sFlush.Base.wWordLen) { _ET9SymCopy(pPrevDefaultWord->Base.sWord, sFlush.Base.sWord, sFlush.Base.wWordLen); if (sFlush.Base.wWordLen > pPrevDefaultWord->Base.wWordLen) { pPrevDefaultWord->Base.wWordLen = sFlush.Base.wWordLen; pPrevDefaultWord->Base.wWordCompLen = 0; } } } /* make sure the locked part is intact */ __AssureLockedString(pLingInfo, pPrevDefaultWord, 0); } } /* END: Record the default object.*/ if (!bDefaultOkay) { pPrevDefaultWord->Base.wWordLen = 0; } WLOG2(fprintf(pLogFile2, "DefaultOkay = %d\n", bDefaultOkay);) WLOG2Word(pLogFile2, "PrevDefaultWord", pPrevDefaultWord); if (bDefaultOkay) { __VerifyLockedSymbs(pLingInfo, pPrevDefaultWord, 0, bNumSymbs); } if (bNumSymbs > bLastBuildLen) { /* this assert is hard to apply when allowing completions for term punct... */ ET9AssertLog(bHasTraceInfo || !bDefaultOkay || (pPrevDefaultWord->Base.wWordLen - pPrevDefaultWord->Base.wWordCompLen) == bNumSymbs - 1 || pLingCmnInfo->Private.bSpcDuringBuild || pLingCmnInfo->Private.bExpandAsDuringBuild); } /* handle capture actions (especially when rewinding) */ __CaptureHandleAction(pLingInfo); __PreCaptureWord(pLingInfo); __CaptureDefault(pLingInfo, pPrevDefaultWord, bNumSymbs, bLastBuildLen); /* handle (new) word capture Here we are capturing the strings before and including special characters. Note that what is captured may not be the same length as the number of keys presses because of spell correction. */ if (bDefaultOkay) { if (bNumSymbs > bLastBuildLen) { ET9U16 wExcludePoint; wExcludePoint = bNumSymbs; if (wExcludePoint) { --wExcludePoint; if (wExcludePoint) { --wExcludePoint; } } /* nothing can be captured unless there is a special tail (non nospecial) */ __ET9GetSpecialCharInfo(pLingInfo, wExcludePoint, &wSpecialPosition, &eSpecialType); if (eSpecialType != NOSPECIAL) { const ET9U16 wFlushStringLen = __CaptureGetFlushStringLength(pLingInfo, 1); ET9ASPECIAL eSpecialTypeDummy; ET9AWPrivWordInfo *pCaptureWord = NULL; WLOG2(fprintf(pLogFile2, "symbol tail indicates possible capture\n");) /* check if actually accepting completions */ __ET9GetSpecialCharInfo(pLingInfo, 0, &wSpecialPosition, &eSpecialTypeDummy); if (bNumSymbs && wSpecialPosition == (bNumSymbs - 1)) { /* bNumSymbs zero would be ok anyway... */ WLOG2(fprintf(pLogFile2, "wSpecialPosition indicates capture completion prevention\n");) if (!pPrevDefaultWord->Base.wWordCompLen && ISCAPTURECANDSRC(pPrevDefaultWord->Body.bWordSrc) && pPrevDefaultWord->Base.wWordLen > wFlushStringLen) { WLOG2(fprintf(pLogFile2, "capture word - from prev defualt\n");) pCaptureWord = pPrevDefaultWord; } else { ET9UINT nStartIndex = pLingCmnInfo->Private.nDefaultIndex; if (nStartIndex && ISREALSRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[0]].Body.bWordSrc)) { nStartIndex = 0; } for (i = nStartIndex; i < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++i) { pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[i]]; if (!pWord->Base.wWordCompLen && ISCAPTURECANDSRC(pWord->Body.bWordSrc) && (pWord->Base.wWordLen > wFlushStringLen) && ISREALSRC(pWord->Body.bWordSrc)) { WLOG2(fprintf(pLogFile2, "capture word - ISCAPTURECANDSRC\n");) pCaptureWord = pWord; break; } } if (pCaptureWord == NULL) { for (i = nStartIndex; i < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++i) { pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[i]]; if (!pWord->Base.wWordCompLen && ISALTCAPTURECANDSRC(pWord->Body.bWordSrc) && (pWord->Base.wWordLen > wFlushStringLen) && GETBASESRC(pWord->Body.bWordSrc) != ET9WORDSRC_NONE && GETBASESRC(pWord->Body.bWordSrc) != ET9WORDSRC_EXACT_LAST) { WLOG2(fprintf(pLogFile2, "capture word - ISALTCAPTURECANDSRC\n");) pCaptureWord = pWord; break; } } } } if (pCaptureWord) { __CaptureWord(pLingInfo, pCaptureWord, (ET9U16)(bNumSymbs - 1)); } else if (bHasTraceInfo) { __CaptureWord(pLingInfo, pPrevDefaultWord, (ET9U16)(bNumSymbs - 1)); } } else { WLOG2(fprintf(pLogFile2, "wSpecialPosition indicates capture completions allowed - using default\n");) if (pPrevDefaultWord->Base.wWordLen <= wFlushStringLen) { /* should we do a more elaborate search here instead? */ WLOG2(fprintf(pLogFile2, "default is too short for capturing...\n");) } else { __CaptureWord(pLingInfo, pPrevDefaultWord, (ET9U16)(bNumSymbs - 1)); } } } } } /* now we can reset the list */ _ET9AW_ResetWordList(pLingInfo); /* at this point we can potentially abort when the build is suppressed */ if (bSuppressBuild) { /* abort */ WLOG2(fprintf(pLogFile2, "aborting build, suppressed\n");) __ProfileEnd(tAW_SelLst_BuildSearch); __RestoreSelectionListOverrideValues(pLingCmnInfo); return ET9STATUS_NONE; } /* prepare matching info */ __MakeSymbFreqsValid(pLingCmnInfo); /* handle magic string */ if (pWordSymbInfo->bNumSymbs == ET9MAXLDBWORDSIZE) { if (_ET9IsMagicStringKey(pWordSymbInfo)) { __ET9AWAddMagicStr(pLingInfo); } } /* add exact match to list */ pLingCmnInfo->Private.nExactIndex = ET9UINT_MAX; if (bNumSymbs && (bExactInList || !bDefaultOkay)) { ET9AWPrivWordInfo sExactWord; _InitPrivWordInfo(&sExactWord); if (!ET9GetExactWord(pWordSymbInfo, &sSimpleWord, pLingInfo->Private.pConvertSymb, pLingInfo->Private.pConvertSymbInfo)) { const ET9BOOL bExactLast = (ET9EXACTLAST(pLingInfo->pLingCmnInfo) && !bExactLock) ? 1 : 0; WLOG2(fprintf(pLogFile2, "bExactLast %c, bExactLock %c\n", (bExactLast ? 'Y' : 'N'), (bExactLock ? 'Y' : 'N'));) _ET9SimpleWordToPrivWord(&sSimpleWord, &sExactWord); sExactWord.Body.bWordSrc = (bExactLast) ? ET9WORDSRC_EXACT_LAST : (ET9WORDSRC_NONE | EXACTOFFSET); sExactWord.Body.xTotFreq = 0; sExactWord.Body.xTapFreq = 1; sExactWord.Body.xWordFreq = 1; sExactWord.Body.wTWordFreq = 0; sExactWord.Body.wEWordFreq = 0; sExactWord.Base.bIsTerm = 1; sExactWord.Body.bIsRetain = 1; #if 0 { ET9UINT nIndex; for (nIndex = 0; nIndex < bNumSymbs && nIndex < ET9_SPC_ED_MAX_FREQ_LEN; ++nIndex) { sExactWord.Body.xTapFreq *= (ET9FREQPART)pWordSymbInfo->SymbsInfo[nIndex].DataPerBaseSym[0].bSymFreq; } #ifdef ET9_USE_FLOAT_FREQS sExactWord.Body.xTapFreq /= (ET9FREQPART)(_ET9pow_f(10, (ET9FLOAT)nIndex)); #else /* ET9_USE_FLOAT_FREQS */ if (bNumSymbs > 4) { sExactWord.Body.xTapFreq = (ET9FREQPART)(sExactWord.Body.xTapFreq >> (4 * (bNumSymbs - 4))); } #endif /* ET9_USE_FLOAT_FREQS */ } #endif sExactWord.Body.xScaledTapFreq = sExactWord.Body.xTapFreq; if (ET9AW_GetBilingualSupported(pLingInfo)) { sExactWord.Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; } else { sExactWord.Body.bLangIndexScoring = ET9AWFIRST_LANGUAGE; } WLOG2Word(pLogFile2, "ExactWord", &sExactWord); if (bExactInList) { if (!bExactLast) { pLingCmnInfo->Private.nExactIndex = 0; } pLingCmnInfo->Private.bStemsAllowed = 1; pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; __ET9AWSelLstInsert(pLingInfo, &sExactWord, FREQ_NORMAL, 0); } if (!bDefaultOkay) { /* this still happens... */ *pPrevDefaultWord = sExactWord; if (pPrevDefaultWord->Base.wWordLen) { --pPrevDefaultWord->Base.wWordLen; } WLOG2Word(pLogFile2, "PrevDefaultWord (fail-safe from exact)", pPrevDefaultWord); __CaptureDefault(pLingInfo, pPrevDefaultWord, bNumSymbs, bLastBuildLen); } } } /* store start value for tracking full word matches (after exact, before required) */ nFullWordMatches = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; /* add required word to the list, when applicable */ if (bNumSymbs && !wFlushPoint && pWordSymbInfo->Private.sRequiredWord.wLen == bNumSymbs) { ET9AWPrivWordInfo sLocalWord; _ET9SimpleWordToPrivWord(&pWordSymbInfo->Private.sRequiredWord, &sLocalWord); sLocalWord.Body.bWordSrc = ET9WORDSRC_REQUIRED; WLOG2Word(pLogFile2, "RequiredWord", &sLocalWord); pLingCmnInfo->Private.bStemsAllowed = __GetStemsAllowed(pLingCmnInfo, bNumSymbs); pLingCmnInfo->Private.wMaxWordLength = __GetMaxWordLength(pLingCmnInfo, bNumSymbs); __SelLstWordSearch(pLingInfo, &sLocalWord, wFlushPoint, bNumSymbs, FREQ_NORMAL, bSpcMode); } /* at this point the default word has a value if it ever gets one... */ __CaptureUpdateFlushInfo(pLingInfo, pPrevDefaultWord); /* setup left hand word */ if (wFlushPoint) { __CaptureGetFlush(pLingInfo, pLeftHandWord); __UpdateSpcInfo(pLingInfo, pLeftHandWord, 0, wFlushPoint, bSpcMode); } else if (bNumSymbs > 1) { pLeftHandWord->Body.bLangIndexScoring = pPrevDefaultWord->Body.bLangIndexScoring; } else if (ET9AW_GetBilingualSupported(pLingInfo)) { pLeftHandWord->Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; } else { pLeftHandWord->Body.bLangIndexScoring = ET9AWFIRST_LANGUAGE; } WLOG2Word(pLogFile2, "LeftHandWord", pLeftHandWord); ET9AssertLog(pLeftHandWord->Base.wWordCompLen <= pLeftHandWord->Base.wWordLen); /* do full builds */ { const ET9U16 wWordLen = (ET9U16)(bNumSymbs - wFlushPoint); ET9U8 bLdbEntries; ET9U8 bSuppEntries; ET9U8 bSuppASEntries; __ProfileStart; bLdbEntries = 0; bSuppEntries = 0; bSuppASEntries = 0; pLingCmnInfo->Private.bStemsAllowed = __GetStemsAllowed(pLingCmnInfo, wWordLen); pLingCmnInfo->Private.wMaxWordLength = __GetMaxWordLength(pLingCmnInfo, wWordLen); if (!bNumSymbs && ET9CONTEXTBASEDPREDICTION(pLingCmnInfo) && ET9NEXTWORDPREDICTION_MODE(pLingCmnInfo)) { /* NWP with NLM for active language */ WLOG2(fprintf(pLogFile2, "performing _ET9AWNLMWordsSearch\n");) _ET9AWLdbNwpWordsSearch(pLingInfo, pLingCmnInfo->Private.dwCurrActiveLanguage, 0); WLOG2(fprintf(pLogFile2, "performing _ET9AW_DLM_WordsSearch\n");) (void)_ET9AW_DLM_WordsSearch(pLingInfo, 0, 0, FREQ_NORMAL); } WLOG2(fprintf(pLogFile2, "performing _ET9AWSuppDBSelListBuild (non AS, after flush point), index = %d, length = %d\n", wFlushPoint, bNumSymbs - wFlushPoint);) _ET9AWSuppDBSelListBuild(pLingInfo, wFlushPoint, wWordLen, &bSuppEntries, FREQ_NORMAL, ET9SUPPDB_NONAS_SOURCES | ET9SUPPDB_GDB_SOURCES, bSpcMode); WLOG2(fprintf(pLogFile2, "performing _ET9AWLdbWordsSearch (after flush point), index = %d, length = %d\n", wFlushPoint, bNumSymbs - wFlushPoint);) WLOG2(fprintf(pLogFile2, "wFirstLdbNum %u, bLangIndex %u, bLastBuildShrinking %u\n", pLingCmnInfo->dwFirstLdbNum, pLeftHandWord->Body.bLangIndexScoring, pLingCmnInfo->Private.bLastBuildShrinking);) _ET9AWLdbWordsSearch(pLingInfo, (pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum), wFlushPoint, wWordLen, &bLdbEntries, FREQ_NORMAL, bSpcMode, 1); WLOG2(fprintf(pLogFile2, "_ET9AWLdbWordsSearch, tags, active %u, dwTotalCount %u, dwCollisionCount %u\n", pLingCmnInfo->Private.ALdb.tags.bActive, pLingCmnInfo->Private.ALdb.tags.dwTotalCount, pLingCmnInfo->Private.ALdb.tags.dwCollisionCount);) if (!(bNumSymbs == 1 && pLingInfo->pLingCmnInfo->Private.bStateNiceLatency)) { _ET9AWLdbWordsSearch(pLingInfo, (pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingCmnInfo->dwFirstLdbNum : pLingCmnInfo->dwSecondLdbNum), wFlushPoint, wWordLen, &bLdbEntries, FREQ_NORMAL, (bHasTraceInfo || (ET9INACTIVELANGSPELLCORRECTENABLED(pLingCmnInfo)) ? bSpcMode : ET9ASPCMODE_OFF), 1); } WLOG2(fprintf(pLogFile2, "_ET9AWLdbWordsSearch, tags, active %u, dwTotalCount %u, dwCollisionCount %u\n", pLingCmnInfo->Private.ALdb.tags.bActive, pLingCmnInfo->Private.ALdb.tags.dwTotalCount, pLingCmnInfo->Private.ALdb.tags.dwCollisionCount);) WLOG2(fprintf(pLogFile2, "performing _ET9AWSuppDBSelListBuild (AS, after flush point), index = %d, length = %d\n", wFlushPoint, bNumSymbs - wFlushPoint);) _ET9AWSuppDBSelListBuild(pLingInfo, wFlushPoint, wWordLen, &bSuppASEntries, FREQ_NORMAL, ET9SUPPDB_AS_SOURCES, bSpcMode); WLOG2(fprintf(pLogFile2, "full build results, bSuppEntries = %u, bSuppASEntries = %u, bLdbEntries = %u, bTotalWords = %u\n", (int)bSuppEntries, (int)bSuppASEntries, (int)bLdbEntries, (int)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords);) /* do post full build processing */ if (bSuppASEntries) { __DoBuildSubstitutionProcessing(pLingInfo); } if (!bNumSymbs && ET9CONTEXTBASEDPREDICTION(pLingCmnInfo) && ET9NEXTWORDPREDICTION_MODE(pLingCmnInfo) && pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords < pLingCmnInfo->Private.bTotalNWP) { /* NWP with NLM */ /* Backoff to unigram for active language */ if (bUsingALM) { WLOG2(fprintf(pLogFile2, "performing NLM unigram\n");) { ET9U8 bWordCount; ET9U32 pdwCandidateIndexList[ET9MAXCOLLECTSIZE]; for (bWordCount = 0; bWordCount < pLingCmnInfo->Private.bMaxUniNWP; ++bWordCount) { pdwCandidateIndexList[bWordCount] = bWordCount; } _ET9AWLdbWordsByIndex(pLingInfo, pLingCmnInfo->Private.dwCurrActiveLanguage, pdwCandidateIndexList, (ET9UINT)pLingCmnInfo->Private.bMaxUniNWP, 0, NULL); } } } __ProfileEnd(tAW_SelLst_BuildFullBuilds); } /* calculate full matches */ nFullWordMatches = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts - nFullWordMatches; ET9AssertLog(!bNumSymbs || !bExactInList || pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); /* find right most special (punct or explicit) char in word */ bHasTermPunct = 0; bHasTermSmartPunct = 0; wSpecialPosition = 0; eSpecialType = NOSPECIAL; if (!bNumSymbs) { bContinue = 0; } else { bContinue = 1; pSymbInfo = &pWordSymbInfo->SymbsInfo[bNumSymbs - 1]; __ET9GetSpecialCharInfo(pLingInfo, wFlushPoint, &wSpecialPosition, &eSpecialType); if (wSpecialPosition == bNumSymbs - wFlushPoint) { if (eSpecialType == SMARTPUNCT) { bHasTermSmartPunct = 1; } else if (eSpecialType == PUNCT) { bHasTermPunct = 1; } else if (__IsExactSymb(pSymbInfo) && _ET9_IsPunctChar(pSymbInfo->DataPerBaseSym[0].sChar[0])) { bHasTermPunct = 1; } } } WLOG2(fprintf(pLogFile2, "bHasTermPunct %u, bHasTermSmartPunct %u, wSpecialPosition %u\n", bHasTermPunct, bHasTermSmartPunct, wSpecialPosition);) /* check for buildarounds */ /* -------------------- Rightmost punct or explicit --------------------------*/ if ((eSpecialType != NOSPECIAL) && !bHasTermSmartPunct && !bHasTermPunct && !(bHasTraceInfo && pWordSymbInfo->Private.sRequiredWord.wLen && pWordSymbInfo->bNumSymbs)) { /* trailing special */ if ((wSpecialPosition > 1 && wSpecialPosition < bNumSymbs - wFlushPoint) || eSpecialType == LOCKPOINT) { WLOG2(fprintf(pLogFile2, "performing trailing BuildAroundSpecial\n");) if (__ET9BuildAroundSpecial(pLingInfo, wFlushPoint, wSpecialPosition, eSpecialType, pLeftHandWord, pPrevDefaultWord, (eSpecialType == LOCKPOINT ? BA_COMPOUND : BA_TRAILING), 0 /* embedded char*/, bSpcMode)) { WLOG2(fprintf(pLogFile2, "added trailing special\n");) bContinue = 0; /* update flush information */ if (!nFullWordMatches && pLeftHandWord->Base.wWordLen) { ET9U16 wFlushSymbolLen = wSpecialPosition + wFlushPoint; /* term punct completion problem... */ ET9AssertLog(pLeftHandWord->Base.wWordLen == wFlushSymbolLen || pLingCmnInfo->Private.bSpcDuringBuild || bHasTraceInfo); __CaptureFlushPoint(pLingInfo, wFlushSymbolLen, pLeftHandWord); WLOG2(fprintf(pLogFile2, "new flush point set @ %d (trailing special)\n", wFlushSymbolLen);) } nFullWordMatches = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; } } /* leading special */ if (bContinue && (eSpecialType != LOCKPOINT) && (wSpecialPosition > 1) && (wSpecialPosition < bNumSymbs - wFlushPoint)) { WLOG2(fprintf(pLogFile2, "performing leading BuildAroundSpecial\n");) if (__ET9BuildAroundSpecial(pLingInfo, wFlushPoint, wSpecialPosition, eSpecialType, pLeftHandWord, pPrevDefaultWord, BA_LEADING, 0 /* embedded char*/, bSpcMode)) { WLOG2(fprintf(pLogFile2, "added leading special\n");) /* update flush information */ /* Note there is something subtle going on here. If there was a successful buildaround with trailing punct, there is a flush because there were no full matches, we still want to do the embedded punct. This is because after the flush, a build should give embedded matches as well. Also, if we do not do this, a subsequent build with the SAME pWordSymbInfo would give a different list. */ if (!nFullWordMatches) { if (pLeftHandWord->Base.wWordLen) { ET9U16 wFlushSymbolLen = wSpecialPosition - 1 + wFlushPoint; /* term punct completion problem... */ ET9AssertLog(pLeftHandWord->Base.wWordLen == wFlushSymbolLen || pLingCmnInfo->Private.bSpcDuringBuild || bHasTraceInfo); __CaptureFlushPoint(pLingInfo, wFlushSymbolLen, pLeftHandWord); WLOG2(fprintf(pLogFile2, "new flush point set @ %d (leading special)\n", wFlushSymbolLen);) } } else { bContinue = 0; } nFullWordMatches = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; } } /* embedded special */ if (bContinue && eSpecialType != LOCKPOINT) { /* we only do embedded if this was an explicit, a single ET9KPUNCTUAITON, or an ET9KSMARTPUNCT. */ ET9SYMB sEmbeddedSymb; pSymbInfo = &pWordSymbInfo->SymbsInfo[wFlushPoint + wSpecialPosition - 1]; if (__IsExactSymb(pSymbInfo) || pSymbInfo->bSymbType == ET9KTSMARTPUNCT || (pSymbInfo->bSymbType == ET9KTPUNCTUATION && pSymbInfo->bNumBaseSyms == 1 && pSymbInfo->DataPerBaseSym[0].bNumSymsToMatch == 1)) { if (*pwLockPoint >= wFlushPoint + wSpecialPosition) { sEmbeddedSymb = pSymbInfo->sLockedSymb; WLOG2(fprintf(pLogFile2, "embedded symb %c (from lock)\n", (char)sEmbeddedSymb);) } else if (pSymbInfo->bSymbType == ET9KTSMARTPUNCT) { sEmbeddedSymb = _ET9_GetEmbPunctChar(pLingInfo, pLingCmnInfo->Private.dwCurrActiveLanguage); WLOG2(fprintf(pLogFile2, "embedded symb %c (from GetEmbPunctChar lang %08x) - RPE\n", (char)sEmbeddedSymb, pLingCmnInfo->Private.dwCurrActiveLanguage);) } else { if (pSymbInfo->eShiftState) { sEmbeddedSymb = pSymbInfo->DataPerBaseSym[0].sUpperCaseChar[0]; } else { sEmbeddedSymb = pSymbInfo->DataPerBaseSym[0].sChar[0]; } WLOG2(fprintf(pLogFile2, "embedded symb %c (from symbinfo)\n", (char)sEmbeddedSymb);) } ET9AssertLog(sEmbeddedSymb); WLOG2(fprintf(pLogFile2, "performing embedded BuildAroundSpecial\n");) if (__ET9BuildAroundSpecial(pLingInfo, wFlushPoint, wSpecialPosition, eSpecialType, pLeftHandWord, pPrevDefaultWord, BA_EMBEDDED, sEmbeddedSymb, bSpcMode)) { WLOG2(fprintf(pLogFile2, "added embedded special, nFullWordMatches = %d, left wordLen = %d\n", nFullWordMatches, pLeftHandWord->Base.wWordLen);) /* update flush information */ if (!nFullWordMatches && pLeftHandWord->Base.wWordLen) { ET9U16 wFlushSymbolLen = wSpecialPosition + wFlushPoint; /* term punct completion problem... */ ET9AssertLog(pLeftHandWord->Base.wWordLen == wFlushSymbolLen || pLingCmnInfo->Private.bSpcDuringBuild || bHasTraceInfo); __CaptureFlushPoint(pLingInfo, wFlushSymbolLen, pLeftHandWord); WLOG2(fprintf(pLogFile2, "new flush point set @ %d (embedded special)\n", wFlushSymbolLen);) } nFullWordMatches = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; } } } } /* -------------------- default WITHOUT completion + punct --------------------------*/ if ((bHasTermPunct || bHasTermSmartPunct) && bDefaultOkay) { if (*pwLockPoint == bNumSymbs) { WLOG2(fprintf(pLogFile2, "at lock point, won't perform a default (no completion) punct build\n");) } else { ET9FREQPART xCurrTermPuncFreq = (ET9FREQPART)ET9_DB_MAX_FREQ; WLOG2(fprintf(pLogFile2, "performing a default (no completion) punct build\n");) if (bNumSymbs - wFlushPoint - 1 > 0) { /* get the portion of the word before the punctuation */ ET9U16 wDefaultLen = __CaptureGetDefaultStringLength(pLingInfo, bNumSymbs); if (pPrevDefaultWord->Base.wWordLen < wDefaultLen) { WLOG2(fprintf(pLogFile2, "LeftHandWord, prev default shorter than expected (%d < %d)\n", pPrevDefaultWord->Base.wWordLen, wDefaultLen);) } *pLeftHandWord = *pPrevDefaultWord; if (pLeftHandWord->Base.wWordLen > wDefaultLen) { pLeftHandWord->Base.wWordLen = wDefaultLen; } pLeftHandWord->Base.wWordCompLen = 0; __UpdateSpcInfo(pLingInfo, pLeftHandWord, 0, (ET9U8)(bNumSymbs - 1), bSpcMode); WLOG2Word(pLogFile2, "LeftHandWord (from prev default)", pLeftHandWord); /* update flush information */ if (!nFullWordMatches) { if (pLeftHandWord->Base.wWordLen) { ET9U16 wFlushSymbolLen = bNumSymbs - 1; __CaptureFlushPoint(pLingInfo, wFlushSymbolLen, pLeftHandWord); WLOG2(fprintf(pLogFile2, "new flush point set @ %d (has non completion punct)\n", wFlushSymbolLen);) } } } /* now add the punct(s) to the list. */ ET9AssertLog(!(bHasTermPunct && bHasTermSmartPunct)); WLOG2(fprintf(pLogFile2, "adding punct candidates\n");) pLingCmnInfo->Private.bStemsAllowed = 1; pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; /* Get termpuncts for respective language in unilingual mode. Get termpuncts for both languages in bilingual mode (primary followed by secondary) with 2 exceptions: (i) If lefthand word is a term in 1st language only, termpuncts from 1st language alone are included. (ii) If lefthand word is a term in 2nd language only, termpuncts from 2nd language alone are included. */ if (pLeftHandWord->Base.bIsTerm && pLeftHandWord->Body.bLangIndexScoring == ET9AWFIRST_LANGUAGE) { const ET9U8 bPunctSeqSize = (ET9U8)(bHasTermSmartPunct ? _ET9_GetNumTermPunct(pLingInfo, pLingCmnInfo->dwFirstLdbNum) : 1); for (i = 0; i < bPunctSeqSize; ++i) { ET9AWPrivWordInfo sPunctWord; _InitPrivWordInfo(&sPunctWord); sPunctWord.Body.xTapFreq = 1; sPunctWord.Body.xWordFreq = (ET9FREQPART)(xCurrTermPuncFreq -= ET9_DB_UNQ_FREQ); sPunctWord.Base.wWordLen = 1; sPunctWord.Body.bWordSrc = ET9WORDSRC_TERMPUNCT; sPunctWord.Body.bLangIndexScoring = pLeftHandWord->Body.bLangIndexScoring; if (bHasTermSmartPunct) { sPunctWord.Base.sWord[0] = _ET9_GetTermPunctChar(pLingInfo, pLingCmnInfo->dwFirstLdbNum, i); } else { sPunctWord.Base.sWord[0] = pWordSymbInfo->SymbsInfo[bNumSymbs - 1].DataPerBaseSym[0].sChar[0]; } ET9AssertLog(sPunctWord.Base.sWord[0]); WLOG2Word(pLogFile2, "PunctWord", &sPunctWord); __SelLstWordSearch(pLingInfo, &sPunctWord, (ET9U16)(bNumSymbs - 1), 1, FREQ_TERM_PUNCT, bSpcMode); } } else if (pLeftHandWord->Base.bIsTerm && pLeftHandWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) { const ET9U8 bPunctSeqSize = (ET9U8)(bHasTermSmartPunct ? _ET9_GetNumTermPunct(pLingInfo, pLingCmnInfo->dwSecondLdbNum) : 1); for (i = 0; i < bPunctSeqSize; ++i) { ET9AWPrivWordInfo sPunctWord; _InitPrivWordInfo(&sPunctWord); sPunctWord.Body.xTapFreq = 1; sPunctWord.Body.xWordFreq = (ET9FREQPART)(xCurrTermPuncFreq -= ET9_DB_UNQ_FREQ); sPunctWord.Base.wWordLen = 1; sPunctWord.Body.bWordSrc = ET9WORDSRC_TERMPUNCT; sPunctWord.Body.bLangIndexScoring = !ET9AW_GetBilingualSupported(pLingInfo) ? ET9AWFIRST_LANGUAGE : pLeftHandWord->Body.bLangIndexScoring; if (bHasTermSmartPunct) { sPunctWord.Base.sWord[0] = _ET9_GetTermPunctChar(pLingInfo, pLingCmnInfo->dwSecondLdbNum, i); } else { sPunctWord.Base.sWord[0] = pWordSymbInfo->SymbsInfo[bNumSymbs - 1].DataPerBaseSym[0].sChar[0]; } ET9AssertLog(sPunctWord.Base.sWord[0]); WLOG2Word(pLogFile2, "PunctWord", &sPunctWord); __SelLstWordSearch(pLingInfo, &sPunctWord, (ET9U16)(bNumSymbs - 1), 1, FREQ_TERM_PUNCT, bSpcMode); } } else { ET9U8 bPunctSeqSize; if (ET9AW_GetBilingualSupported(pLingInfo) && (pLingCmnInfo->Private.dwCurrActiveLanguage == pLingInfo->pLingCmnInfo->dwSecondLdbNum)) { bSecondLangTermPunctFirst = 1; bPunctSeqSize = (ET9U8)(bHasTermSmartPunct ? _ET9_GetNumTermPunct(pLingInfo, pLingCmnInfo->dwSecondLdbNum) : 1); } else { bPunctSeqSize = (ET9U8)(bHasTermSmartPunct ? _ET9_GetNumTermPunct(pLingInfo, pLingCmnInfo->dwFirstLdbNum) : 1); } for (i = 0; i < bPunctSeqSize; ++i) { ET9AWPrivWordInfo sPunctWord; _InitPrivWordInfo(&sPunctWord); sPunctWord.Body.xTapFreq = 1; sPunctWord.Body.xWordFreq = (ET9FREQPART)(xCurrTermPuncFreq -= ET9_DB_UNQ_FREQ); sPunctWord.Base.wWordLen = 1; sPunctWord.Body.bWordSrc = ET9WORDSRC_TERMPUNCT; sPunctWord.Body.bLangIndexScoring = !ET9AW_GetBilingualSupported(pLingInfo) ? ET9AWFIRST_LANGUAGE : pLeftHandWord->Body.bLangIndexScoring; if (bHasTermSmartPunct) { if (bSecondLangTermPunctFirst) { sPunctWord.Base.sWord[0] = _ET9_GetTermPunctChar(pLingInfo, pLingCmnInfo->dwSecondLdbNum, i); } else { sPunctWord.Base.sWord[0] = _ET9_GetTermPunctChar(pLingInfo, pLingCmnInfo->dwFirstLdbNum, i); } } else { sPunctWord.Base.sWord[0] = pWordSymbInfo->SymbsInfo[bNumSymbs - 1].DataPerBaseSym[0].sChar[0]; } ET9AssertLog(sPunctWord.Base.sWord[0]); WLOG2Word(pLogFile2, "PunctWord", &sPunctWord); __SelLstWordSearch(pLingInfo, &sPunctWord, (ET9U16)(bNumSymbs - 1), 1, FREQ_TERM_PUNCT, bSpcMode); } if (ET9AW_GetBilingualSupported(pLingInfo)) { if (bSecondLangTermPunctFirst) { bPunctSeqSize = (ET9U8)(bHasTermSmartPunct ? _ET9_GetNumTermPunct(pLingInfo, pLingCmnInfo->dwFirstLdbNum) : 1); } else { bPunctSeqSize = (ET9U8)(bHasTermSmartPunct ? _ET9_GetNumTermPunct(pLingInfo, pLingCmnInfo->dwSecondLdbNum) : 1); } for (i = 0; i < bPunctSeqSize; ++i) { ET9AWPrivWordInfo sPunctWord; _InitPrivWordInfo(&sPunctWord); sPunctWord.Body.xTapFreq = 1; sPunctWord.Body.xWordFreq = (ET9FREQPART)(xCurrTermPuncFreq -= ET9_DB_UNQ_FREQ); sPunctWord.Base.wWordLen = 1; sPunctWord.Body.bWordSrc = ET9WORDSRC_TERMPUNCT; sPunctWord.Body.bLangIndexScoring = !ET9AW_GetBilingualSupported(pLingInfo) ? ET9AWFIRST_LANGUAGE : pLeftHandWord->Body.bLangIndexScoring; if (bHasTermSmartPunct) { if (bSecondLangTermPunctFirst) { sPunctWord.Base.sWord[0] = _ET9_GetTermPunctChar(pLingInfo, pLingCmnInfo->dwFirstLdbNum, i); } else { sPunctWord.Base.sWord[0] = _ET9_GetTermPunctChar(pLingInfo, pLingCmnInfo->dwSecondLdbNum, i); } } else { sPunctWord.Base.sWord[0] = pWordSymbInfo->SymbsInfo[bNumSymbs - 1].DataPerBaseSym[0].sChar[0]; } ET9AssertLog(sPunctWord.Base.sWord[0]); WLOG2Word(pLogFile2, "PunctWord", &sPunctWord); __SelLstWordSearch(pLingInfo, &sPunctWord, (ET9U16)(bNumSymbs - 1), 1, FREQ_TERM_PUNCT, bSpcMode); } } } } } /* -------------------- build append using explicit (single) --------------------------*/ if (bNumSymbs > 1 && __IsExactSymb(&pWordSymbInfo->SymbsInfo[bNumSymbs - 1]) && bHasDiscreteOnlyInfo) { if (*pwLockPoint == bNumSymbs) { WLOG2(fprintf(pLogFile2, "at lock point, won't add build append candidate (single)\n");) bFound = 0; } else { bFound = __LeftHandWordFromPrevDefaultWord(pLingInfo, bNumSymbs, pLeftHandWord, pPrevDefaultWord); } if (bFound) { ET9SYMB sSymb; ET9AWPrivWordInfo sLocalWord; WLOG2(fprintf(pLogFile2, "adding build append single explicit candidate\n");) WLOG2Word(pLogFile2, "build append, LeftHandWord", pLeftHandWord); __VerifyLockedSymbs(pLingInfo, pLeftHandWord, 0, pLeftHandWord->Base.wWordLen); pLingCmnInfo->Private.bStemsAllowed = 1; pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; pSymbInfo = &pWordSymbInfo->SymbsInfo[bNumSymbs - 1]; sSymb = pSymbInfo->eShiftState ? *pSymbInfo->DataPerBaseSym->sUpperCaseChar : *pSymbInfo->DataPerBaseSym->sChar; _InitPrivWordInfo(&sLocalWord); sLocalWord.Base.wWordLen = 1; sLocalWord.Base.sWord[0] = sSymb; sLocalWord.Body.wEWordFreq = 0; sLocalWord.Body.wTWordFreq = 0; sLocalWord.Body.bWordSrc = _ET9_IsPunctChar(sSymb) ? ET9WORDSRC_BUILDAPPENDPUNCT : ET9WORDSRC_BUILDAPPEND; WLOG2Word(pLogFile2, "build append, BuildAppendWord (single)", &sLocalWord); __SelLstWordSearch(pLingInfo, &sLocalWord, (ET9U16)(bNumSymbs - 1), 1, FREQ_NORMAL, bSpcMode); } } /* -------------------- build append using explicit (sequence) --------------------------*/ if (bNumSymbs > 1 && __IsExactSymb(&pWordSymbInfo->SymbsInfo[bNumSymbs - 1])) { if (*pwLockPoint == bNumSymbs) { WLOG2(fprintf(pLogFile2, "at lock point, won't add build append candidate (sequence)\n");) bFound = 0; } else { ET9U16 wExplicitPoint; /* find where the explicits started */ wExplicitPoint = bNumSymbs; if (bNumSymbs > 1) { pSymbInfo = &pWordSymbInfo->SymbsInfo[bNumSymbs - 2]; for (; wExplicitPoint > 1 && wExplicitPoint > *pwLockPoint; --wExplicitPoint, --pSymbInfo) { /* ignore invalid Insure++ warnings */ if (!__IsExactSymb(pSymbInfo)) { break; } } } WLOG2(fprintf(pLogFile2, "build append, wExplicitPoint = %d, wLockPoint = %d\n", wExplicitPoint, *pwLockPoint);) /* get the earliest prev default or explicit if there was a lock */ if (wExplicitPoint == 1) { _InitPrivWordInfo(pLeftHandWord); bFound = 1; WLOG2Word(pLogFile2, "build append, from beginning", pLeftHandWord); } else if (wExplicitPoint == *pwLockPoint) { ET9STATUS wStatus; wStatus = ET9GetExactWord(pWordSymbInfo, &sSimpleWord, pLingInfo->Private.pConvertSymb, pLingInfo->Private.pConvertSymbInfo); if (wStatus) { bFound = 0; } else { _ET9SimpleWordToPrivWord(&sSimpleWord, pLeftHandWord); if (pLeftHandWord->Base.wWordLen < *pwLockPoint) { bFound = 0; } else { pLeftHandWord->Base.wWordLen = *pwLockPoint; ++wExplicitPoint; bFound = 1; WLOG2Word(pLogFile2, "build append, from exact", pLeftHandWord); } } } else { if (__CaptureGetDefault(pLingInfo, pLeftHandWord, wExplicitPoint)) { if (pLeftHandWord->Base.wWordCompLen) { WLOG2Word(pLogFile2, "build append, prev default with compl len", pLeftHandWord); pLeftHandWord->Base.wWordLen = (ET9U16)(pLeftHandWord->Base.wWordLen - pLeftHandWord->Base.wWordCompLen); pLeftHandWord->Base.wWordCompLen = 0; } bFound = 1; WLOG2Word(pLogFile2, "build append, from prev default", pLeftHandWord); } else { bFound = 0; } } if (!bFound) { /* no use trying just the prev default here, handle in the "single" build above */ WLOG2(fprintf(pLogFile2, "build append, failed to build a good left hand side\n");) } else { /* add explicits that exist between the default and the current explicit */ ET9SYMB *psSymb; pSymbInfo = &pWordSymbInfo->SymbsInfo[wExplicitPoint - 1]; psSymb = &pLeftHandWord->Base.sWord[pLeftHandWord->Base.wWordLen]; for (; wExplicitPoint < bNumSymbs; ++wExplicitPoint, ++pSymbInfo, ++psSymb) { if (pLeftHandWord->Base.wWordLen >= ET9MAXWORDSIZE) { WLOG2(fprintf(pLogFile2, "build append, too long left hand side...\n");) break; } if (wExplicitPoint <= *pwLockPoint) { *psSymb = pSymbInfo->sLockedSymb; } else { *psSymb = pSymbInfo->eShiftState ? *pSymbInfo->DataPerBaseSym->sUpperCaseChar : *pSymbInfo->DataPerBaseSym->sChar; } ++pLeftHandWord->Base.wWordLen; WLOG2Word(pLogFile2, "build append, LeftHandWord got an explicit", pLeftHandWord); } } } if (bFound) { ET9SYMB sSymb; ET9AWPrivWordInfo sLocalWord; WLOG2(fprintf(pLogFile2, "build append, adding trailing explicit candidate\n");) WLOG2Word(pLogFile2, "build append, LeftHandWord", pLeftHandWord); __VerifyLockedSymbs(pLingInfo, pLeftHandWord, 0, pLeftHandWord->Base.wWordLen); pLingCmnInfo->Private.bStemsAllowed = 1; pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; pSymbInfo = &pWordSymbInfo->SymbsInfo[bNumSymbs - 1]; sSymb = pSymbInfo->eShiftState ? *pSymbInfo->DataPerBaseSym->sUpperCaseChar : *pSymbInfo->DataPerBaseSym->sChar; _InitPrivWordInfo(&sLocalWord); sLocalWord.Base.wWordLen = 1; sLocalWord.Base.sWord[0] = sSymb; sLocalWord.Body.wEWordFreq = 0; sLocalWord.Body.wTWordFreq = 0; sLocalWord.Body.bWordSrc = _ET9_IsPunctChar(sSymb) ? ET9WORDSRC_BUILDAPPENDPUNCT : ET9WORDSRC_BUILDAPPEND; WLOG2Word(pLogFile2, "build append, BuildAppendWord (sequence)", &sLocalWord); __SelLstWordSearch(pLingInfo, &sLocalWord, (ET9U16)(bNumSymbs - 1), 1, FREQ_NORMAL, bSpcMode); } } /* -------------------- auto append --------------------------*/ if (bNumSymbs > 1 && ET9AUTOAPPENDINLIST(pLingCmnInfo) && bHasDiscreteOnlyInfo) { /* after a lock the default will have the correct value for auto append (no need for a special case) */ if (*pwLockPoint == bNumSymbs) { WLOG2(fprintf(pLogFile2, "at lock point, won't add auto append candidates\n");) bFound = 0; } else if (pWordSymbInfo->SymbsInfo[bNumSymbs - 1].eInputType == ET9MULTITAPKEY) { WLOG2(fprintf(pLogFile2, "at multitap symb, won't add auto append candidates\n");) bFound = 0; } else { bFound = __LeftHandWordFromPrevDefaultWord(pLingInfo, bNumSymbs, pLeftHandWord, pPrevDefaultWord); WLOG2(if (!bFound) { fprintf(pLogFile2, "prev default too short for auto append (skipping)\n"); }) } if (bLastInputIsTrace) { bFound = 0; } if (bFound) { ET9U8 bSyms; ET9U8 bKeyIndex; ET9U8 bFirstLevelOnly; ET9U8 bAutoAppendSymbCount; ET9SYMB *psChar; ET9DataPerBaseSym *pSymbData; ET9AWPrivWordInfo sAutoAppendWord; ET9U32 dwLdbNum; WLOG2(fprintf(pLogFile2, "adding auto append candidates\n");) WLOG2Word(pLogFile2, "LeftHandWord", pLeftHandWord); /* check to see if word being autoappended is uppercase... if so, */ /* downshift it for autoappended entries */ pSymbInfo = pWordSymbInfo->SymbsInfo; if ((*pwLockPoint == 0) && pSymbInfo->bSymbType == ET9KTLETTER && !pSymbInfo->eShiftState) { dwLdbNum = (pLeftHandWord->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; if (_ET9SymIsUpper(pLeftHandWord->Base.sWord[0], pWordSymbInfo->Private.dwLocale)) { pLeftHandWord->Base.sWord[0] = _ET9SymToLower(pLeftHandWord->Base.sWord[0], dwLdbNum); WLOG2Word(pLogFile2, "Downshifted LeftHandWord", pLeftHandWord); } } pSymbInfo = &pWordSymbInfo->SymbsInfo[bNumSymbs - 1]; switch (pSymbInfo->eInputType) { case ET9DISCRETEKEY: case ET9CUSTOMSET: case ET9MULTITAPKEY: bFirstLevelOnly = 0; break; default: bFirstLevelOnly = 1; break; } __VerifyLockedSymbs(pLingInfo, pLeftHandWord, 0, pLeftHandWord->Base.wWordLen); pLingCmnInfo->Private.bStemsAllowed = 1; pLingCmnInfo->Private.wMaxWordLength = ET9MAXWORDSIZE; bAutoAppendSymbCount = 0; bKeyIndex = bFirstLevelOnly ? 1 : pSymbInfo->bNumBaseSyms; pSymbData = pSymbInfo->DataPerBaseSym; while (bKeyIndex--) { psChar = pSymbData->sChar; bSyms = pSymbData->bNumSymsToMatch; while (bSyms--) { if (_ET9AWLdbIsSymbolUsed(pLingInfo, pLingCmnInfo->dwFirstLdbNum, *psChar) || (ET9AW_GetBilingualSupported(pLingInfo) && _ET9AWLdbIsSymbolUsed(pLingInfo, pLingCmnInfo->dwSecondLdbNum, *psChar))) { _InitPrivWordInfo(&sAutoAppendWord); sAutoAppendWord.Body.xTapFreq = 1; sAutoAppendWord.Body.xWordFreq = (ET9FREQPART)(_ET9_IsNumeric(*psChar) ? *psChar : (ET9_DB_MAX_FREQ - (*psChar * ET9_DB_UNQ_FREQ))); /* unicode order */ sAutoAppendWord.Body.bWordSrc = ET9WORDSRC_AUTOAPPEND; sAutoAppendWord.Base.wWordLen = 1; sAutoAppendWord.Base.sWord[0] = pSymbInfo->eShiftState ? _ET9SymToUpper(*psChar, pLingCmnInfo->dwLdbNum) : *psChar; /* this is the correct LDB to use */ if (ET9AW_GetBilingualSupported(pLingInfo)) { sAutoAppendWord.Body.bLangIndexScoring = ET9AWBOTH_LANGUAGES; } else { sAutoAppendWord.Body.bLangIndexScoring = ET9AWFIRST_LANGUAGE; } if (bAutoAppendSymbCount < MAX_AUTO_APPEND_SYMBS && (ET9UINT)(bAutoAppendSymbCount + 3) < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) { WLOG2Word(pLogFile2, "AutoAppendWord", &sAutoAppendWord); __SelLstWordSearch(pLingInfo, &sAutoAppendWord, (ET9U16)(bNumSymbs - 1), 1, FREQ_TERM_PUNCT, bSpcMode); } else { WLOG2Word(pLogFile2, "too many auto append words, discarding", &sAutoAppendWord); } ++bAutoAppendSymbCount; } ++psChar; } ++pSymbData; } } } /* -------------------- default space candidate --------------------------*/ /* using the default and a "single" key press to generate a delayed "auto space" */ if (bNumSymbs > 1 && ET9AUTOSPACE(pLingCmnInfo) && pPrevDefaultWord->Base.wWordLen && !pLingCmnInfo->Private.bLastBuildShrinking && !pWordSymbInfo->Private.sRequiredWord.wLen) { /* look for start of tail */ ET9UINT nTailIndex; for (nTailIndex = bNumSymbs; nTailIndex > 0; --nTailIndex) { ET9SymbInfo const * const pSymb = &pWordSymbInfo->SymbsInfo[nTailIndex - 1]; if (pSymb->bTraceIndex || pSymb->bTraceLockPoint) { break; } } /* verify tail content */ if (nTailIndex && nTailIndex < bNumSymbs) { const ET9UINT nTailLen = bNumSymbs - nTailIndex; ET9UINT nIndex; for (nIndex = 0; nIndex < nTailLen; ++nIndex) { ET9SymbInfo const * const pSymb = &pWordSymbInfo->SymbsInfo[nIndex + nTailIndex]; if (!nIndex && (pSymb->bSymbType == ET9KTLETTER || pSymb->bSymbType == ET9KTNUMBER || pSymb->bSymbType == ET9KTUNKNOWN)) { continue; } if (nIndex && (pSymb->bSymbType == ET9KTPUNCTUATION || pSymb->bSymbType == ET9KTNUMBER)) { continue; } break; } /* create candidate if ok and fits */ if (nIndex == nTailLen && __CaptureGetDefault(pLingInfo, pLeftHandWord, (ET9U16)(bNumSymbs - nTailLen + 1)) && (pLeftHandWord->Base.wWordLen + 1 + nTailLen) <= ET9MAXWORDSIZE) { pLeftHandWord->Base.sWord[pLeftHandWord->Base.wWordLen++] = ' '; WLOG2(fprintf(pLogFile2, "adding auto space candidate\n");) WLOG2Word(pLogFile2, "LeftHandWord", pLeftHandWord); { ET9AWPrivWordInfo sLocalWord; _InitPrivWordInfo(&sLocalWord); sLocalWord.Body.xWordFreq = (ET9FREQPART)ET9_DB_MAX_FREQ; sLocalWord.Body.bWordSrc = ET9WORDSRC_AUTOSPACE; sLocalWord.Base.wWordLen = (ET9U16)nTailLen; for (nIndex = 0; nIndex < nTailLen; ++nIndex) { ET9SymbInfo const * const pSymb = &pWordSymbInfo->SymbsInfo[nIndex + nTailIndex]; sLocalWord.Base.sWord[nIndex] = pSymb->eShiftState ? pSymb->DataPerBaseSym->sUpperCaseChar[0] : pSymb->DataPerBaseSym->sChar[0]; if (!nIndex) { ET9U16 wRecordNum; if (!_ET9AWLdbASFindEntry(pLingInfo, pLingCmnInfo->Private.dwCurrActiveLanguage, sLocalWord.Base.sWord, 1, sLocalWord.Base.sSubstitution, &sLocalWord.Base.wSubstitutionLen, &wRecordNum) && sLocalWord.Base.wSubstitutionLen == 1) { sLocalWord.Base.sWord[0] = sLocalWord.Base.sSubstitution[0]; } sLocalWord.Base.wSubstitutionLen = 0; } } sLocalWord.Body.bLangIndexScoring = (pLingCmnInfo->Private.dwCurrActiveLanguage == pLingCmnInfo->dwSecondLdbNum) ? ET9AWSECOND_LANGUAGE : ET9AWFIRST_LANGUAGE; WLOG2Word(pLogFile2, "AutoSpaceWord", &sLocalWord); __SelLstWordSearch(pLingInfo, &sLocalWord, (ET9U16)nTailIndex, sLocalWord.Base.wWordLen, FREQ_NORMAL, ET9ASPCMODE_OFF); } } } } /* -------------------- runtime segmentation candidate --------------------------*/ __AddSegmentedCandidate(pLingInfo, bSpcMode, wFlushPoint); /* -------------------- done building --------------------------*/ pLingCmnInfo->Private.bDoneBuilding = 1; __ProfileEnd(tAW_SelLst_BuildSearch); /* export ordering info to list */ __WordOrder_ExportPriorityAccessList(pLingCmnInfo); /* if no words at this point we are done... */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords && !bHasTermSmartPunct && !bHasTermPunct) { WLOG2(fprintf(pLogFile2, "__ET9AWDoSelLstBuild, (no words) END *****\n");) pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords = 0; __RestoreSelectionListOverrideValues(pLingCmnInfo); return ET9STATUS_NONE; } __LogPartialSelList(pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "done building", pLogFile2); #ifdef ET9_DEBUG { ET9INT snCount; pAmbigWord = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; for (snCount = pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize; snCount; --snCount, ++pAmbigWord) { ET9AssertLog(pAmbigWord->Base.wWordLen <= ET9MAXWORDSIZE); } } #endif /* final call */ _ET9_PrecalculateSettings(pLingInfo, bNumSymbs - 1); /* Assign LM etc freqs here */ __CalculateInputScores(pLingInfo); __CalculateInputScoresSpm(pLingInfo); /* Process for gestures */ if (__DoGestureProcessing(pLingInfo, pwGestureValue)) { WLOG2(fprintf(pLogFile2, "__ET9AWDoSelLstBuild, (gesture won) END *****\n");) *pbTotalWords = (ET9U8)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; __RestoreSelectionListOverrideValues(pLingCmnInfo); return ET9STATUS_NONE; } __UpdateBuiltInWordFreqs(pLingInfo); __UpdateCustomWordFreqs(pLingInfo); __CalculateFinalTotWordFreqs(pLingInfo); /* look for exact and exactish and see if exact won */ WLOG2B(fprintf(pLogFile2, "Searching for exact and exactish\n");) pAmbigWord = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; bFound = 0; pWord = NULL; pExactWord = NULL; if (!ET9EXACTLAST(pLingInfo->pLingCmnInfo)) { for (i = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; bExactInList && i; --i, ++pAmbigWord) { if (!ISEXACTSRC(pAmbigWord->Body.bWordSrc)) { if (__ET9AWIsLikeExactMatch(pLingInfo, pAmbigWord, 0)) { WLOG2BWord(pLogFile2, "..isLikeExact", pAmbigWord); if (!pWord || __PriorityCompare(pLingCmnInfo, pAmbigWord, pWord, SS_Final) > 0) { WLOG2BWord(pLogFile2, "....new top", pAmbigWord); pWord = pAmbigWord; } } else { bFound = 1; } } else { WLOG2BWord(pLogFile2, "..found exact word", pAmbigWord); pExactWord = pAmbigWord; } } } bExactWon = 0; if (pExactWord && ISGENUINESRC(pExactWord->Body.bWordSrc)) { pExactWord->Body.bWordSrc = GETBASESRC(pExactWord->Body.bWordSrc); if (pWord && __PriorityCompare(pLingCmnInfo, pExactWord, pWord, SS_Final) >= 0) { WLOG2B(fprintf(pLogFile2, "..exact won!\n");) bExactWon = 1; } WLOG2BWord(pLogFile2, "..exact tag", pExactWord); pExactWord->Body.bWordSrc |= EXACTOFFSET; } if (pWord && bFound) { WLOG2BWord(pLogFile2, "..exactish tag", pWord); pWord->Body.bWordSrc |= EXACTISHOFFSET; } /* before sorting: disposable quality words goes into the stem pool (e.g. RDB words, not UDB words) stem quality words goes (back) into the stem pool (can have changed source during build) */ pAmbigWord = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; for (i = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; i; --i, ++pAmbigWord) { if (pAmbigWord->Body.bWordSrc == ET9WORDSRC_STEMPOOL) { continue; } if (pAmbigWord->Body.bWordQuality <= DISPOSABLE_QUALITY) { WLOG2Word(pLogFile2, "disposable quality word will be put into the stem pool", pAmbigWord); pAmbigWord->Body.bWordSrc = ET9WORDSRC_STEMPOOL; } else if (pAmbigWord->Body.bWordQuality <= STEM_QUALITY) { WLOG2Word(pLogFile2, "stem quality word will be put into the stem pool", pAmbigWord); pAmbigWord->Body.bWordSrc = ET9WORDSRC_STEMPOOL; } } /* sort & final */ __ET9AWSortPriorityList(pLingCmnInfo); __ET9AWPriorityListFinalOrder(pLingCmnInfo); __TagExactLastAsRealExact(pLingCmnInfo); __DedupeAfterSpmShifting(pLingInfo); /* pre-default post process the list (must be before setting default index) */ __DoBuildPreDefaultPostProcessing(pLingInfo); /* setting default index - keep rules simple! (not a speed factor) */ { ET9AWPrivWordInfo const * const pWord0 = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords > 0 ? &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[0]] : NULL; ET9AWPrivWordInfo const * const pWord1 = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords > 1 ? &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[1]] : NULL; ET9BOOL bExactish1 = (pWord1 && ISEXACTISHSRC(pWord1->Body.bWordSrc)) ? 1 : 0; if (!bNumSymbs) { /* NWP */ WLOG2(fprintf(pLogFile2, "[1] DEFAULT: 0 - NWP\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= 1) { WLOG2(fprintf(pLogFile2, "[2] DEFAULT: 0 - nTotalWords <= 1\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (pWord0->Body.bIsBlack && !pWord1->Body.bIsBlack) { WLOG2(fprintf(pLogFile2, "[3] DEFAULT: 1 - first is black\n");) pLingCmnInfo->Private.nDefaultIndex = 1; } else if (pWord1->Body.bIsSegment && pWord1->Body.bNLMOrder > 0 && !pWord0->Body.bIsSegment) { WLOG2(fprintf(pLogFile2, "[4] DEFAULT: 1 - Segment\n");) pLingCmnInfo->Private.nDefaultIndex = 1; } else if (bNumSymbs == 1 && __HasInitialNonLetter(pWord1->Base.sWord, pWord1->Base.wWordLen)) { WLOG2(fprintf(pLogFile2, "[5] DEFAULT: 0 - bNumSymbs == 1 && __HasInitialNonLetter\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (_ET9HasExplicitNumber(pWordSymbInfo)) { WLOG2(fprintf(pLogFile2, "[6] DEFAULT: 0 - HasExplicitNumber\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactLock && !ET9DEVSTATEINHBEXACTLOCK_MODE(pLingCmnInfo->Private.dwDevStateBits)) { WLOG2(fprintf(pLogFile2, "[7] DEFAULT: 0 - bExactLock && !ET9DEVSTATEINHBEXACTLOCK_MODE\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bLastInputIsMultitap) { WLOG2(fprintf(pLogFile2, "[8] DEFAULT: 0 - bLastInputIsMultitap\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (!ISREALSRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[1]].Body.bWordSrc) && pExactWord && !__HasNumericStem(pLingCmnInfo, pExactWord->Base.sWord, pExactWord->Base.wWordLen)) { WLOG2(fprintf(pLogFile2, "[9] DEFAULT: 0 - !ISREALSRC && !HasNumericStem\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && ET9EXACTISDEFAULT(pLingInfo->pLingCmnInfo)) { WLOG2(fprintf(pLogFile2, "[10] DEFAULT: 0 - ET9EXACTISDEFAULT\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (!bExactInList || ET9EXACTLAST(pLingInfo->pLingCmnInfo) || (!pWord && pExactWord && ISGENUINESRC(pExactWord->Body.bWordSrc) && !ISNONOVERRIDEWRD(pExactWord))) { WLOG2(fprintf(pLogFile2, "[11] DEFAULT: 0 - !ExactInList || !Exactish && ExactWord && ISGENUINESRC && !ISNONOVERRIDEWRD\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (pExactWord && (GETBASESRC(pExactWord->Body.bWordSrc) >= ET9WORDSRC_EXACT) && !ISGENUINESRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[1]].Body.bWordSrc)) { WLOG2(fprintf(pLogFile2, "[12] DEFAULT: 0 - SRC >= EXACT && !ISGENUINESRC\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactWon && pExactWord && (bNumSymbs != 1 || !_ET9_IsNumericString(pExactWord->Base.sWord, 1))) { /* used to NOT allow any numeric strings to win here, now all allowed exception 1) single digits are disallowed */ WLOG2(fprintf(pLogFile2, "[13] DEFAULT: 0 - bNumSymbs != 1 || !IsNumericString\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && (bHasRegionalInfo || pWordSymbInfo->Private.bRequiredHasRegionalInfo) && pWordSymbInfo->Private.bRequiredInhibitOverride) { WLOG2(fprintf(pLogFile2, "[14] DEFAULT: 0 - bRequiredHasRegionalInfo && bRequiredInhibitOverride\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && bHasRegionalInfo && __ExactHasOverrideInhibitor(pLingCmnInfo) && !pWord1->Body.bLimitedCount) { WLOG2(fprintf(pLogFile2, "[15] DEFAULT: 0 - ExactHasOverrideInhibitor\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && bHasRegionalInfo && __HasUserShiftOverrideInhibit(pLingCmnInfo)) { WLOG2(fprintf(pLogFile2, "[16] DEFAULT: 0 - HasUserShiftOverrideInhibit\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && bHasRegionalInfo && bNumSymbs > 1 && bHasAllShiftedInfo) { WLOG2(fprintf(pLogFile2, "[17] DEFAULT: 0 - bNumSymbs > 1 && bHasAllShiftedInfo\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && bHasRegionalInfo && pWordSymbInfo->Private.bClearSymbEpisodeCount >= 2) { WLOG2(fprintf(pLogFile2, "[18] DEFAULT: 0 - bClearSymbEpisodeCount >= 2\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && bHasRegionalInfo && !bExactish1 && _ET9IsInhibitTapOverrideAfterTrace(pWordSymbInfo)) { WLOG2(fprintf(pLogFile2, "[19] DEFAULT: 0 - IsInhibitTapOverrideAfterTrace\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (bExactInList && bHasRegionalInfo && __HasStemDistanceOverrideInhibit(pLingCmnInfo)) { WLOG2(fprintf(pLogFile2, "[20] DEFAULT: 0 - HasStemDistanceOverrideInhibit\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else if (pLingCmnInfo->Private.bDemoteLimitedRegionality && !pWord0->Body.bLimitedCount && pWord1->Body.bLimitedCount) { WLOG2(fprintf(pLogFile2, "[21] DEFAULT: 0 - Demote word with limited regionality\n");) pLingCmnInfo->Private.nDefaultIndex = 0; } else { WLOG2(fprintf(pLogFile2, "[99] DEFAULT: 1\n");) pLingCmnInfo->Private.nDefaultIndex = 1; } if (bExactInList && pExactWord && !pLingCmnInfo->Private.nDefaultIndex && pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords > 1) { if (ISGENUINESRC(pWord1->Body.bWordSrc) && pWord1->Body.bNLMOrder > 2 && !ISGENUINESRC(pWord0->Body.bWordSrc)) { if (pWord1->Body.bIsHidden || pWord1->Body.bIsQuarantine || pWord1->Body.bIsWeak || pWord1->Body.bIsGray) { WLOG2(fprintf(pLogFile2, "[LM-1] blocked - Quarantine/Weak/Gray word\n");) } else if (__HasUserShiftOverrideInhibit(pLingCmnInfo) && !_ET9SymIsUpper(pWord1->Body.sPureFirstChar, pWordSymbInfo->Private.dwLocale)) { WLOG2(fprintf(pLogFile2, "[LM-1] blocked - User shift without capitalized override\n");) } else { WLOG2(fprintf(pLogFile2, "[LM-1] DEFAULT: 1 - !ISGENUINESRC(exact) && bNLMOrder > 2\n");) pLingCmnInfo->Private.nDefaultIndex = 1; } } } } /* post process the list (must be after setting default index) */ __DoBuildPostProcessing(pLingInfo); __DoBilingualReorder(pLingInfo); /* possibly default a "DEFAULT-SPACE-CHAR" */ __DoDefaultSpaceCharProcessing(pLingInfo); /* final process the list (must be after all other list ordering) */ __DoBuildFinalProcessing(pLingInfo); /* special default index handling for the required word */ __DoRequiredWordProcessing(pLingInfo, wFlushPoint); /* shrink the collected size down to target size (OEM size) */ __ShrinkSelListToTargetSize(pLingInfo); /* assign public source (might not be consecutive in the array (holes)) */ for (i = 0; i < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++i) { pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[i]]; pWord->Base.bWordSource = ET9AWORDSOURCE_CONSTRUCTED; switch (GETBASESRC(pWord->Body.bWordSrc)) { case ET9WORDSRC_NONE: pWord->Base.bWordSource = ET9AWORDSOURCE_NEWWORD; break; case ET9WORDSRC_TERMPUNCT: pWord->Base.bWordSource = ET9AWORDSOURCE_TERMPUNCT; break; case ET9WORDSRC_AUTOAPPEND: pWord->Base.bWordSource = ET9AWORDSOURCE_AUTOAPPEND; break; case ET9WORDSRC_STEM: case ET9WORDSRC_STEMPOOL: pWord->Base.bWordSource = ET9AWORDSOURCE_STEM; break; case ET9WORDSRC_QUDB: if (!wFlushPoint) { pWord->Base.bWordSource = ET9AWORDSOURCE_CUSTOM; } break; case ET9WORDSRC_RUDB: if (!wFlushPoint) { if (pWord->Body.bIsUDBWord) { pWord->Base.bWordSource = ET9AWORDSOURCE_CUSTOM; } else { pWord->Base.bWordSource = ET9AWORDSOURCE_LDB; } } break; case ET9WORDSRC_LDB: if (!wFlushPoint) { pWord->Base.bWordSource = ET9AWORDSOURCE_LDB; } break; case ET9WORDSRC_DLM: if (!wFlushPoint) { pWord->Base.bWordSource = ET9AWORDSOURCE_CUSTOM; } break; case ET9WORDSRC_MDB: if (!wFlushPoint) { pWord->Base.bWordSource = ET9AWORDSOURCE_MDB; } break; case ET9WORDSRC_CDB: if (!wFlushPoint) { pWord->Base.bWordSource = ET9AWORDSOURCE_CDB; } break; case ET9WORDSRC_LAS_SHORTCUT: case ET9WORDSRC_ASDB_SHORTCUT: if (!wFlushPoint && pWord->Base.wSubstitutionLen) { pWord->Base.bWordSource = ET9AWORDSOURCE_ASDB; } break; case ET9WORDSRC_GDB: if (!wFlushPoint) { pWord->Base.bWordSource = ET9AWORDSOURCE_GDB; } break; default: break; } if (pWord->Body.dwWordIndex && pWord->Base.bWordSource != ET9AWORDSOURCE_GDB) { pWord->Base.bWordSource = ET9AWORDSOURCE_LDB; } /* Is word from a user dictionary (UDB/CDB or DLM)? */ { const ET9BOOL bDLMActive = (pLingCmnInfo->pDLMInfo && ET9DLMENABLED(pLingCmnInfo)) ? 1 : 0; pWord->Base.bIsDeletable = ((pWord->Body.bIsUDBWord || bDLMActive) && !_ET9FindSpacesAndUnknown(pWord->Base.sWord, pWord->Base.wWordLen)) ? 1 : 0; } } /* result info */ ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= 0xFF); *pbTotalWords = (ET9U8)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; if (!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs && *pbTotalWords > pLingCmnInfo->Private.bTotalNWP) { *pbTotalWords = pLingCmnInfo->Private.bTotalNWP; } WLOG2(fprintf(pLogFile2, "pbTotalWords = %d\n", (int)*pbTotalWords);) #ifdef ET9_ACTIVATE_SLST_STATS ++pLingCmnInfo->Private.sStats.dwBuildCount; if (pLingCmnInfo->Private.sStats.dwInsertCount > pLingCmnInfo->Private.sStats.dwMaxListInserts) { pLingCmnInfo->Private.sStats.dwMaxListInserts = pLingCmnInfo->Private.sStats.dwInsertCount; } pLingCmnInfo->Private.sStats.dwInsertCount = 0; #endif __LogFullSelList(pLingInfo, "ET9AWSelLstBuild - all done", NULL); /* done */ ET9AssertLog(pLingCmnInfo->Private.nExactIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords || ISEXACTSRC(pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[pLingCmnInfo->Private.nExactIndex]].Body.bWordSrc)); #ifdef ET9_DEBUG /* verify locked chars and dupe quality */ if (*pwLockPoint) { ET9INT snCount; ET9INT snIndex; ET9SYMB *psSymb; ET9AWPrivWordInfo *pWord1; ET9SymbInfo *pSymbInfo; for (snIndex = 0; snIndex < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++snIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[snIndex]]; ET9AssertLog(pWord1->Body.bWordQuality != DUPE_QUALITY); if (GETBASESRC(pWord1->Body.bWordSrc) == ET9WORDSRC_MAGICSTRING) { continue; } if (pWord1->Base.wWordLen < *pwLockPoint) { WLOG2Word(pLogFile2, "word verified", pWord1); WLOG2(fprintf(pLogFile2, "word shorter than lock, index = %d\n", snIndex);) } ET9AssertLog(pWord1->Base.wWordLen >= *pwLockPoint); psSymb = pWord1->Base.sWord; pSymbInfo = pWordSymbInfo->SymbsInfo; for (snCount = *pwLockPoint; snCount; --snCount, ++psSymb, ++pSymbInfo) { ET9SYMB sLockedSymb = pSymbInfo->sLockedSymb; if (pLingInfo->Private.pConvertSymb != NULL) { pLingInfo->Private.pConvertSymb(pLingInfo->Private.pConvertSymbInfo, &sLockedSymb); } /* this is the final lock verification, must be exact match to converted lock */ if (*psSymb != sLockedSymb && !pLingCmnInfo->Private.bExpandAsDuringBuild) { WLOG2Word(pLogFile2, "word verified", pWord1); WLOG2(fprintf(pLogFile2, "lock symb missmatch index = %d, pos = %d, locked symb = %x\n", snIndex, (psSymb - pWord1->Base.sWord), sLockedSymb);) } ET9AssertLog(*psSymb == sLockedSymb || pLingCmnInfo->Private.bExpandAsDuringBuild); } } } /* verify that the list has no duplicates */ { ET9INT snLook; ET9INT snIndex; ET9AWPrivWordInfo *pWord1; ET9AWPrivWordInfo *pWord2; for (snIndex = 0; snIndex + 1 < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++snIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[snIndex]]; for (snLook = snIndex + 1; snLook < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++snLook) { pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[snLook]]; if (pWord1->Base.wWordLen == pWord2->Base.wWordLen) { ET9AssertLog(_ET9symbncmp(pWord1->Base.sWord, pWord2->Base.sWord, pWord1->Base.wWordLen)); } if (pWord1->Base.wSubstitutionLen == pWord2->Base.wWordLen) { ET9AssertLog(_ET9symbncmp(pWord1->Base.sSubstitution, pWord2->Base.sWord, pWord1->Base.wSubstitutionLen)); } if (pWord1->Base.wWordLen == pWord2->Base.wSubstitutionLen) { ET9AssertLog(_ET9symbncmp(pWord1->Base.sWord, pWord2->Base.sSubstitution, pWord1->Base.wWordLen)); } if (pWord1->Base.wSubstitutionLen && pWord1->Base.wSubstitutionLen == pWord2->Base.wSubstitutionLen) { /* only make this check if not generating lowercase shortcut */ if (!ET9DOWNSHIFTDEFAULTENABLED(pLingCmnInfo) || snIndex != (ET9INT)pLingCmnInfo->Private.nDefaultIndex) { ET9AssertLog(_ET9symbncmp(pWord1->Base.sSubstitution, pWord2->Base.sSubstitution, pWord1->Base.wSubstitutionLen)); } } } } } /* verify completion point compliance and frequency compliance */ { ET9UINT nIndex; ET9BOOL bCompletionBeforeCompletionPoint; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9AssertLog(pWord->Base.wWordLen); ET9AssertLog(pWord->Body.xTapFreq >= 0); ET9AssertLog(pWord->Body.xTotFreq >= 0); ET9AssertLog(pWord->Body.xWordFreq >= 0); bCompletionBeforeCompletionPoint = (ET9U8)( pWord->Base.wWordCompLen && pWordSymbInfo->bNumSymbs && /* not prediction */ pWordSymbInfo->bNumSymbs < pLingCmnInfo->Private.wWordCompletionPoint && !pWord->Body.bEditDistSpc && !pWord->Body.bEditDistFree); ET9AssertLog(!bCompletionBeforeCompletionPoint); } } /* verify no function key symbol in list */ { ET9UINT nIndex; for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { ET9AWPrivWordInfo * const pWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; ET9U16 wSymbIndex; for (wSymbIndex = 0; wSymbIndex < pWord->Base.wWordLen; ++wSymbIndex) { ET9AssertLog(!ET9IsFunctionKeySymbol(pWord->Base.sWord[wSymbIndex]) || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_AUTOSPACE || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_ASDB_SHORTCUT || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_LAS_SHORTCUT || GETBASESRC(pWord->Body.bWordSrc) == ET9WORDSRC_MAGICSTRING || pWord->Body.bIsSegment || pLingCmnInfo->Private.bStateSpaceSegmentationEnabled || pLingInfo->Private.pConvertSymb != NULL); } } } #endif WLOG2(fprintf(pLogFile2, "__ET9AWDoSelLstBuild, (normal) END *****\n");) __ProfileEnd(tAW_SelLst_PostProcessing); __RestoreSelectionListOverrideValues(pLingCmnInfo); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWDoSCSelLstBuild(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9U8 * const pbTotalWords) { ET9UINT i; ET9U8 bDummy; ET9U8 bFound; ET9AWPrivWordInfo *pWord; ET9AWPrivWordInfo *pAmbigWord; ET9AWPrivWordInfo *pExactWord = NULL; ET9SimpleWord sSimpleWord; ET9AWPrivWordInfo sExactWord; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9U8 bNumSymbs = pWordSymbInfo->bNumSymbs; const ET9U16 wWordLen = bNumSymbs; const ET9ASLMODE eSelectionListMode = pLingCmnInfo->Private.eSelectionListMode; const ET9ASLCORRECTIONMODE eSelectionListCorrectionMode = pLingCmnInfo->Private.eSelectionListCorrectionMode; const ET9ASLMODE eCurrSelectionListMode = pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pbTotalWords != NULL); *pbTotalWords = 0; __PreCalculateWSI(pLingInfo); __WordOrder_Initialize(pLingCmnInfo); _ET9AW_InitWordCollections(pLingInfo, 0, 0); _ET9AW_ResetWordList(pLingInfo); pLingCmnInfo->Private.bDoneBuilding = 0; /* update settings temporarily */ pLingCmnInfo->Private.eSelectionListMode = ET9ASLMODE_COMPLETIONSPROMOTED; pLingCmnInfo->Private.eSelectionListCorrectionMode = ET9ASLCORRECTIONMODE_LOW; pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = ET9ASLMODE_COMPLETIONSPROMOTED; /* handle spc tracking */ pLingCmnInfo->Private.bSpcDuringBuild = (ET9BOOL)(ET9_SPC_IS_ACTIVE(pLingCmnInfo->Private.ASpc.eMode) != 0); pLingCmnInfo->Private.bExpandAsDuringBuild = (ET9BOOL)(ET9EXPANDAUTOSUB(pLingCmnInfo) != 0); /* add exact match to list */ if (!ET9GetExactWord(pWordSymbInfo, &sSimpleWord, pLingInfo->Private.pConvertSymb, pLingInfo->Private.pConvertSymbInfo)) { _ET9SimpleWordToPrivWord(&sSimpleWord, &sExactWord); sExactWord.Body.bWordSrc = ET9WORDSRC_NONE | EXACTOFFSET; sExactWord.Body.xTotFreq = 0xFFFF; sExactWord.Body.xTapFreq = 0xFFFF; sExactWord.Body.xWordFreq = 1; sExactWord.Base.bIsTerm = 1; pLingCmnInfo->Private.bStemsAllowed = 0; pLingCmnInfo->Private.wMaxWordLength = bNumSymbs; __ET9AWSelLstInsert(pLingInfo, &sExactWord, FREQ_NORMAL, 0); } _ET9AWSuppDBSelListBuild(pLingInfo, 0, wWordLen, &bDummy, FREQ_NORMAL, ET9SUPPDB_NONAS_SOURCES, ET9ASPCMODE_EXACT); _ET9AWLdbWordsSearch(pLingInfo, dwLdbNum, 0, wWordLen, &bDummy, FREQ_NORMAL, ET9ASPCMODE_EXACT, 0); if (ET9AW_GetBilingualSupported(pLingInfo)) { ET9U32 dwAltLdbNum; if (dwLdbNum == pLingCmnInfo->dwFirstLdbNum) { dwAltLdbNum = pLingCmnInfo->dwSecondLdbNum; } else { dwAltLdbNum = pLingCmnInfo->dwFirstLdbNum; } _ET9AWLdbWordsSearch(pLingInfo, dwAltLdbNum, 0, wWordLen, &bDummy, FREQ_NORMAL, ET9ASPCMODE_EXACT, 0); } _ET9AWSuppDBSelListBuild(pLingInfo, 0, wWordLen, &bDummy, FREQ_NORMAL, ET9SUPPDB_AS_SOURCES, ET9ASPCMODE_EXACT); /* if no words at this point we are done... */ if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { pLingCmnInfo->Private.eSelectionListMode = eSelectionListMode; pLingCmnInfo->Private.eSelectionListCorrectionMode = eSelectionListCorrectionMode; pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = eCurrSelectionListMode; return ET9STATUS_NONE; } /* do post processing */ pAmbigWord = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; bFound = 0; pWord = 0; for (i = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; i; --i, ++pAmbigWord) { if (!ISEXACTSRC(pAmbigWord->Body.bWordSrc)) { if (__ET9AWIsLikeExactMatch(pLingInfo, pAmbigWord, 0)) { if (!pWord || __PriorityCompare(pLingCmnInfo, pAmbigWord, pWord, SS_Final) > 0) { pWord = pAmbigWord; } } else { bFound = 1; } } else { pExactWord = pAmbigWord; } } if (pExactWord && ISGENUINESRC(pExactWord->Body.bWordSrc)) { pExactWord->Body.bWordSrc = GETBASESRC(pExactWord->Body.bWordSrc); pExactWord->Body.bWordSrc |= EXACTOFFSET; } if (pWord && bFound) { pWord->Body.bWordSrc |= EXACTISHOFFSET; } /* before sorting: disposable quality words goes into the stem pool (e.g. RDB words, not UDB words) stem quality words goes (back) into the stem pool (can have changed source during build) */ pAmbigWord = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; for (i = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; i; --i, ++pAmbigWord) { if (pAmbigWord->Body.bWordQuality <= STEM_QUALITY) { pAmbigWord->Body.bWordSrc = ET9WORDSRC_STEMPOOL; } } /* sort */ __ET9AWSortPriorityList(pLingCmnInfo); pLingCmnInfo->Private.nDefaultIndex = 0; /* result info */ ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= 0xFF); *pbTotalWords = (ET9U8)(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords > 0xFF ? 0xFF : pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); /* restore settings */ pLingCmnInfo->Private.eSelectionListMode = eSelectionListMode; pLingCmnInfo->Private.eSelectionListCorrectionMode = eSelectionListCorrectionMode; pLingCmnInfo->Private.sWordC.pCurrC->eCurrSelectionListMode = eCurrSelectionListMode; return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ ET9STATUS ET9FARCALL _ET9ASpellCheckSelLstBuild(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9U8 * const pbTotalWords, ET9U8 * const pbDefaultListIndex) { ET9STATUS wStatus; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pbTotalWords != NULL); ET9AssertLog(pbDefaultListIndex != NULL); WLOG2(fprintf(pLogFile2, "_ET9ASpellCheckSelLstBuild\n");) if (pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs > ET9MAXWORDSIZE) { return ET9STATUS_ERROR; } pLingCmnInfo->Base.bSymbsInfoInvalidated = 0; wStatus = __ET9AWDoSCSelLstBuild(pLingInfo, dwLdbNum, pbTotalWords); pLingCmnInfo->Private.bLastBuildLen = pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs; pLingCmnInfo->Private.sWordC.pCurrC->dwStateBits |= ET9SLEXACTINLIST; ET9AssertLog(pLingCmnInfo->Private.nDefaultIndex <= 0xFF); *pbDefaultListIndex = (ET9U8)pLingCmnInfo->Private.nDefaultIndex; if (!*pbTotalWords) { wStatus = ET9STATUS_NO_MATCHING_WORDS; } return wStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9U32 ET9LOCALCALL __ET9AWCalculateWordListChecksum(ET9AWLingInfo * const pLingInfo) { ET9U8 bIndex; ET9SYMB *psSymb; ET9U16 wCount; ET9U32 dwHashValue = 0; ET9AWWordInfo *pWord; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pLingInfo->pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= 0xFF); for (bIndex = 0; bIndex < pLingInfo->pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++bIndex) { if (ET9AWSelLstGetWord(pLingInfo, &pWord, bIndex) != ET9STATUS_NONE) { continue; } psSymb = pWord->sWord; for (wCount = pWord->wWordLen; wCount; --wCount, ++psSymb) { dwHashValue = *psSymb + (65599 * dwHashValue); } psSymb = pWord->sSubstitution; for (wCount = pWord->wSubstitutionLen; wCount; --wCount, ++psSymb) { dwHashValue = *psSymb + (65599 * dwHashValue); } } return dwHashValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __ET9AWRemoveDuplicates(ET9AWLingInfo * const pLingInfo) { ET9UINT nLook; ET9UINT nIndex; ET9AWPrivWordInfo *pWord1; ET9AWPrivWordInfo *pWord2; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AssertLog(pLingInfo != NULL); /* remove initial words with length zero */ while (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[0]]; if (pWord1->Base.wWordLen) { break; } __RemoveWord(pLingCmnInfo, 0); } /* remove duplicates */ for (nIndex = 0; nIndex + 1 < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nIndex]]; for (nLook = nIndex + 1; nLook < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nLook) { ET9BOOL bRemove = 0; /* keep here */ pWord2 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nLook]]; if (!pWord2->Base.wWordLen) { bRemove = 1; } else if (pWord1->Base.wWordLen == pWord2->Base.wWordLen) { /* keep declarations here */ ET9U16 wCmpLen = pWord1->Base.wWordLen; ET9SYMB *pStr1 = pWord1->Base.sWord; ET9SYMB *pStr2 = pWord2->Base.sWord; ++wCmpLen; while (--wCmpLen) { if (*pStr1++ != *pStr2++) { break; } } if (!wCmpLen) { bRemove = 1; } } if (bRemove) { if (!pWord1->Base.wSubstitutionLen && pWord2->Base.wSubstitutionLen) { _ET9SymCopy(pWord1->Base.sSubstitution, pWord2->Base.sSubstitution, pWord2->Base.wSubstitutionLen); pWord1->Base.wSubstitutionLen = pWord2->Base.wSubstitutionLen; } __RemoveWord(pLingCmnInfo, nLook); --nLook; /* need to revisit this position */ } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __ET9AWPostShiftWord(ET9AWLingInfo * const pLingInfo, const ET9POSTSHIFTMODE eMode, ET9AWWordInfo * const pWord) { ET9BOOL bFirst; ET9STATUS wStatus; ET9U16 wCount; ET9SYMB *psSymb; ET9U32 dwLdbNum; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pWord != NULL); /* handle shift change */ bFirst = 1; wCount = pWord->wWordLen; psSymb = pWord->sWord; dwLdbNum = (pWord->bLangIndex == ET9AWSECOND_LANGUAGE) ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum; for (; wCount; --wCount, ++psSymb, bFirst = 0) { if (eMode == ET9POSTSHIFTMODE_UPPER || (eMode == ET9POSTSHIFTMODE_INITIAL && bFirst)) { *psSymb = _ET9SymToUpper(*psSymb, dwLdbNum); } else { *psSymb = _ET9SymToLower(*psSymb, dwLdbNum); } } bFirst = 1; wCount = pWord->wSubstitutionLen; psSymb = pWord->sSubstitution; for (; wCount; --wCount, ++psSymb, bFirst = 0) { if (eMode == ET9POSTSHIFTMODE_UPPER || (eMode == ET9POSTSHIFTMODE_INITIAL && bFirst)) { *psSymb = _ET9SymToUpper(*psSymb, dwLdbNum); } else { *psSymb = _ET9SymToLower(*psSymb, dwLdbNum); } } /* handle OTFM */ wStatus = _ET9_ConvertBuildBuf(pLingInfo, pWord); if (wStatus == ET9STATUS_NO_CHAR || wStatus == ET9STATUS_ERROR) { pWord->wWordLen = 0; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * / * * */ static ET9STATUS ET9LOCALCALL __ET9AWPostShift(ET9AWLingInfo * const pLingInfo, const ET9POSTSHIFTMODE eMode, ET9U8 * const pbTotalWords, ET9U8 * const pbCurrListIndex) { ET9U8 bIndex; ET9STATUS wStatus; ET9AWWordInfo *pWord; ET9AWWordInfo sActiveWord; ET9AWPrivWordInfo *pWord1; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9AssertLog(pLingInfo != NULL); ET9AssertLog(pbTotalWords != NULL); ET9AssertLog(pbCurrListIndex != NULL); /* record the active word */ wStatus = ET9AWSelLstGetWord(pLingInfo, &pWord, *pbCurrListIndex); if (wStatus) { return wStatus; } sActiveWord = *pWord; /* update the list */ if (eMode == ET9POSTSHIFTMODE_DEFAULT) { { ET9U16 wGestureValue; wStatus = __ET9AWDoSelLstBuild(pLingInfo, pbTotalWords, 0, &wGestureValue); } if (wStatus) { return wStatus; } } else { ET9AssertLog(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords <= 0xFF); for (bIndex = 0; bIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++bIndex) { wStatus = ET9AWSelLstGetWord(pLingInfo, &pWord, bIndex); if (wStatus) { return wStatus; } __ET9AWPostShiftWord(pLingInfo, eMode, pWord); } __ET9AWRemoveDuplicates(pLingInfo); *pbTotalWords = (ET9U8)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; } /* find the active word again */ if (*pbCurrListIndex >= pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { *pbCurrListIndex = (ET9U8)(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1); } for (bIndex = 0; bIndex + 1 < (ET9U8)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++bIndex) { pWord1 = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[bIndex]]; if (pWord1->Base.wWordLen == sActiveWord.wWordLen) { /* keep declarations here */ ET9U16 wCmpLen = pWord1->Base.wWordLen; ET9SYMB *pStr1 = pWord1->Base.sWord; ET9SYMB *pStr2 = sActiveWord.sWord; ET9U32 dwLdbNum = (pWord1->Body.bLangIndexScoring == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; ET9U32 dwLdbNum2 = (sActiveWord.bLangIndex == ET9AWSECOND_LANGUAGE) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; ++wCmpLen; while (--wCmpLen) { if (_ET9SymToLower(*pStr1++, dwLdbNum) != _ET9SymToLower(*pStr2++, dwLdbNum2)) { break; } } if (!wCmpLen) { *pbCurrListIndex = bIndex; break; } } } /* save new shift mode */ pLingCmnInfo->Base.pWordSymbInfo->Private.eCurrPostShiftMode = eMode; /* done */ __LogPartialSelList(pLingInfo->pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords - 1, "After Post Shift OP", pLogFile2); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * Do post shift. * Setting shift mode "next" will move to the next post shift mode of lower, initial, upper and default. * If the next'ed list didn't change anything in the list it will move on to the next etc until there is * a change or the modes are exhausted. * * @param pLingInfo Pointer to alphabetic information structure. * @param eMode Post shift mode to set. * @param pbTotalWords (out) number of candidate words found. * @param pbCurrListIndex (in/out) current active candidate. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWSelLstPostShift(ET9AWLingInfo * const pLingInfo, const ET9POSTSHIFTMODE eMode, ET9U8 * const pbTotalWords, ET9U8 * const pbCurrListIndex) { ET9STATUS wStatus; if ((wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo)) != ET9STATUS_NONE) { return wStatus; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; /* user postshifting deactivates auto-capping AND downshifting logic */ ET9AssertLog(pLingCmnInfo); pLingCmnInfo->Base.pWordSymbInfo->Private.bCompoundingDownshift = 0; pLingCmnInfo->Base.pWordSymbInfo->Private.eAutocapWord = ET9AUTOCAP_OFF; if (pbTotalWords == NULL || pbCurrListIndex == NULL) { return ET9STATUS_INVALID_MEMORY; } if (pLingCmnInfo->Base.bSymbsInfoInvalidated) { return ET9STATUS_NEED_SELLIST_BUILD; } if (!pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return ET9STATUS_NEED_SELLIST_BUILD; } if (eMode >= ET9POSTSHIFTMODE_LAST) { return ET9STATUS_BAD_PARAM; } WLOG2(fprintf(pLogFile2, "Post Shift - mode %d\n", eMode);) /* handle explicit modes */ if (eMode != ET9POSTSHIFTMODE_NEXT) { return __ET9AWPostShift(pLingInfo, eMode, pbTotalWords, pbCurrListIndex); } /* handle mode "next" */ { ET9INT nModeCount; ET9U32 dwNewCheckSum; const ET9U32 dwOldCheckSum = __ET9AWCalculateWordListChecksum(pLingInfo); ET9POSTSHIFTMODE eCurrMode = pLingCmnInfo->Base.pWordSymbInfo->Private.eCurrPostShiftMode; for (nModeCount = 4; nModeCount; --nModeCount) { switch (eCurrMode) { case ET9POSTSHIFTMODE_LOWER: eCurrMode = ET9POSTSHIFTMODE_INITIAL; break; case ET9POSTSHIFTMODE_INITIAL: eCurrMode = ET9POSTSHIFTMODE_UPPER; break; case ET9POSTSHIFTMODE_UPPER: eCurrMode = ET9POSTSHIFTMODE_DEFAULT; break; case ET9POSTSHIFTMODE_DEFAULT: eCurrMode = ET9POSTSHIFTMODE_LOWER; break; default: ET9AssertLog(0); break; } if ((wStatus = __ET9AWPostShift(pLingInfo, eCurrMode, pbTotalWords, pbCurrListIndex)) != ET9STATUS_NONE) { return wStatus; } dwNewCheckSum = __ET9AWCalculateWordListChecksum(pLingInfo); WLOG2(fprintf(pLogFile2, "Post Shift - old checksum = %x, new checksum = %x\n", dwOldCheckSum, dwNewCheckSum);) if (dwNewCheckSum != dwOldCheckSum) { break; } WLOG2(fprintf(pLogFile2, "Post Shift - trying next mode\n");) } } } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * Get post shift mode. * * @param pLingInfo Pointer to alphabetic information structure. * @param peMode (out) current post shift mode. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWGetPostShiftMode(ET9AWLingInfo * const pLingInfo, ET9POSTSHIFTMODE * const peMode) { ET9STATUS wStatus; if ((wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo)) != ET9STATUS_NONE) { return wStatus; } if (!peMode) { return ET9STATUS_INVALID_MEMORY; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; *peMode = pLingCmnInfo->Base.pWordSymbInfo->Private.eCurrPostShiftMode; if (pLingCmnInfo->Base.bSymbsInfoInvalidated || !pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords) { return ET9STATUS_NEED_SELLIST_BUILD; } } return ET9STATUS_NONE; } #ifdef ET9_ASLST_ACTIVATE_LOG #undef ET9_DEBUGLOG2 #undef WLOG2 #undef __LogCDB #undef WLOG2Word #undef WLOG2String #endif #endif /* ET9_ALPHABETIC_MODULE */ /*! @} */ /* ----------------------------------< eof >--------------------------------- */