/******************************************************************************* ;******************************************************************************* ;** ** ;** COPYRIGHT 2011-2012 NUANCE COMMUNICATIONS ** ;** ** ;** NUANCE COMMUNICATIONS PROPRIETARY INFORMATION ** ;** ** ;** This software is supplied under the terms of a license agreement ** ;** or non-disclosure agreement with Nuance Communications and may not ** ;** be copied or disclosed except in accordance with the terms of that ** ;** agreement. ** ;** ** ;******************************************************************************* ;** ** ;** FileName: et9adlm.c ** ;** ** ;** Description: Dynamic LM source file. ** ;** ** ;******************************************************************************* ;******************************************************************************/ /*! \addtogroup et9adlm Dynamic LM for alphabetic * XT9 alphabetic dynamic LM features. * @{ */ #include "et9api.h" #ifdef ET9_ALPHABETIC_MODULE #include "et9adlm.h" #include "et9amisc.h" #include "et9aldb.h" #include "et9amdb.h" #include "et9sym.h" #include "et9asym.h" #include "et9asys.h" #ifdef ET9_DEBUGLOG2 #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG2 ACTIVATED ***") #endif #include #include #define WLOG2(q) { if (pLogFile2 == NULL) { pLogFile2 = fopen("zzzET9ADLM.txt", "w+"); } {q} fflush(pLogFile2); } static FILE *pLogFile2 = NULL; #else #define WLOG2(q) #endif #if 0 #ifdef _WIN32 #define _ET9_DLM_REVERSE_HASH #pragma message ("*** DLM reverse hash active ***") #endif #endif #if 0 #define _ET9_DLM_AUTO_CONTENT #pragma message ("*** DLM auto content active ***") #endif #if defined(_ET9_DLM_REVERSE_HASH) || defined(_ET9_DLM_AUTO_CONTENT) #include #endif /* ************************************************************************************************************** */ /* * DATA TYPES ************************************************************************************************* */ /* ************************************************************************************************************** */ #define __DLM_Version 1 /* < -IDR- */ #define __DLM_MAX_EntityCount 0xFFF0 /* < -IDR- */ #define __DLM_WordCount 7000 /* < -IDR- */ #define __DLM_PoolCharCount (__DLM_WordCount * 7) /* < -IDR- */ #define __DLM_ContextCount 20000 /* < -IDR- */ #define __DLM_ContextLengths 2 /* < -IDR- */ #define __DLM_MaxListChunks 10 /* < -IDR- * */ #define __DLM_ChunkWordCount 6 /* < -IDR- */ #define __DLM_ListChunkCount (__DLM_ContextCount * 3 / 5) /* < -IDR- */ #define __DLM_Index_NULL 0xFFFF /* < -IDR- */ #define __DLM_Index_Unused 0xFFFE /* < -IDR- */ #define __DLM_AnyLanguage 0xFFFF /* < -IDR- */ #define __DLM_UserMaxQuality 99 /* < -IDR- */ #define __DLM_UserAddQuality 15 /* < -IDR- */ #define __DLM_StaticXQuality 3 /* < -IDR- */ #define __DLM_StaticCQuality 2 /* < -IDR- */ #define __DLM_NormalQuality 0 /* < -IDR- */ #define __DLM_InitialLowQuality -1 /* < -IDR- */ #define __DLM_CutLowQuality (__DLM_InitialLowQuality - 10) /* < -IDR- */ #define __DLM_HiddenQuality -14 /* < -IDR- */ #define __DLM_UserKillQuality -15 /* < -IDR- */ #define __DLM_UserMinQuality -99 /* < -IDR- */ #define __DLM_MAX_USE_VALUE 0xFFFF /* < -IDR- */ #define __DLM_GetWords_CutOffLevel __DLM_InitialLowQuality /* < -IDR- */ #define __DLM_FindWord_CutOffLevel __DLM_InitialLowQuality /* < -IDR- */ #if __DLM_ContextLengths > ET9NLM_CONTEXT_COUNT #error Currently no support for this context length (check ET9NLM_CONTEXT_COUNT) #endif #if __DLM_WordCount > __DLM_MAX_EntityCount #error Too big __DLM_WordCount #endif #if __DLM_ContextCount > __DLM_MAX_EntityCount #error __DLM_ContextCount #endif #if __DLM_ListChunkCount > __DLM_MAX_EntityCount #error __DLM_ListChunkCount #endif #if __DLM_PoolCharCount > __DLM_MAX_EntityCount #error __DLM_PoolCharCount #endif #if __DLM_MaxListChunks * __DLM_ChunkWordCount > 0xFF #error Too big prediction lists #endif #if __DLM_CutLowQuality <= __DLM_HiddenQuality #error Config error #endif #if __DLM_HiddenQuality <= __DLM_UserKillQuality #error Config error #endif /* Start packing on byte boundaries */ #pragma pack (1) /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef struct __DLM_PredictionListItem_s { ET9U16 wWordIndex; /* < -IDR- */ ET9U16 wUseCount; /* < -IDR- */ } __DLM_PredictionListItem; /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef struct __DLM_PredictionListChunk_s { __DLM_PredictionListItem pItems[__DLM_ChunkWordCount]; /* < -IDR- */ ET9U16 wNextChunkIndex; /* < -IDR- */ } __DLM_PredictionListChunk; /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * * */ typedef struct __DLM_PredictionList_s { ET9U32 dwHashValue; /* < -IDR- */ ET9U16 wFirstChunkIndex; /* < -IDR- */ ET9U8 bPredictionCount; /* < -IDR- */ } __DLM_PredictionList; /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef struct __DLM_PredictionsList_s { ET9U16 wUsedPredictionCount; /* < -IDR- */ __DLM_PredictionList pPredictions[__DLM_ContextCount]; /* < -IDR- */ ET9U16 wUsedListChunkCount; /* < -IDR- */ __DLM_PredictionListChunk pPredictionListChunks[__DLM_ListChunkCount]; /* < -IDR- */ } __DLM_PredictionsList; /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef struct __DLM_WordItem_s { ET9U32 dwStringHash; /* < -IDR- */ ET9U16 wCharIndex; /* < -IDR- */ ET9U16 wUseCount; /* < -IDR- */ ET9U16 wAccessIndex; /* < -IDR- */ ET9U16 wLanguageId; /* < -IDR- / */ ET9U8 bWordLength; /* < -IDR- */ ET9S8 sbQuality; /* < -IDR- */ ET9BOOL bIsCapitalized; /* < -IDR- */ } __DLM_WordItem; /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * */ typedef struct __DLM_WordStore_s { ET9BOOL bHasKillWords; /* < -IDR- */ ET9U16 wFreeCharCount; /* < -IDR- */ ET9SYMB psPoolChars[__DLM_PoolCharCount]; /* < -IDR- */ ET9U16 wWordCount; /* < -IDR- */ __DLM_WordItem pWordItems[__DLM_WordCount]; /* < -IDR- */ ET9U16 pwWordAccess[__DLM_WordCount]; /* < -IDR- */ } __DLM_WordStore; /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * * */ struct _ET9DLM_info_s { ET9U8 bVersion; /* < -IDR- */ ET9U16 wInitOk; /* < -IDR- */ ET9U32 dwDataSize; /* < -IDR- */ ET9U16 wDataWordCount; /* < -IDR- */ ET9U16 wDataPoolCharCount; /* < -IDR- */ ET9U16 wDataContextCount; /* < -IDR- */ ET9U16 wDataListChunkCount; /* < -IDR- */ ET9U32 dwUpdateCounter; /* < -IDR- */ __DLM_PredictionsList pPredictionsLists[__DLM_ContextLengths]; /* < -IDR- */ __DLM_WordStore sWordStore; /* < -IDR- */ }; /* Back to default packing */ #pragma pack () /* ************************************************************************************************************** */ /* * REVERSE HASH SUPPORT *************************************************************************************** */ /* ************************************************************************************************************** */ #ifdef _ET9_DLM_REVERSE_HASH #define __MAX_REVERSE_HASH_COUNT (__DLM_ContextCount) static ET9BOOL __bReverseHashModified = 0; static char __pcReverseHashFileName[] = "xx-dlm-rh.bin"; typedef struct __reverseHashItem_s { ET9U32 dwHashValue; ET9U16 wStringLen; ET9SYMB psString[__DLM_ContextLengths * ET9MAXWORDSIZE + (__DLM_ContextLengths - 1)]; } __reverseHashItem; typedef struct __reverseHashContext_s { ET9U32 dwUsedCount; __reverseHashItem pItems[__MAX_REVERSE_HASH_COUNT]; } __reverseHashContext; typedef struct __reverseHashTable_s { __reverseHashContext pContexts[__DLM_ContextLengths]; } __reverseHashTable; static __reverseHashTable __reverseHash; static void ET9LOCALCALL __ReverseHash_Persist(void) { if (__bReverseHashModified) { FILE * pf = fopen(__pcReverseHashFileName, "wb"); if (pf) { __bReverseHashModified = 0; fseek(pf, 0, SEEK_SET); fwrite(&__reverseHash, sizeof(__reverseHash), 1, pf); fflush(pf); fclose(pf); } } } static void ET9LOCALCALL __ReverseHash_Reset(void) { ET9UINT nIndex; for (nIndex = 0; nIndex < __DLM_ContextLengths; ++nIndex) { __reverseHash.pContexts[nIndex].dwUsedCount = 0; } __bReverseHashModified = 1; __ReverseHash_Persist(); } static void ET9LOCALCALL __ReverseHash_Restore(void) { ET9BOOL bRestoredOk = 0; FILE *pf; __ReverseHash_Persist(); pf = fopen(__pcReverseHashFileName, "rb"); if (pf) { if (fread(&__reverseHash, sizeof(__reverseHash), 1, pf) == 1) { bRestoredOk = 1; } fclose(pf); } if (!bRestoredOk) { __ReverseHash_Reset(); } } static __reverseHashItem const * ET9LOCALCALL __ReverseHash_GetItem(const ET9UINT nContext, const ET9U32 dwHashValue) { ET9U32 dwIndex; ET9Assert(nContext < __DLM_ContextLengths); ET9Assert(__reverseHash.pContexts[nContext].dwUsedCount <= __MAX_REVERSE_HASH_COUNT); for (dwIndex = 0; dwIndex < __reverseHash.pContexts[nContext].dwUsedCount; ++dwIndex) { if (__reverseHash.pContexts[nContext].pItems[dwIndex].dwHashValue == dwHashValue) { return &__reverseHash.pContexts[nContext].pItems[dwIndex]; } } return NULL; } static void ET9LOCALCALL __ReverseHash_RemoveItem(const ET9UINT nContext, const ET9U32 dwHashValue) { ET9U32 dwIndex; ET9Assert(nContext < __DLM_ContextLengths); ET9Assert(__reverseHash.pContexts[nContext].dwUsedCount <= __MAX_REVERSE_HASH_COUNT); for (dwIndex = 0; dwIndex < __reverseHash.pContexts[nContext].dwUsedCount; ++dwIndex) { if (__reverseHash.pContexts[nContext].pItems[dwIndex].dwHashValue == dwHashValue) { __reverseHash.pContexts[nContext].pItems[dwIndex] = __reverseHash.pContexts[nContext].pItems[--__reverseHash.pContexts[nContext].dwUsedCount]; __bReverseHashModified = 1; return; } } ET9Assert(0); } static void ET9LOCALCALL __ReverseHash_AddItem(const ET9UINT nContext, __reverseHashItem const * const pItem) { ET9Assert(nContext < __DLM_ContextLengths); if (__reverseHash.pContexts[nContext].dwUsedCount >= __MAX_REVERSE_HASH_COUNT) { ET9Assert(0); return; } __reverseHash.pContexts[nContext].pItems[__reverseHash.pContexts[nContext].dwUsedCount++] = *pItem; __bReverseHashModified = 1; } #endif /* _ET9_DLM_REVERSE_HASH */ /* ************************************************************************************************************** */ /* * LOG SUPPORT ************************************************************************************************ */ /* ************************************************************************************************************** */ #define __DLM_WORDQ_TOSTRING(x) (x == _ET9AW_DLM_WordQuality_UserHigh ? "UserHigh" : x == _ET9AW_DLM_WordQuality_UserNormal ? "UserNormal" : x == _ET9AW_DLM_WordQuality_UserQuarantine ? "UserQuarantine" : x == _ET9AW_DLM_WordQuality_ScanHigh ? "ScanHigh" : x == _ET9AW_DLM_WordQuality_ScanLow ? "ScanLow" : "\?\?\?") #define __DLM_LINDEX_TOSTRING(x) (x == ET9AWFIRST_LANGUAGE ? "1st" : x == ET9AWSECOND_LANGUAGE ? "2nd" : x == ET9AWBOTH_LANGUAGES ? "both" : x == ET9AWUNKNOWN_LANGUAGE ? "unknown" : "\?\?\?") #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static ET9UINT ET9LOCALCALL WLOG2String(FILE *pF, char *pcComment, ET9SYMB const * const psString, const ET9INT snLen, const ET9BOOL bReverse, const ET9BOOL bSilent) { ET9INT snIndex; ET9UINT nLenUsed = 0; ET9Assert(pF); ET9Assert(pcComment); if (pcComment && *pcComment) { fprintf(pF, "%s: ", pcComment); } if (psString) { for (snIndex = bReverse ? (snLen - 1): 0; bReverse ? snIndex >= 0: snIndex < snLen; bReverse ? --snIndex : ++snIndex) { ET9SYMB sSymb = psString[snIndex]; if (sSymb >= 0x20 && sSymb <= 0xFF) { nLenUsed += fprintf(pF, "%c", (unsigned char)sSymb); } else { nLenUsed += fprintf(pF, "<%x>", (int)sSymb); } } } else { nLenUsed += fprintf(pF, "NULL"); } if (!bSilent) { fprintf(pF, " (%d)\n", snLen); } fflush(pF); #ifdef ET9_DEBUG { ET9U16 wCount = (ET9U16)snLen; ET9SYMB const *pSymb = psString; while (pSymb && wCount--) { ET9Assert(*pSymb && *pSymb != (ET9SYMB)0xcccc); ++pSymb; } } #endif return nLenUsed; } #else /* ET9_DEBUGLOG2 */ #define WLOG2String(pF,pcComment,psString,snLen,bReverse,bSilent) #endif /* ET9_DEBUGLOG2 */ #ifdef ET9_DEBUGLOG2 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __LogDLM(ET9AWLingInfo * const pLingInfo, FILE * const pF, const ET9BOOL bForceLog) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; static ET9U32 dwOldUpdateCounter = 0; if (!pDLM) { fprintf(pF, "\nNo DLM\n\n"); return; } if (!bForceLog && dwOldUpdateCounter == pDLM->dwUpdateCounter) { return; } else { dwOldUpdateCounter = pDLM->dwUpdateCounter; } fprintf(pF, "\nDLM, size %u, Update-Counter %u\n\n\n", pDLM->dwDataSize, pDLM->dwUpdateCounter); { ET9UINT nIndex; { ET9UINT nCharUseCount; nCharUseCount = 0; for (nIndex = 0; nIndex < __DLM_PoolCharCount; ++nIndex) { if (pDLM->sWordStore.psPoolChars[nIndex]) { ++nCharUseCount; } } fprintf(pF, "Unigram word list, WordCount %5u (%6.2f%%), CharCount %5u (%6.2f%%)\n\n", pDLM->sWordStore.wWordCount, (float)pDLM->sWordStore.wWordCount / __DLM_WordCount * 100.0f, nCharUseCount, (float)nCharUseCount / __DLM_PoolCharCount * 100.0f); for (nIndex = 0; nIndex < __DLM_ContextLengths; ++nIndex) { __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nIndex]; fprintf(pF, "%u-gram, UsedPredictionCount %5u (%6.2f%%), UsedListChunkCount %5u (%6.2f%%)\n\n", nIndex + 2, pPredictionList->wUsedPredictionCount, (float)pPredictionList->wUsedPredictionCount / __DLM_ContextCount * 100.0f, pPredictionList->wUsedListChunkCount, (float)pPredictionList->wUsedListChunkCount / __DLM_ListChunkCount * 100.0f); } fprintf(pF, "\n"); } fprintf(pF, "-------------------- Unigram word list\n\n"); fprintf(pF, " Word LanguageId Quality UseCount Capitalized String\n\n"); for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); ET9Assert(pDLM->sWordStore.pwWordAccess[pWord->wAccessIndex] == nIndex); fprintf(pF, " %5u %04x %3d %5u %c ", nIndex, pWord->wLanguageId, pWord->sbQuality, pWord->wUseCount, (pWord->bIsCapitalized ? 'Y' : 'N')); #if 0 fprintf(pF, "%08X %04x %04x %02u ", pWord->Body.dwStringHash, pWord->wCharIndex, pWord->wAccessIndex, pWord->bWordLength); #endif WLOG2String(pF, "", &pDLM->sWordStore.psPoolChars[pWord->wCharIndex], pWord->bWordLength, 0, 1); fprintf(pF, "\n"); } fprintf(pF, "\n\n"); } { ET9UINT nContextIndex; for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths && nContextIndex < ET9NLM_CONTEXT_COUNT; ++nContextIndex) { __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; ET9U16 wPredictionIndex; fprintf(pF, "-------------------- Context %u\n\n\n", nContextIndex + 1); for (wPredictionIndex = 0; wPredictionIndex < pPredictionList->wUsedPredictionCount; ++wPredictionIndex) { __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; fprintf(pF, "Prediction %5u, dwHashValue %10u", wPredictionIndex, pPrediction->dwHashValue); #ifdef _ET9_DLM_REVERSE_HASH { __reverseHashItem const * const pItem = __ReverseHash_GetItem(nContextIndex, pPrediction->dwHashValue); fprintf(pF, ", reverse-hash "); if (pItem) { WLOG2String(pF, "", pItem->psString, pItem->wStringLen, 0, 1); } else { fprintf(pF, "no-reverse-hash-found"); } } #else fprintf(pF, ", no-reverse-hash"); #endif fprintf(pF, "\n "); if (!pPrediction->bPredictionCount) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[pPrediction->wFirstChunkIndex]]; fprintf(pF, "SINGLE :: "); WLOG2String(pF, "", &pDLM->sWordStore.psPoolChars[pWord->wCharIndex], pWord->bWordLength, 0, 1); } else { ET9U8 bWordIndex; __DLM_PredictionListChunk const * pCurrChunk; fprintf(pF, "MULTI %u :: [ ", pPrediction->bPredictionCount); pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { if (pCurrChunk->wNextChunkIndex > __DLM_ListChunkCount) { fprintf(pF, " *** corrupt chunk list ***"); break; } pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; fprintf(pF, "][ "); } { __DLM_PredictionListItem const * const pItem = &pCurrChunk->pItems[bChunkWordIndex]; __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[pItem->wWordIndex]]; fprintf(pF, "{"); WLOG2String(pF, "", &pDLM->sWordStore.psPoolChars[pWord->wCharIndex], pWord->bWordLength, 0, 1); #if 0 fprintf(pF, " %u %u %u ", pItem->wWordIndex, pDLM->sWordStore.pwWordAccess[pItem->wWordIndex], pWord->wCharIndex); #endif fprintf(pF, " %u} ", pItem->wUseCount); } } fprintf(pF, "]"); } fprintf(pF, "\n\n"); } fprintf(pF, "\n"); } } fflush(pF); } #else /* ET9_DEBUGLOG2 */ #define __LogDLM(pLingInfo,pF,bForceLog) #endif /* ET9_DEBUGLOG2 */ /* ************************************************************************************************************** */ /* * LOCAL FUNCTIONS ******************************************************************************************** */ /* ************************************************************************************************************** */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9BOOL ET9LOCALCALL __HasStrangeWordContent(ET9SYMB const * const psString, const ET9UINT nStringLen) { ET9UINT nIndex; ET9UINT nAtCount = 0; ET9BOOL bPrevIsPunct = 0; for (nIndex = 0; nIndex < nStringLen; ++nIndex) { ET9BOOL bSkip = 0; switch (psString[nIndex]) { case '.': if (bPrevIsPunct) { bSkip = 1; } break; case '@': ++nAtCount; break; case '[': case ']': case '{': case '}': case '<': case '>': case '#': case '$': case '%': case '*': case '=': case '+': case '|': case '\\': case '/': bSkip = 1; break; } bPrevIsPunct = (psString[nIndex] == '.') ? 1 : 0; if (bSkip || nAtCount > 1) { return 1; } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9BOOL ET9LOCALCALL __IsNumericOrNumericAndPunct(ET9SYMB const * const psString, const ET9UINT nStringLen) { ET9UINT nDigitCount = 0; ET9UINT nOtherCount = 0; ET9UINT nIndex; for (nIndex = 0; nIndex < nStringLen && !nOtherCount; ++nIndex) { switch (_ET9_GetSymbolClass(psString[nIndex])) { case ET9_NumbrSymbClass: ++nDigitCount; break; case ET9_PunctSymbClass: break; default: ++nOtherCount; break; } } if (!nOtherCount && nDigitCount) { return 1; } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static void ET9LOCALCALL __CleanUpWord(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9SYMB const * const psSrc, const ET9U16 wSrcLen, ET9SYMB * const psDst, ET9U16 * const pwDstLen) { ET9Assert(wSrcLen <= ET9MAXWORDSIZE); /* creaty string copy */ _ET9SymCopy(psDst, psSrc, wSrcLen); *pwDstLen = wSrcLen; /* if emoticon, keep as is */ if (_ET9StringLikelyEmoticon(psDst, *pwDstLen)) { return; } /* if URL, keep as is */ if (_ET9StringLikelyURL(psDst, *pwDstLen)) { return; } /* if phone number, keep as is */ if (_ET9StringLikelyPhoneNumber(psDst, *pwDstLen)) { return; } /* if surrounded by "(, etc, skip them */ while (*pwDstLen && (psDst[*pwDstLen - 1] == '"' || psDst[*pwDstLen - 1] == '(' || psDst[*pwDstLen - 1] == ')' || psDst[*pwDstLen - 1] == ',' || psDst[*pwDstLen - 1] == ':' || psDst[*pwDstLen - 1] == ';')) { --*pwDstLen; } while (*pwDstLen && (psDst[0] == '"' || psDst[0] == '(' || psDst[0] == ')' || psDst[0] == ',' || psDst[0] == ':' || psDst[0] == ';')) { _ET9SymMove(&psDst[0], &psDst[1], *pwDstLen - 1); --*pwDstLen; } if (!*pwDstLen) { return; } /* if single punct, skip it */ if (*pwDstLen == 1 && _ET9_IsPunctChar(psDst[0])) { *pwDstLen = 0; return; } /* if word exist in the LDB, keep as is (only check if an end has punct) */ if (_ET9_IsPunctChar(psDst[*pwDstLen - 1]) || _ET9_IsPunctChar(psDst[0])) { ET9U8 bExact; ET9U8 bLowercase; ET9U32 dwContextWordIndex; if (_ET9AWLdbFindEntry(pLingInfo, dwLdbNum, ET9MGD_FULL_WORD, psDst, *pwDstLen, &dwContextWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { return; } } /* if surrounded by .!? etc, skip them */ while (*pwDstLen && (psDst[*pwDstLen - 1] == '.' || psDst[*pwDstLen - 1] == '!' || psDst[*pwDstLen - 1] == '?' || psDst[*pwDstLen - 1] == '"' || psDst[*pwDstLen - 1] == '(' || psDst[*pwDstLen - 1] == ')' || psDst[*pwDstLen - 1] == ',' || _ET9_IsTermPunct(pLingInfo, dwLdbNum, psDst[*pwDstLen - 1]))) { --*pwDstLen; } while (*pwDstLen && (psDst[0] == '.' || psDst[0] == '!' || psDst[0] == '?' || psDst[0] == '"' || psDst[0] == '(' || psDst[0] == ')' || psDst[0] == ',' || _ET9_IsTermPunct(pLingInfo, dwLdbNum, psDst[0]))) { _ET9SymMove(&psDst[0], &psDst[1], *pwDstLen - 1); --*pwDstLen; } if (!*pwDstLen) { return; } /* if contains strange punctuation or repeated puncts, skip word */ if (__HasStrangeWordContent(psDst, *pwDstLen)) { *pwDstLen = 0; return; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9BOOL ET9LOCALCALL __ScreenWord(ET9SYMB const * const psWord, const ET9U16 wWordLen, const _ET9AW_DLM_WordQuality eQuality) { /* rules that applies to all words */ /* single char punctuation - remove */ if (wWordLen == 1 && _ET9_IsPunctChar(psWord[0])) { return 1; } /* rules that only applies to words of "non user explicit" quality */ if (eQuality >= _ET9AW_DLM_WordQuality_UserHigh || eQuality == _ET9AW_DLM_WordQuality_ScanHigh) { return 0; } /* digits only, or digits mixed with puncts - remove */ if (__IsNumericOrNumericAndPunct(psWord, wWordLen)) { return 1; } /* done */ return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9UINT ET9LOCALCALL __UpdateContextData(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9SimpleWord const * const pLocalContext, const ET9UINT nLocalContextLen) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9UINT nContextIndex; ET9UINT nSrcIndex; ET9UINT nSrcLocalIndex; ET9UINT nDstIndex; nSrcIndex = 0; nSrcLocalIndex = 0; nDstIndex = 0; /* init internal contexts */ for (nContextIndex = 0; nContextIndex < ET9NLM_CONTEXT_COUNT; ++nContextIndex) { pLingCmnInfo->Private.sCurrContext.pDLMContextWords[nContextIndex].wLen = 0; } while (nDstIndex < ET9NLM_CONTEXT_COUNT && (nSrcIndex < ET9NLM_CONTEXT_COUNT || nSrcLocalIndex < nLocalContextLen)) { ET9SimpleWord * const pDstWord = &pLingCmnInfo->Private.sCurrContext.pDLMContextWords[nDstIndex]; if (nSrcLocalIndex < nLocalContextLen) { *pDstWord = pLocalContext[nSrcLocalIndex++]; } else { *pDstWord = pLingCmnInfo->Private.pContextWords[nSrcIndex++]; } /* if empty, break */ if (!pDstWord->wLen) { #if 0 pDstWord->wLen = 0; pDstWord->sString[pDstWord->wLen++] = '<'; pDstWord->sString[pDstWord->wLen++] = 'P'; pDstWord->sString[pDstWord->wLen++] = 'S'; pDstWord->sString[pDstWord->wLen++] = '>'; #else pDstWord->wLen = 0; pDstWord->sString[pDstWord->wLen++] = '.'; #endif ++nDstIndex; break; } /* if surrounded by "(, etc, skip them */ while (pDstWord->wLen && (pDstWord->sString[pDstWord->wLen - 1] == '"' || pDstWord->sString[pDstWord->wLen - 1] == ',' || pDstWord->sString[pDstWord->wLen - 1] == '(' || pDstWord->sString[pDstWord->wLen - 1] == ')')) { --pDstWord->wLen; } while (pDstWord->wLen && (pDstWord->sString[0] == '"' || pDstWord->sString[0] == ',' || pDstWord->sString[0] == '(' || pDstWord->sString[0] == ')')) { _ET9SymMove(&pDstWord->sString[0], &pDstWord->sString[1], pDstWord->wLen - 1); --pDstWord->wLen; } if (!pDstWord->wLen) { continue; } /* if single hyphen, skip it */ if (pDstWord->wLen == 1 && pDstWord->sString[0] == '-') { continue; } #if 0 /* if new paragraph, keep and skip the rest */ if (pDstWord->sString[pDstWord->wLen - 1] == 0x2029) { pDstWord->wLen = 0; pDstWord->sString[pDstWord->wLen++] = '<'; pDstWord->sString[pDstWord->wLen++] = 'P'; pDstWord->sString[pDstWord->wLen++] = 'S'; pDstWord->sString[pDstWord->wLen++] = '>'; ++nDstIndex; break; } /* if new line, keep and skip the rest */ if (pDstWord->sString[pDstWord->wLen - 1] == 0xA || pDstWord->sString[pDstWord->wLen - 1] == 0xD || pDstWord->sString[pDstWord->wLen - 1] == 0x2028) { pDstWord->wLen = 0; pDstWord->sString[pDstWord->wLen++] = '<'; pDstWord->sString[pDstWord->wLen++] = 'L'; pDstWord->sString[pDstWord->wLen++] = 'S'; pDstWord->sString[pDstWord->wLen++] = '>'; ++nDstIndex; break; } #endif /* if word exist in the LDB, keep as is (only check if an end has punct) */ if (_ET9_IsPunctChar(pDstWord->sString[pDstWord->wLen - 1]) || _ET9_IsPunctChar(pDstWord->sString[0])) { ET9U8 bExact; ET9U8 bLowercase; ET9U32 dwContextWordIndex; if (_ET9AWLdbFindEntry(pLingInfo, dwLdbNum, ET9MGD_FULL_WORD, pDstWord->sString, pDstWord->wLen, &dwContextWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { ++nDstIndex; continue; } } /* if sentence terminator, keep terminator and skip the rest */ { const ET9SYMB sLastSymb = pDstWord->sString[pDstWord->wLen - 1]; if (sLastSymb == '.' || sLastSymb == '!' || sLastSymb == '?' || sLastSymb == ':' || sLastSymb == ';' || _ET9_IsTermPunct(pLingInfo, dwLdbNum, sLastSymb)) { pDstWord->wLen = 1; pDstWord->sString[0] = sLastSymb; ++nDstIndex; break; } } /* if numeric string, make it */ if (_ET9_IsNumericString(pDstWord->sString, pDstWord->wLen)) { const ET9SYMB sSymb1 = (pDstWord->wLen > 1) ? pDstWord->sString[pDstWord->wLen - 2] : '0'; const ET9SYMB sSymb2 = (pDstWord->wLen > 0) ? pDstWord->sString[pDstWord->wLen - 1] : '0'; pDstWord->wLen = 0; pDstWord->sString[pDstWord->wLen++] = '<'; pDstWord->sString[pDstWord->wLen++] = 'N'; pDstWord->sString[pDstWord->wLen++] = 'U'; pDstWord->sString[pDstWord->wLen++] = 'M'; if ((dwLdbNum & ET9PLIDMASK) == ET9PLIDEnglish) { pDstWord->sString[pDstWord->wLen++] = '-'; if (sSymb1 != '1' && sSymb2 >= '1' && sSymb2 <= '3') { if (sSymb2 == '1') { pDstWord->sString[pDstWord->wLen++] = 'S'; pDstWord->sString[pDstWord->wLen++] = 'T'; } else if (sSymb2 == '2') { pDstWord->sString[pDstWord->wLen++] = 'N'; pDstWord->sString[pDstWord->wLen++] = 'D'; } else { pDstWord->sString[pDstWord->wLen++] = 'R'; pDstWord->sString[pDstWord->wLen++] = 'D'; } } else { pDstWord->sString[pDstWord->wLen++] = 'T'; pDstWord->sString[pDstWord->wLen++] = 'H'; } } pDstWord->sString[pDstWord->wLen++] = '>'; ++nDstIndex; continue; } /* if numeric and punct (numeric only above), make it */ if (__IsNumericOrNumericAndPunct(pDstWord->sString, pDstWord->wLen)) { pDstWord->wLen = 0; pDstWord->sString[pDstWord->wLen++] = '<'; pDstWord->sString[pDstWord->wLen++] = 'N'; pDstWord->sString[pDstWord->wLen++] = 'U'; pDstWord->sString[pDstWord->wLen++] = 'M'; pDstWord->sString[pDstWord->wLen++] = 'P'; pDstWord->sString[pDstWord->wLen++] = 'U'; pDstWord->sString[pDstWord->wLen++] = 'N'; pDstWord->sString[pDstWord->wLen++] = 'C'; pDstWord->sString[pDstWord->wLen++] = 'T'; pDstWord->sString[pDstWord->wLen++] = '>'; ++nDstIndex; continue; } /* if strange content make it */ if (__HasStrangeWordContent(pDstWord->sString, pDstWord->wLen)) { pDstWord->wLen = 0; pDstWord->sString[pDstWord->wLen++] = '<'; pDstWord->sString[pDstWord->wLen++] = 'U'; pDstWord->sString[pDstWord->wLen++] = 'N'; pDstWord->sString[pDstWord->wLen++] = 'K'; pDstWord->sString[pDstWord->wLen++] = '>'; ++nDstIndex; continue; } /* keep as is... */ ++nDstIndex; } return nDstIndex; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9U16 ET9LOCALCALL __NewListChunk(__DLM_PredictionsList * const pPredictionList) { ET9U16 wIndex; if (pPredictionList->wUsedListChunkCount >= __DLM_ListChunkCount) { ET9Assert(0); return __DLM_Index_NULL; } for (wIndex = 0; wIndex < __DLM_ListChunkCount; ++wIndex) { __DLM_PredictionListChunk * const pChunk = &pPredictionList->pPredictionListChunks[wIndex]; if (pChunk->wNextChunkIndex == __DLM_Index_Unused) { pChunk->wNextChunkIndex = __DLM_Index_NULL; ++pPredictionList->wUsedListChunkCount; return wIndex; } } ET9Assert(0); return __DLM_Index_NULL; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __FreeListChunkChain(__DLM_PredictionsList * const pPredictionList, ET9U16 * const pwListChunkIndex) { while (*pwListChunkIndex != __DLM_Index_NULL) { __DLM_PredictionListChunk * const pListChunk = &pPredictionList->pPredictionListChunks[*pwListChunkIndex]; WLOG2(fprintf(pLogFile2, "__FreeListChunkChain, wListChunkIndex %u\n", *pwListChunkIndex);) ET9Assert(pPredictionList->wUsedListChunkCount); --pPredictionList->wUsedListChunkCount; *pwListChunkIndex = pListChunk->wNextChunkIndex; pListChunk->wNextChunkIndex = __DLM_Index_Unused; } } #ifdef ET9_DEBUG /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __ValidatePrediction(__DLM_PredictionsList const * const pPredictionList, __DLM_PredictionList const * const pPrediction) { ET9U8 bListItemCount = 0; ET9U16 pwWordIndex[__DLM_MaxListChunks * __DLM_ChunkWordCount]; ET9U8 bWordIndex; __DLM_PredictionListChunk const * pCurrChunk; if (pPrediction->bPredictionCount < 2) { return; } pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { ET9Assert(pCurrChunk->wNextChunkIndex < __DLM_ListChunkCount); pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; } pwWordIndex[bListItemCount++] = pCurrChunk->pItems[bChunkWordIndex].wWordIndex; } { ET9U8 bIndex; ET9BOOL bDirty; for (bDirty = 1; bDirty; ) { bDirty = 0; for (bIndex = 0; bIndex + 1 < bListItemCount; ++bIndex) { if (pwWordIndex[bIndex] > pwWordIndex[bIndex + 1]) { const ET9U16 wTMP = pwWordIndex[bIndex]; pwWordIndex[bIndex] = pwWordIndex[bIndex + 1]; pwWordIndex[bIndex + 1] = wTMP; bDirty = 1; } } } } { ET9U8 bIndex; for (bIndex = 0; bIndex + 1 < bListItemCount; ++bIndex) { ET9Assert(pwWordIndex[bIndex] != pwWordIndex[bIndex + 1]); } } } #else #define __ValidatePrediction(pPredictionList, pPrediction) #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __PredictionOrderChanged(ET9AWLingInfo * const pLingInfo, const ET9UINT nContextIndex) { ++pLingInfo->pLingCmnInfo->Private.__DLM_Private.pPredictionCache[nContextIndex].dwCurrentState; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9U16 ET9LOCALCALL __FindPrediction(ET9AWLingInfo * const pLingInfo, const ET9UINT nContextIndex, const ET9U32 dwHashValue) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; __DLM_PredictionsList * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; _DLM_PredictionCacheItem * const pCache = &pLingCmnInfo->Private.__DLM_Private.pPredictionCache[nContextIndex]; ET9Assert(nContextIndex < __DLM_ContextLengths); /* check the cache - both for minor and major change */ if (pCache->dwIndexState == pCache->dwCurrentState && pCache->dwIndexHash == dwHashValue) { ET9Assert(pCache->wIndex >= pPredictionList->wUsedPredictionCount || pPredictionList->pPredictions[pCache->wIndex].dwHashValue == dwHashValue); #ifdef ET9_DEBUG if (pCache->wIndex >= pPredictionList->wUsedPredictionCount) { /* verify that the hash value is not in the list */ ET9U16 wIndex; for (wIndex = 0; wIndex < pPredictionList->wUsedPredictionCount; ++wIndex) { ET9Assert(pPredictionList->pPredictions[wIndex].dwHashValue != dwHashValue); } } #endif return pCache->wIndex; } else if (pCache->wIndex < pPredictionList->wUsedPredictionCount && pPredictionList->pPredictions[pCache->wIndex].dwHashValue == dwHashValue) { pCache->dwIndexHash = dwHashValue; pCache->dwIndexState = pCache->dwCurrentState; return pCache->wIndex; } /* look for the item */ { ET9U16 wIndex; for (wIndex = 0; wIndex < pPredictionList->wUsedPredictionCount; ++wIndex) { if (pPredictionList->pPredictions[wIndex].dwHashValue == dwHashValue) { /* found */ pCache->wIndex = wIndex; pCache->dwIndexHash = dwHashValue; pCache->dwIndexState = pCache->dwCurrentState; return wIndex; } } } /* not found */ pCache->wIndex = __DLM_Index_NULL; pCache->dwIndexHash = dwHashValue; pCache->dwIndexState = pCache->dwCurrentState; return __DLM_Index_NULL; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __FreePrediction(ET9AWLingInfo * const pLingInfo, const ET9UINT nContextIndex, const ET9U16 wPredictionIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; __DLM_PredictionsList * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; __DLM_PredictionList * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; ET9Assert(nContextIndex < __DLM_ContextLengths); ET9Assert(wPredictionIndex < pPredictionList->wUsedPredictionCount); WLOG2(fprintf(pLogFile2, "__FreePrediction, nContextIndex %u, wPredictionIndex %u\n", nContextIndex, wPredictionIndex);) /* release list chunks (zero indicates word index rather than chunk index) */ if (pPrediction->bPredictionCount) { __FreeListChunkChain(pPredictionList, &pPrediction->wFirstChunkIndex); } #ifdef _ET9_DLM_REVERSE_HASH { __ReverseHash_RemoveItem(nContextIndex, pPrediction->dwHashValue); } #endif /* remove by moving up predictions after this one in the list */ _ET9ByteMove((ET9U8*)&pPredictionList->pPredictions[wPredictionIndex], (ET9U8*)&pPredictionList->pPredictions[wPredictionIndex + 1], (pPredictionList->wUsedPredictionCount - wPredictionIndex - 1) * sizeof(__DLM_PredictionList)); --pPredictionList->wUsedPredictionCount; { __DLM_PredictionList * const pRemovedPrediction = &pPredictionList->pPredictions[pPredictionList->wUsedPredictionCount]; pRemovedPrediction->dwHashValue = 0; pRemovedPrediction->bPredictionCount = 0; pRemovedPrediction->wFirstChunkIndex = __DLM_Index_Unused; } /* notify change */ __PredictionOrderChanged(pLingInfo, nContextIndex); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U32 ET9LOCALCALL __CalculateWordHashValue(ET9SYMB const * const psBuf, const ET9U16 wBufLen) { ET9U32 dwHashValue = 0; ET9UINT nCount; ET9SYMB const * psSymb; psSymb = psBuf; for (nCount = wBufLen; nCount; --nCount, ++psSymb) { dwHashValue = _ET9SymToLower(*psSymb, 0) + (65599 * dwHashValue); } return dwHashValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static ET9U16 ET9LOCALCALL __FindWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen, const ET9U16 wLanguageId, const ET9BOOL bCaseSensitive) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U32 dwBufHash = __CalculateWordHashValue(psBuf, wBufLen); ET9SYMB psBufOther[ET9MAXWORDSIZE]; ET9UINT nIndex; WLOG2(fprintf(pLogFile2, "__FindWord, wLanguageId %02x, bCaseSensitive %u, string ", wLanguageId, bCaseSensitive);) WLOG2String(pLogFile2, "", psBuf, wBufLen, 0, 1); WLOG2(fprintf(pLogFile2, "\n");) for (nIndex = 0; nIndex < wBufLen && nIndex < ET9MAXWORDSIZE; ++nIndex) { psBufOther[nIndex] = bCaseSensitive ? psBuf[nIndex] : _ET9SymToOther(psBuf[nIndex], wLanguageId); } for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); if (pWord->dwStringHash != dwBufHash) { continue; } if (pWord->bWordLength != wBufLen) { continue; } if (pWord->wLanguageId != wLanguageId) { continue; } { ET9UINT nCount; ET9SYMB const * psSymb1; ET9SYMB const * psSymb2; ET9SYMB const * psSymb3; psSymb1 = psBuf; psSymb2 = psBufOther; psSymb3 = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = wBufLen; nCount; --nCount, ++psSymb1, ++psSymb2, ++psSymb3) { if (*psSymb1 != *psSymb3 && *psSymb2 != *psSymb3) { break; } } if (!nCount) { WLOG2(fprintf(pLogFile2, " found @ %u, wAccessIndex %u, sbQuality %d, wUseCount %u\n", nIndex, pWord->wAccessIndex, pWord->sbQuality, pWord->wUseCount);) return pWord->wAccessIndex; } } } WLOG2(fprintf(pLogFile2, " not found\n");) return __DLM_Index_NULL; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __MoveWord(ET9AWLingInfo * const pLingInfo, const ET9UINT nWordIndex, const ET9UINT nMoveToItemIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9UINT nWordItemIndex = pDLM->sWordStore.pwWordAccess[nWordIndex]; ET9Assert(nMoveToItemIndex < pDLM->sWordStore.wWordCount); if (nMoveToItemIndex < nWordItemIndex) { ET9UINT nMoveCount; /* move */ nMoveCount = nWordItemIndex - nMoveToItemIndex; { const __DLM_WordItem sTmp = pDLM->sWordStore.pWordItems[nWordItemIndex]; _ET9ByteMove((ET9U8*)&pDLM->sWordStore.pWordItems[nMoveToItemIndex + 1], (ET9U8*)&pDLM->sWordStore.pWordItems[nMoveToItemIndex], (ET9U32)(nMoveCount * sizeof(__DLM_WordItem))); pDLM->sWordStore.pWordItems[nMoveToItemIndex] = sTmp; } /* update access indexes */ { ET9UINT nIndex; for (nIndex = 0; nIndex < __DLM_WordCount; ++nIndex) { if (pDLM->sWordStore.pwWordAccess[nIndex] == __DLM_Index_Unused) { continue; } if (pDLM->sWordStore.pwWordAccess[nIndex] >= nMoveToItemIndex && pDLM->sWordStore.pwWordAccess[nIndex] < nMoveToItemIndex + nMoveCount) { ++pDLM->sWordStore.pwWordAccess[nIndex]; } } pDLM->sWordStore.pwWordAccess[nWordIndex] = (ET9U16)nMoveToItemIndex; } } else if (nMoveToItemIndex > nWordItemIndex) { ET9UINT nMoveCount; /* move */ nMoveCount = nMoveToItemIndex - nWordItemIndex; { const __DLM_WordItem sTmp = pDLM->sWordStore.pWordItems[nWordItemIndex]; _ET9ByteMove((ET9U8*)&pDLM->sWordStore.pWordItems[nWordItemIndex], (ET9U8*)&pDLM->sWordStore.pWordItems[nWordItemIndex + 1], (ET9U32)(nMoveCount * sizeof(__DLM_WordItem))); pDLM->sWordStore.pWordItems[nMoveToItemIndex] = sTmp; } /* update access indexes */ { ET9UINT nIndex; for (nIndex = 0; nIndex < __DLM_WordCount; ++nIndex) { if (pDLM->sWordStore.pwWordAccess[nIndex] == __DLM_Index_Unused) { continue; } if (pDLM->sWordStore.pwWordAccess[nIndex] > nWordItemIndex && pDLM->sWordStore.pwWordAccess[nIndex] <= nMoveToItemIndex) { --pDLM->sWordStore.pwWordAccess[nIndex]; } } pDLM->sWordStore.pwWordAccess[nWordIndex] = (ET9U16)nMoveToItemIndex; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __MarkWordUnused(ET9AWLingInfo * const pLingInfo, const ET9UINT nWordIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; WLOG2(fprintf(pLogFile2, "__MarkWordUnused, nWordIndex %u\n", nWordIndex);) /* move last if not last */ { const ET9U16 nWordItemIndex = pDLM->sWordStore.pwWordAccess[nWordIndex]; if (nWordItemIndex + 1 < pDLM->sWordStore.wWordCount) { WLOG2(fprintf(pLogFile2, "__MarkWordUnused, moving last (%u -> %u)\n", nWordItemIndex, (pDLM->sWordStore.wWordCount - 1));) __MoveWord(pLingInfo, nWordIndex, pDLM->sWordStore.wWordCount - 1); } } /* mark */ { const ET9U16 nWordItemIndex = pDLM->sWordStore.pwWordAccess[nWordIndex]; __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[nWordItemIndex]; WLOG2(fprintf(pLogFile2, "__MarkWordUnused, marking %u\n", nWordItemIndex);) ET9Assert(nWordItemIndex + 1 == pDLM->sWordStore.wWordCount); ET9Assert(pWord->wCharIndex < __DLM_PoolCharCount); ET9Assert(pWord->wAccessIndex < __DLM_WordCount); ET9Assert(pDLM->sWordStore.pwWordAccess[pWord->wAccessIndex] == nWordItemIndex); /* characters */ { ET9UINT nCharIndex; for (nCharIndex = 0; nCharIndex < pWord->bWordLength; ++nCharIndex) { pDLM->sWordStore.psPoolChars[pWord->wCharIndex + nCharIndex] = 0; } } /* word */ pDLM->sWordStore.pwWordAccess[pWord->wAccessIndex] = __DLM_Index_Unused; pWord->wCharIndex = __DLM_Index_Unused; pWord->wAccessIndex = __DLM_Index_Unused; /* count */ --pDLM->sWordStore.wWordCount; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __UpdatePredictionsForMissingWords(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9UINT nContextIndex; for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths; ++nContextIndex) { __DLM_PredictionsList * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; ET9U16 wPredictionIndex; for (wPredictionIndex = 0; wPredictionIndex < pPredictionList->wUsedPredictionCount; ++wPredictionIndex) { __DLM_PredictionList * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; ET9BOOL bRemovePrediction = 0; if (!pPrediction->bPredictionCount) { if (pDLM->sWordStore.pwWordAccess[pPrediction->wFirstChunkIndex] == __DLM_Index_Unused) { pPrediction->wFirstChunkIndex = __DLM_Index_NULL; bRemovePrediction = 1; } } else { ET9U8 bListItemCount = 0; __DLM_PredictionListItem *pListItems[__DLM_MaxListChunks * __DLM_ChunkWordCount]; ET9U8 bWordIndex; __DLM_PredictionListChunk * pCurrChunk; /* loop list items to identify removed ones */ pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { ET9Assert(pCurrChunk->wNextChunkIndex < __DLM_ListChunkCount); pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; } if (pDLM->sWordStore.pwWordAccess[pCurrChunk->pItems[bChunkWordIndex].wWordIndex] != __DLM_Index_Unused) { pListItems[bListItemCount++] = &pCurrChunk->pItems[bChunkWordIndex]; } } if (bListItemCount < pPrediction->bPredictionCount) { /* write back active items */ pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < bListItemCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { ET9Assert(pCurrChunk->wNextChunkIndex < __DLM_ListChunkCount); pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; } pCurrChunk->pItems[bChunkWordIndex] = *pListItems[bWordIndex]; } /* free empty list chunks (original zero indicates direct index) */ if (pPrediction->bPredictionCount) { if (bListItemCount) { __FreeListChunkChain(pPredictionList, &pCurrChunk->wNextChunkIndex); } else { __FreeListChunkChain(pPredictionList, &pPrediction->wFirstChunkIndex); } } /* update item count */ pPrediction->bPredictionCount = bListItemCount; /* remove prediction? */ if (!pPrediction->bPredictionCount) { bRemovePrediction = 1; } /* validate */ __ValidatePrediction(pPredictionList, pPrediction); } } if (bRemovePrediction) { __FreePrediction(pLingInfo, nContextIndex, wPredictionIndex); --wPredictionIndex; /* repeat index */ } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __CompressCharacterPool(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9UINT nIndex; WLOG2(fprintf(pLogFile2, "__CompressCharacterPool\n");) for (nIndex = 0; nIndex < (ET9UINT)(__DLM_PoolCharCount - pDLM->sWordStore.wFreeCharCount); ++nIndex) { if (!pDLM->sWordStore.psPoolChars[nIndex]) { ET9UINT nLook; for (nLook = nIndex + 1; nLook < (ET9UINT)(__DLM_PoolCharCount - pDLM->sWordStore.wFreeCharCount); ++nLook) { if (pDLM->sWordStore.psPoolChars[nLook]) { /* move the free space to the end of the pool */ const ET9UINT nFoundFree = nLook - nIndex; const ET9UINT nUsedCharCount = __DLM_PoolCharCount - pDLM->sWordStore.wFreeCharCount; WLOG2(fprintf(pLogFile2, " found %u chars at %u\n", nFoundFree, nIndex);) _ET9SymMove(&pDLM->sWordStore.psPoolChars[nIndex], &pDLM->sWordStore.psPoolChars[nLook], nUsedCharCount - nLook); pDLM->sWordStore.wFreeCharCount = (ET9U16)(pDLM->sWordStore.wFreeCharCount + nFoundFree); /* clear freed space */ { const ET9UINT nStartPos = __DLM_PoolCharCount - pDLM->sWordStore.wFreeCharCount; ET9UINT nCharIndex; for (nCharIndex = nStartPos; nCharIndex < nStartPos + nFoundFree; ++nCharIndex) { pDLM->sWordStore.psPoolChars[nCharIndex] = 0; } } /* adjust word to char indexes */ { ET9UINT nWordIndex; for (nWordIndex = 0; nWordIndex < pDLM->sWordStore.wWordCount; ++nWordIndex) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[nWordIndex]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); if (pWord->wCharIndex < nIndex) { continue; } pWord->wCharIndex = (ET9U16)(pWord->wCharIndex - nFoundFree); } } break; } } } } #ifdef ET9_DEBUG { ET9U16 pwCounts[__DLM_PoolCharCount] = { 0 }; ET9UINT nWordIndex; for (nWordIndex = 0; nWordIndex < pDLM->sWordStore.wWordCount; ++nWordIndex) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[nWordIndex]; ET9Assert(pWord->wCharIndex < __DLM_PoolCharCount); ++pwCounts[pWord->wCharIndex]; ET9Assert(pwCounts[pWord->wCharIndex] == 1); } } #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __FreeWords(ET9AWLingInfo * const pLingInfo, const ET9UINT nFreeCount) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9Assert(nFreeCount <= pDLM->sWordStore.wWordCount); WLOG2(fprintf(pLogFile2, "__FreeWords, nFreeCount %u\n", nFreeCount);) /* mark words and characters unused */ { ET9UINT nWordItemIndex; ET9UINT nWordFreeCount; nWordItemIndex = pDLM->sWordStore.wWordCount - 1; for (nWordFreeCount = 0; nWordFreeCount < nFreeCount; ++nWordFreeCount, --nWordItemIndex) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[nWordItemIndex]; ET9Assert(pWord->wCharIndex < __DLM_PoolCharCount); ET9Assert(pWord->wAccessIndex < __DLM_WordCount); ET9Assert(pDLM->sWordStore.pwWordAccess[pWord->wAccessIndex] == nWordItemIndex); __MarkWordUnused(pLingInfo, pWord->wAccessIndex); } } /* update all predictions that used the words */ __UpdatePredictionsForMissingWords(pLingInfo); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __FreeCharacters(ET9AWLingInfo * const pLingInfo, const ET9UINT nFreeCount) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9UINT nTargetFreeCharCount = pDLM->sWordStore.wFreeCharCount + nFreeCount; WLOG2(fprintf(pLogFile2, "__FreeCharacters, nFreeCount %u\n", nFreeCount);) for (;;) { /* compress free space */ __CompressCharacterPool(pLingInfo); if (pDLM->sWordStore.wFreeCharCount >= nTargetFreeCharCount) { break; } /* remove words to free up characters */ { const ET9U16 wOldWordCount = pDLM->sWordStore.wWordCount; ET9UINT wWordItemIndex; ET9UINT nWordFreeCount; ET9UINT nWordFreeChars; nWordFreeChars = 0; nWordFreeCount = 0; for (wWordItemIndex = pDLM->sWordStore.wWordCount - 1; wWordItemIndex > 0 && nWordFreeChars < nFreeCount; --wWordItemIndex) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[wWordItemIndex]; ++nWordFreeCount; nWordFreeChars += pWord->bWordLength; } if (nWordFreeCount < 10) { nWordFreeCount = 10; } else if (nWordFreeCount > 100) { nWordFreeCount = 100; } __FreeWords(pLingInfo, nWordFreeCount); if (pDLM->sWordStore.wWordCount >= wOldWordCount) { ET9Assert(0); break; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __FreePredictions(ET9AWLingInfo * const pLingInfo, const ET9UINT nContextIndex, const ET9UINT nFreeCount) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; __DLM_PredictionsList * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; ET9Assert(nFreeCount <= pPredictionList->wUsedPredictionCount); WLOG2(fprintf(pLogFile2, "__FreePredictions, nContextIndex %u, nFreeCount %u\n", nContextIndex, nFreeCount);) /* loop tail predictions */ { ET9U16 wPredictionIndex; ET9UINT nPredictionFreeCount; wPredictionIndex = pPredictionList->wUsedPredictionCount - 1; for (nPredictionFreeCount = 0; nPredictionFreeCount < nFreeCount && wPredictionIndex; ++nPredictionFreeCount, --wPredictionIndex) { ET9Assert(pPredictionList->pPredictions[wPredictionIndex].wFirstChunkIndex != __DLM_Index_Unused); __FreePrediction(pLingInfo, nContextIndex, wPredictionIndex); } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __FreeListChunks(ET9AWLingInfo * const pLingInfo, const ET9UINT nContextIndex, const ET9UINT nFreeCount) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; __DLM_PredictionsList * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; if (!pPredictionList->wUsedPredictionCount) { ET9Assert(0); return; } WLOG2(fprintf(pLogFile2, "__FreeListChunks, nContextIndex %u, nFreeCount %u\n", nContextIndex, nFreeCount);) /* loop tail predictions and remove those with list chunks */ { ET9U16 wPredictionIndex; ET9UINT nPredictionFreeCount; nPredictionFreeCount = 0; for (wPredictionIndex = pPredictionList->wUsedPredictionCount - 1; wPredictionIndex && nPredictionFreeCount < nFreeCount; --wPredictionIndex) { __DLM_PredictionList * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; ET9Assert(pPredictionList->pPredictions[wPredictionIndex].wFirstChunkIndex != __DLM_Index_Unused); if (!pPrediction->bPredictionCount) { continue; } nPredictionFreeCount += (pPrediction->bPredictionCount / __DLM_ChunkWordCount) + ((pPrediction->bPredictionCount % __DLM_ChunkWordCount) ? 1 : 0); __FreePrediction(pLingInfo, nContextIndex, wPredictionIndex); } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __AssureUpdateSpace(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9UINT nAssureCount = 2; const ET9UINT nTargetCount = 10; ET9UINT nContextIndex; if (pDLM->sWordStore.wWordCount + nAssureCount > __DLM_WordCount) { __FreeWords(pLingInfo, nTargetCount); ET9Assert(pDLM->sWordStore.wWordCount + nAssureCount <= __DLM_WordCount); } if (pDLM->sWordStore.wFreeCharCount < nAssureCount * ET9MAXWORDSIZE) { __FreeCharacters(pLingInfo, (nAssureCount * ET9MAXWORDSIZE) - pDLM->sWordStore.wFreeCharCount); ET9Assert(pDLM->sWordStore.wFreeCharCount >= nAssureCount * ET9MAXWORDSIZE); } for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths; ++nContextIndex) { if (pDLM->pPredictionsLists[nContextIndex].wUsedPredictionCount + nAssureCount > __DLM_ContextCount) { __FreePredictions(pLingInfo, nContextIndex, nTargetCount); ET9Assert(pDLM->pPredictionsLists[nContextIndex].wUsedPredictionCount + nAssureCount <= __DLM_ContextCount); } if (pDLM->pPredictionsLists[nContextIndex].wUsedListChunkCount + nAssureCount > __DLM_ListChunkCount) { __FreeListChunks(pLingInfo, nContextIndex, nTargetCount); ET9Assert(pDLM->pPredictionsLists[nContextIndex].wUsedListChunkCount + nAssureCount <= __DLM_ListChunkCount); } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __AgeUseCounts(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; WLOG2(fprintf(pLogFile2, "__AgeUseCounts\n");) /* unigram */ { ET9U16 wWordIndex; for (wWordIndex = 0; wWordIndex < pDLM->sWordStore.wWordCount; ++wWordIndex) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[wWordIndex]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); pWord->wUseCount /= 2; if (!pWord->wUseCount) { pWord->wUseCount = 1; } } } /* predictions */ { ET9UINT nContextIndex; for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths; ++nContextIndex) { ET9UINT nChunkIndex; for (nChunkIndex = 0; nChunkIndex < __DLM_ListChunkCount; ++nChunkIndex) { __DLM_PredictionListChunk * const pChunk = &pDLM->pPredictionsLists[nContextIndex].pPredictionListChunks[nChunkIndex]; if (pChunk->wNextChunkIndex == __DLM_Index_Unused) { continue; } { ET9UINT nItemIndex; for (nItemIndex = 0; nItemIndex < __DLM_ChunkWordCount; ++nItemIndex) { pChunk->pItems[nItemIndex].wUseCount /= 2; if (!pChunk->pItems[nItemIndex].wUseCount) { pChunk->pItems[nItemIndex].wUseCount = 1; } } } } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __UpdateWordRecency(ET9AWLingInfo * const pLingInfo, const ET9U16 wWordIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U16 wWordItemIndex = pDLM->sWordStore.pwWordAccess[wWordIndex]; const ET9S8 sbWordQuality = pDLM->sWordStore.pWordItems[wWordItemIndex].sbQuality; /* move to the top by default */ ET9U16 wMoveTo = wWordItemIndex; ET9Assert(wWordIndex != __DLM_Index_Unused); ET9Assert(wWordItemIndex != __DLM_Index_Unused); if (sbWordQuality >= __DLM_UserAddQuality || sbWordQuality <= __DLM_UserKillQuality) { wMoveTo = 0; } else { ET9U16 wIndex; ET9U16 wKillCount = 0; ET9U16 wStaticCount = 0; for (wIndex = 0; wIndex < pDLM->sWordStore.wWordCount; ++wIndex) { if (pDLM->sWordStore.pWordItems[wIndex].sbQuality >= __DLM_UserAddQuality) { ++wStaticCount; } else if (pDLM->sWordStore.pWordItems[wIndex].sbQuality <= __DLM_UserKillQuality) { ++wKillCount; ++wStaticCount; } else { break; } } pDLM->sWordStore.bHasKillWords = wKillCount ? 1 : 0; wMoveTo = wStaticCount; { const ET9U16 wMinStay = __DLM_WordCount / 10; if (wMoveTo + wMinStay > __DLM_WordCount) { wMoveTo = __DLM_WordCount - wMinStay; } } } __MoveWord(pLingInfo, wWordIndex, wMoveTo); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static ET9U16 ET9LOCALCALL __AddNewWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psWord, const ET9U16 wWordLen, const ET9U16 wLanguageId, const ET9S8 sbQuality) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9U16 wWordIndex; WLOG2(fprintf(pLogFile2, "__AddNewWord, wLanguageId %02x, sbQuality %d\n", wLanguageId, sbQuality);) for (wWordIndex = 0; wWordIndex < __DLM_WordCount; ++wWordIndex) { if (pDLM->sWordStore.pwWordAccess[wWordIndex] != __DLM_Index_Unused) { continue; } { const ET9U16 wWordItemIndex = pDLM->sWordStore.wWordCount++; __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[wWordItemIndex]; ET9Assert(pWord->wCharIndex == __DLM_Index_Unused); ET9Assert(__DLM_PoolCharCount >= pDLM->sWordStore.wFreeCharCount); pWord->wUseCount = 1; pWord->wAccessIndex = wWordIndex; pWord->wLanguageId = wLanguageId; pWord->bWordLength = (ET9U8)wWordLen; pWord->sbQuality = sbQuality; pWord->dwStringHash = __CalculateWordHashValue(psWord, wWordLen); pWord->wCharIndex = (ET9U16)(__DLM_PoolCharCount - pDLM->sWordStore.wFreeCharCount); if (pWord->sbQuality <= __DLM_UserKillQuality) { pDLM->sWordStore.bHasKillWords = 1; } pDLM->sWordStore.pwWordAccess[wWordIndex] = wWordItemIndex; pDLM->sWordStore.wFreeCharCount = (ET9U16)(pDLM->sWordStore.wFreeCharCount - wWordLen); { ET9UINT nCount; ET9SYMB const * psSymbSrc; ET9SYMB * psSymbDst; psSymbSrc = psWord; psSymbDst = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = wWordLen; nCount; --nCount, ++psSymbSrc, ++psSymbDst) { if (nCount == wWordLen) { pWord->bIsCapitalized = _ET9SymIsUpper(*psSymbSrc, wLanguageId); } else if (pWord->bIsCapitalized) { pWord->bIsCapitalized = !_ET9SymIsUpper(*psSymbSrc, wLanguageId); } *psSymbDst = *psSymbSrc; } } __UpdateWordRecency(pLingInfo, wWordIndex); break; } } if (wWordIndex >= __DLM_WordCount) { ET9Assert(0); return __DLM_Index_NULL; } return wWordIndex; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * */ static void ET9LOCALCALL __CollectWordProperties(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen, ET9U32 * const pdwLdbNum, ET9S8 * const psbQuality, ET9U16 * const pwWordIndex, ET9BOOL * const pbFoundUserAdd) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U32 dwPrimaryLdbNum = pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; const ET9U32 dwSecondaryLdbNum = pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingCmnInfo->dwFirstLdbNum : pLingCmnInfo->dwSecondLdbNum; ET9U32 dwOtherLdbNum = 0; ET9S8 sbBestQualityPrimary = __DLM_UserMinQuality; ET9S8 sbBestQualitySecondary = __DLM_UserMinQuality; ET9S8 sbBestQualityOther = __DLM_UserMinQuality; ET9U16 wWordIndexPrimary = __DLM_Index_NULL; ET9U16 wWordIndexSecondary = __DLM_Index_NULL; ET9U16 wWordIndexOther = __DLM_Index_NULL; /* init */ *pbFoundUserAdd = 0; *psbQuality = __DLM_UserMinQuality; *pwWordIndex = __DLM_Index_NULL; /* applicable? */ if (*pdwLdbNum == __DLM_AnyLanguage) { return; } ET9Assert((*pdwLdbNum & ET9PLIDMASK) == ET9PLIDNone); if ((dwPrimaryLdbNum & ET9PLIDMASK) == ET9PLIDNone) { ET9Assert((dwSecondaryLdbNum & ET9PLIDMASK) == ET9PLIDNone); *pdwLdbNum = __DLM_AnyLanguage; *psbQuality = __DLM_UserMinQuality; *pwWordIndex = __DLM_Index_NULL; return; } /* DLM */ { _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U16 wPrimaryLanguageId = (ET9U16)dwPrimaryLdbNum; const ET9U16 wSecondaryLanguageId = (ET9U16)dwSecondaryLdbNum; const ET9U32 dwBufHash = __CalculateWordHashValue(psBuf, wBufLen); ET9UINT nIndex; for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); if (pWord->dwStringHash != dwBufHash) { continue; } if (pWord->sbQuality <= __DLM_UserKillQuality) { continue; } if (pWord->bWordLength != wBufLen) { continue; } { ET9UINT nCount; ET9SYMB const * psSymb; ET9SYMB const * psSymbWord; psSymb = psBuf; psSymbWord = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = wBufLen; nCount; --nCount, ++psSymb, ++psSymbWord) { if (*psSymb != *psSymbWord) { break; } } if (!nCount) { if (pWord->sbQuality >= __DLM_UserAddQuality) { *pbFoundUserAdd = 1; } else { if (sbBestQualityPrimary < pWord->sbQuality && pWord->wLanguageId == wPrimaryLanguageId) { sbBestQualityPrimary = pWord->sbQuality; wWordIndexPrimary = pWord->wAccessIndex; } if (sbBestQualitySecondary < pWord->sbQuality && pWord->wLanguageId == wSecondaryLanguageId) { sbBestQualitySecondary = pWord->sbQuality; wWordIndexSecondary = pWord->wAccessIndex; } if (sbBestQualityOther < pWord->sbQuality && pWord->wLanguageId != wPrimaryLanguageId && pWord->wLanguageId != wSecondaryLanguageId) { sbBestQualityOther = pWord->sbQuality; wWordIndexOther = pWord->wAccessIndex; dwOtherLdbNum = pWord->wLanguageId; } } } } } WLOG2(fprintf(pLogFile2, "__CollectWordProperties, sbBestQualityPrimary %d, sbBestQualitySecondary %d\n", sbBestQualityPrimary, sbBestQualitySecondary);) /* final decision if static quality and active language */ if (sbBestQualityPrimary >= sbBestQualitySecondary && sbBestQualityPrimary > __DLM_NormalQuality) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found in DLM, primary (1)\n");) *pdwLdbNum = dwPrimaryLdbNum; *psbQuality = sbBestQualityPrimary; *pwWordIndex = wWordIndexPrimary; return; } if (sbBestQualitySecondary > __DLM_NormalQuality) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found in DLM, secondary (1)\n");) *pdwLdbNum = dwSecondaryLdbNum; *psbQuality = sbBestQualitySecondary; *pwWordIndex = wWordIndexSecondary; return; } } /* LDB(s) */ { ET9U8 bExact; ET9U8 bLowercase; ET9U32 dwWordIndex; if (_ET9AWLdbFindEntry(pLingInfo, dwPrimaryLdbNum, ET9MGD_FULL_WORD, psBuf, wBufLen, &dwWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found word in primary LDB %08x, bExact %u\n", dwPrimaryLdbNum, bExact);) *pdwLdbNum = dwPrimaryLdbNum; *psbQuality = bExact ? __DLM_StaticXQuality : __DLM_StaticCQuality; *pwWordIndex = wWordIndexPrimary; return; } if (_ET9AWLdbFindEntry(pLingInfo, dwSecondaryLdbNum, ET9MGD_FULL_WORD, psBuf, wBufLen, &dwWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found word in secondary LDB %08x, bExact %u\n", dwSecondaryLdbNum, bExact);) *pdwLdbNum = dwSecondaryLdbNum; *psbQuality = bExact ? __DLM_StaticXQuality : __DLM_StaticCQuality; *pwWordIndex = wWordIndexSecondary; return; } } /* re-evaluate DLM findings */ if (sbBestQualityPrimary >= sbBestQualitySecondary && sbBestQualityPrimary > __DLM_UserMinQuality) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found in DLM, primary (2)\n");) *pdwLdbNum = dwPrimaryLdbNum; *psbQuality = sbBestQualityPrimary; *pwWordIndex = wWordIndexPrimary; return; } if (sbBestQualitySecondary > __DLM_UserMinQuality) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found in DLM, secondary (2)\n");) *pdwLdbNum = dwSecondaryLdbNum; *psbQuality = sbBestQualitySecondary; *pwWordIndex = wWordIndexSecondary; return; } if (sbBestQualityOther > __DLM_UserMinQuality && sbBestQualityOther < __DLM_UserAddQuality) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found in DLM, other (2)\n");) *pdwLdbNum = dwOtherLdbNum; *psbQuality = sbBestQualityOther; *pwWordIndex = wWordIndexOther; return; } /* MDB */ { if (_ET9AWMdbFind(pLingInfo, psBuf, wBufLen, 0) == ET9STATUS_WORD_EXISTS) { WLOG2(fprintf(pLogFile2, "__CollectWordProperties, found word in MDB\n");) *pdwLdbNum = dwPrimaryLdbNum; *psbQuality = __DLM_StaticCQuality; *pwWordIndex = wWordIndexPrimary; return; } } /* not found */ WLOG2(fprintf(pLogFile2, "__CollectWordProperties, not found, (bFoundUserAdd %c)\n", (*pbFoundUserAdd ? 'Y' : 'N'));) *pdwLdbNum = dwPrimaryLdbNum; *psbQuality = __DLM_UserMinQuality; *pwWordIndex = __DLM_Index_NULL; ET9Assert((*pdwLdbNum & ET9PLIDMASK) != ET9PLIDNone); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static ET9S8 ET9LOCALCALL __GetQualityValue(ET9AWLingInfo * const pLingInfo, const ET9BOOL bIsAnyLanguageAdd, const ET9BOOL bFoundUserAdd, const _ET9AW_DLM_WordQuality eQuality, const ET9S8 sbBestQuality) { if (bIsAnyLanguageAdd) { return __DLM_UserAddQuality; } if (sbBestQuality >= __DLM_NormalQuality) { return sbBestQuality; } if (bFoundUserAdd) { ET9Assert(sbBestQuality != __DLM_UserKillQuality); return __DLM_NormalQuality; } if (eQuality == _ET9AW_DLM_WordQuality_ScanLow && pLingInfo->pLingCmnInfo->Private.bQuarantineControlScanAction) { return __DLM_HiddenQuality; } if (eQuality == _ET9AW_DLM_WordQuality_UserNormal || eQuality == _ET9AW_DLM_WordQuality_UserHigh) { return __DLM_NormalQuality; } ET9Assert(eQuality < _ET9AW_DLM_WordQuality_UserNormal); { ET9S8 sbQuality; if (eQuality == _ET9AW_DLM_WordQuality_ScanLow || eQuality == _ET9AW_DLM_WordQuality_ScanHigh) { sbQuality = pLingInfo->pLingCmnInfo->Private.bQuarantineControlScanAction; } else { sbQuality = pLingInfo->pLingCmnInfo->Private.bQuarantineControlUserAction; } sbQuality = -sbQuality; ET9Assert(sbQuality <= __DLM_NormalQuality && sbQuality > __DLM_CutLowQuality); return sbQuality; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * */ static ET9U16 ET9LOCALCALL __UpdateWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen, const ET9BOOL bUpdateUse, const _ET9AW_DLM_WordQuality eQuality, const ET9BOOL bInhibitNewWord, ET9U32 * const pdwLdbNum) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U16 wHiddenRevealThresholdCount = 5; const ET9BOOL bIsAnyLanguageAdd = (*pdwLdbNum == __DLM_AnyLanguage) ? 1 : 0; const ET9U32 dwTmpLdbNum = pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum; ET9S8 sbBestQuality; ET9U16 wWordLen; ET9SYMB psWord[ET9MAXWORDSIZE]; ET9U16 wWordIndex; ET9BOOL bFoundUserAdd; ET9Assert(!*pdwLdbNum || *pdwLdbNum == __DLM_AnyLanguage); ET9Assert(eQuality > _ET9AW_DLM_WordQuality_First && eQuality < _ET9AW_DLM_WordQuality_Last); WLOG2(fprintf(pLogFile2, "__UpdateWord, pdwLdbNum %08x, bUpdateUse %u, eQuality %s, bInhibitNewWord %u, string ", *pdwLdbNum, bUpdateUse, __DLM_WORDQ_TOSTRING(eQuality), bInhibitNewWord);) WLOG2String(pLogFile2, "", psBuf, wBufLen, 0, 1); WLOG2(fprintf(pLogFile2, "\n");) ET9Assert(!bIsAnyLanguageAdd || eQuality == _ET9AW_DLM_WordQuality_UserHigh); if (!wBufLen || wBufLen > ET9MAXWORDSIZE) { ET9Assert(0); return __DLM_Index_NULL; } if (pDLM->sWordStore.wFreeCharCount < ET9MAXWORDSIZE) { ET9Assert(0); return __DLM_Index_NULL; } if (!bIsAnyLanguageAdd && !_ET9_LanguageSpecific_IsDLMLanguage(dwTmpLdbNum)) { WLOG2(fprintf(pLogFile2, "__UpdateWord, skip, not a DLM language (%08x)\n", dwTmpLdbNum);) return __DLM_Index_NULL; } /* clean up non user words */ if (bIsAnyLanguageAdd) { wWordLen = wBufLen; _ET9SymCopy(psWord, psBuf, wBufLen); } else { __CleanUpWord(pLingInfo, dwTmpLdbNum, psBuf, wBufLen, psWord, &wWordLen); } WLOG2(fprintf(pLogFile2, "__UpdateWord, clean string ");) WLOG2String(pLogFile2, "", psWord, wWordLen, 0, 1); WLOG2(fprintf(pLogFile2, "\n");) if (__ScreenWord(psWord, wWordLen, eQuality)) { WLOG2(fprintf(pLogFile2, "__UpdateWord, word screened out\n");) return __DLM_Index_NULL; } if (!wWordLen) { return __DLM_Index_NULL; } /* assure that there are no NULL chars in the string */ { ET9UINT nIndex; for (nIndex = 0; nIndex < wWordLen; ++nIndex) { if (!psWord[nIndex]) { return __DLM_Index_NULL; } } } /* check if the word is already known */ __CollectWordProperties(pLingInfo, psWord, wWordLen, pdwLdbNum, &sbBestQuality, &wWordIndex, &bFoundUserAdd); ET9Assert((*pdwLdbNum & ET9PLIDMASK) != ET9PLIDNone); /* potentialy undo kill word(s) */ if (pDLM->sWordStore.bHasKillWords && (bUpdateUse || bIsAnyLanguageAdd) && (eQuality > _ET9AW_DLM_WordQuality_ScanLow)) { ET9UINT nWordCount; ET9SYMB psBufOC[ET9MAXWORDSIZE]; const ET9U32 dwBufHash = __CalculateWordHashValue(psWord, wWordLen); /* create other case version */ { ET9UINT nCount; ET9SYMB const * psSymbSrc; ET9SYMB * psSymbOC; psSymbSrc = psWord; psSymbOC = psBufOC; for (nCount = wWordLen; nCount; --nCount, ++psSymbSrc, ++psSymbOC) { *psSymbOC = _ET9SymToOther(*psSymbSrc, pLingInfo->pLingCmnInfo->dwFirstLdbNum); } } /* find and update */ for (nWordCount = pDLM->sWordStore.wWordCount; nWordCount; --nWordCount) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[nWordCount - 1]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); if (pWord->sbQuality > __DLM_UserKillQuality) { continue; } if (pWord->dwStringHash != dwBufHash) { continue; } if (pWord->bWordLength != wWordLen) { continue; } { ET9UINT nCount; ET9SYMB const * psSymbWord; ET9SYMB const * psSymb; ET9SYMB const * psSymbOC; psSymb = psWord; psSymbOC = psBufOC; psSymbWord = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = wWordLen; nCount; --nCount, ++psSymbWord, ++psSymb, ++psSymbOC) { if (*psSymbWord != *psSymb && *psSymbWord != *psSymbOC) { break; } } if (!nCount) { WLOG2(fprintf(pLogFile2, "__UpdateWord, killed word reactivated\n");) pWord->sbQuality = __GetQualityValue(pLingInfo, bIsAnyLanguageAdd, bFoundUserAdd, eQuality, sbBestQuality); pWord->wLanguageId = (ET9U16)*pdwLdbNum; __UpdateWordRecency(pLingInfo, pWord->wAccessIndex); /* bump word count to assure that nothing was missed due to an update */ ++nWordCount; /* invalidate word index */ wWordIndex = __DLM_Index_NULL; } } } } /* check if the word is already known */ if (wWordIndex == __DLM_Index_NULL) { WLOG2(fprintf(pLogFile2, "__UpdateWord, finding word\n");) wWordIndex = __FindWord(pLingInfo, psWord, wWordLen, (ET9U16)*pdwLdbNum, 1); } if (wWordIndex != __DLM_Index_NULL) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[wWordIndex]]; ET9Assert(pWord->dwStringHash == __CalculateWordHashValue(psWord, wWordLen)); WLOG2(fprintf(pLogFile2, "__UpdateWord, old word\n");) if (bUpdateUse) { if (pWord->wUseCount == __DLM_MAX_USE_VALUE) { __AgeUseCounts(pLingInfo); } ET9Assert(pWord->wUseCount < __DLM_MAX_USE_VALUE); ++pWord->wUseCount; if (pWord->sbQuality <= __DLM_UserKillQuality) { ET9Assert(0); pWord->sbQuality = __DLM_InitialLowQuality; } else if (pWord->sbQuality == __DLM_HiddenQuality) { if (eQuality > _ET9AW_DLM_WordQuality_ScanHigh || pWord->wUseCount >= wHiddenRevealThresholdCount) { pWord->sbQuality = -(ET9S8)pLingInfo->pLingCmnInfo->Private.bQuarantineControlScanAction; } } else if (pWord->sbQuality < __DLM_NormalQuality) { if (eQuality == _ET9AW_DLM_WordQuality_ScanLow || eQuality == _ET9AW_DLM_WordQuality_ScanHigh) { if (!pLingInfo->pLingCmnInfo->Private.bQuarantineControlScanAction) { pWord->sbQuality = __DLM_NormalQuality; } } else { ++pWord->sbQuality; } } if (pWord->sbQuality > __DLM_UserKillQuality && pWord->sbQuality <= __DLM_NormalQuality) { if (sbBestQuality >= __DLM_NormalQuality && sbBestQuality < __DLM_UserAddQuality) { pWord->sbQuality = sbBestQuality; } } __UpdateWordRecency(pLingInfo, wWordIndex); } return wWordIndex; } /* if not found and not allowed to add new then abort here */ if (bInhibitNewWord) { WLOG2(fprintf(pLogFile2, "__UpdateWord, not adding new word, inhibited\n");) return __DLM_Index_NULL; } /* add as new word */ { WLOG2(fprintf(pLogFile2, "__UpdateWord, adding new word, dwLdbNum %08x, sbBestQuality %d, bFoundUserAdd %c\n", *pdwLdbNum, sbBestQuality, (bFoundUserAdd ? 'Y' : 'N'));) wWordIndex = __AddNewWord(pLingInfo, psWord, wWordLen, (ET9U16)*pdwLdbNum, __GetQualityValue(pLingInfo, bIsAnyLanguageAdd, bFoundUserAdd, eQuality, sbBestQuality)); } return wWordIndex; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U32 ET9LOCALCALL __CalculateContextHashValue(ET9AWLingInfo * const pLingInfo, const ET9UINT nContextLen) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U32 dwHashValue = 0; ET9UINT nContextIndex; if (nContextLen > ET9NLM_CONTEXT_COUNT) { ET9Assert(0); return 0; } for (nContextIndex = 0; nContextIndex < nContextLen; ++nContextIndex) { ET9SimpleWord const * const pContextWord = &pLingCmnInfo->Private.sCurrContext.pDLMContextWords[nContextIndex]; ET9UINT nCount; ET9SYMB const * psSymb; ET9Assert(pContextWord->wLen); dwHashValue = 0xB7 + (65599 * dwHashValue); psSymb = pContextWord->sString; for (nCount = pContextWord->wLen; nCount; --nCount, ++psSymb) { dwHashValue = *psSymb + (65599 * dwHashValue); } } #ifdef _ET9_DLM_REVERSE_HASH { __reverseHashItem const * const pItem = __ReverseHash_GetItem(nContextLen - 1, dwHashValue); if (!pItem) { __reverseHashItem sItem; ET9UINT nContextCount; sItem.dwHashValue = dwHashValue; sItem.wStringLen = 0; for (nContextCount = nContextLen; nContextCount; --nContextCount) { ET9SimpleWord * const pContextWord = &pLingCmnInfo->Private.sCurrContext.pDLMContextWords[nContextCount - 1]; if (!pContextWord->wLen) { sItem.psString[sItem.wStringLen++] = 0xB7; } else { _ET9SymCopy(&sItem.psString[sItem.wStringLen], pContextWord->sString, pContextWord->wLen); sItem.wStringLen = (ET9U16)(sItem.wStringLen + pContextWord->wLen); } if (nContextCount > 1) { sItem.psString[sItem.wStringLen++] = ' '; } } __ReverseHash_AddItem(nContextLen - 1, &sItem); } } #endif return dwHashValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static void ET9LOCALCALL __UpdatePredictionCount(ET9AWLingInfo * const pLingInfo, __DLM_PredictionsList * const pPredictionList, __DLM_PredictionList * const pPrediction, const ET9U16 wWordIndex, const ET9BOOL bUpdateUse) { ET9U8 bWordIndex; ET9U8 bChunkCount; __DLM_PredictionListChunk * pCurrChunk; __DLM_PredictionListChunk * pPrevChunk; ET9Assert(wWordIndex != __DLM_Index_NULL); ET9Assert(pPrediction->wFirstChunkIndex < __DLM_ListChunkCount); WLOG2(fprintf(pLogFile2, "__UpdatePredictionCount, wWordIndex %u, bUpdateUse %u\n", wWordIndex, bUpdateUse);) bChunkCount = 1; pPrevChunk = NULL; pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { ET9Assert(bChunkCount < __DLM_MaxListChunks); ET9Assert(pCurrChunk->wNextChunkIndex < __DLM_ListChunkCount); ++bChunkCount; pPrevChunk = pCurrChunk; pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; } if (pCurrChunk->pItems[bChunkWordIndex].wWordIndex == wWordIndex) { /* word found */ WLOG2(fprintf(pLogFile2, " word found, bChunkCount %u, bChunkWordIndex %u\n", bChunkCount, bChunkWordIndex);) if (!bUpdateUse) { WLOG2(fprintf(pLogFile2, " not updating use or recency\n");) } else { if (pCurrChunk->pItems[bChunkWordIndex].wUseCount == __DLM_MAX_USE_VALUE) { __AgeUseCounts(pLingInfo); } ET9Assert(pCurrChunk->pItems[bChunkWordIndex].wUseCount < __DLM_MAX_USE_VALUE); ++pCurrChunk->pItems[bChunkWordIndex].wUseCount; if (bChunkWordIndex) { /* not first in chunk, move to first in chunk */ const __DLM_PredictionListItem sTmp = pCurrChunk->pItems[bChunkWordIndex]; _ET9ByteMove((ET9U8*)&pCurrChunk->pItems[1], (ET9U8*)&pCurrChunk->pItems[0], (ET9U32)(bChunkWordIndex * sizeof(__DLM_PredictionListItem))); pCurrChunk->pItems[0] = sTmp; WLOG2(fprintf(pLogFile2, " not first in chunk, moved to first in chunk\n");) } else if (pPrevChunk) { /* first in chunk, move to first in previous chunk */ const __DLM_PredictionListItem sTmp = pCurrChunk->pItems[0]; pCurrChunk->pItems[0] = pPrevChunk->pItems[__DLM_ChunkWordCount - 1]; _ET9ByteMove((ET9U8*)&pPrevChunk->pItems[1], (ET9U8*)&pPrevChunk->pItems[0], (ET9U32)((__DLM_ChunkWordCount - 1) * sizeof(__DLM_PredictionListItem))); pPrevChunk->pItems[0] = sTmp; WLOG2(fprintf(pLogFile2, " first in chunk, moved to first in previous chunk\n");) } } /* validate */ __ValidatePrediction(pPredictionList, pPrediction); return; } } /* add word */ if (pPrediction->bPredictionCount % __DLM_ChunkWordCount) { /* use empty spot in last chunk */ const ET9U8 bItemIndex = pPrediction->bPredictionCount++; __DLM_PredictionListItem * const pItem = &pCurrChunk->pItems[bItemIndex % __DLM_ChunkWordCount]; ET9Assert(bItemIndex < __DLM_MaxListChunks * __DLM_ChunkWordCount); pItem->wUseCount = 1; pItem->wWordIndex = wWordIndex; WLOG2(fprintf(pLogFile2, " adding word, use empty spot in last chunk\n");) } else if (bChunkCount >= __DLM_MaxListChunks) { /* replace in last chunk (first in last chunk) */ __DLM_PredictionListItem * const pItem = &pCurrChunk->pItems[0]; _ET9ByteMove((ET9U8*)&pCurrChunk->pItems[1], (ET9U8*)&pCurrChunk->pItems[0], (ET9U32)((__DLM_ChunkWordCount - 1) * sizeof(__DLM_PredictionListItem))); pItem->wUseCount = 1; pItem->wWordIndex = wWordIndex; WLOG2(fprintf(pLogFile2, " adding word, replace in last chunk\n");) } else { /* add new chunk */ const ET9U16 wListChunk = __NewListChunk(pPredictionList); if (wListChunk == __DLM_Index_NULL) { WLOG2(fprintf(pLogFile2, "__UpdatePredictionCount, failed to get new list chunk\n");) ET9Assert(0); } else { __DLM_PredictionListChunk * pNewChunk = &pPredictionList->pPredictionListChunks[wListChunk]; __DLM_PredictionListItem * const pItem = &pNewChunk->pItems[0]; ++pPrediction->bPredictionCount; ET9Assert(pPrediction->bPredictionCount <= __DLM_MaxListChunks * __DLM_ChunkWordCount); pCurrChunk->wNextChunkIndex = wListChunk; pItem->wUseCount = 1; pItem->wWordIndex = wWordIndex; } WLOG2(fprintf(pLogFile2, " adding word, add new chunk\n");) } /* validate */ __ValidatePrediction(pPredictionList, pPrediction); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9U16 ET9LOCALCALL __UpdateContext(ET9AWLingInfo * const pLingInfo, const ET9UINT nContextIndex, const ET9U16 wWordIndex, const ET9BOOL bUpdateUse) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; __DLM_PredictionsList * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; const ET9U32 dwContextHash = __CalculateContextHashValue(pLingInfo, nContextIndex + 1); ET9BOOL bIsNewPrediction = 0; ET9U16 wPredictionIndex; ET9Assert(nContextIndex < __DLM_ContextLengths); WLOG2(fprintf(pLogFile2, "__UpdateContext, wWordIndex %5u, dwContextHash %10u, bUpdateUse %u\n", wWordIndex, dwContextHash, bUpdateUse);) wPredictionIndex = __FindPrediction(pLingInfo, nContextIndex, dwContextHash); if (wPredictionIndex < pPredictionList->wUsedPredictionCount) { /* entry found */ __DLM_PredictionList * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; ET9Assert(pPrediction->dwHashValue == dwContextHash); ET9Assert(pPrediction->wFirstChunkIndex != __DLM_Index_Unused); /* upgrade from single to chunk? */ if (!pPrediction->bPredictionCount && (bUpdateUse || pPrediction->wFirstChunkIndex != wWordIndex)) { const ET9U16 wListChunk = __NewListChunk(pPredictionList); if (wListChunk == __DLM_Index_NULL) { WLOG2(fprintf(pLogFile2, "__UpdateContext, failed to get new list chunk\n");) } else { __DLM_PredictionListChunk * const pChunk = &pPredictionList->pPredictionListChunks[wListChunk]; WLOG2(fprintf(pLogFile2, "__UpdateContext, converting from single to first list chunk\n");) pChunk->pItems[0].wUseCount = 1; pChunk->pItems[0].wWordIndex = pPrediction->wFirstChunkIndex; pPrediction->bPredictionCount = 1; pPrediction->wFirstChunkIndex = wListChunk; } } if (pPrediction->bPredictionCount) { __UpdatePredictionCount(pLingInfo, pPredictionList, pPrediction, wWordIndex, bUpdateUse); } } else { /* new prediction entry */ bIsNewPrediction = 1; if (pPredictionList->wUsedPredictionCount >= __DLM_ContextCount) { ET9Assert(0); return __DLM_Index_NULL; } wPredictionIndex = pPredictionList->wUsedPredictionCount++; { __DLM_PredictionList * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; ET9Assert(pPrediction->wFirstChunkIndex == __DLM_Index_Unused); pPrediction->dwHashValue = dwContextHash; pPrediction->bPredictionCount = 0; pPrediction->wFirstChunkIndex = wWordIndex; } /* notify change */ __PredictionOrderChanged(pLingInfo, nContextIndex); } /* move entry to the top of the list */ if (wPredictionIndex) { if (!bUpdateUse && !bIsNewPrediction) { WLOG2(fprintf(pLogFile2, "__UpdateContext, not moving prediction to the top\n");) } else { /* move to the top by default */ ET9U16 wMoveTo = 0; ET9UINT nMoveCount = wPredictionIndex - wMoveTo; /* move to just before the first single in the list, if there is enough singles (otherwise to the top) */ if (!pPredictionList->pPredictions[wPredictionIndex].bPredictionCount) { ET9U16 wLook; for (wLook = 0; wLook < pPredictionList->wUsedPredictionCount; ++wLook) { if (!pPredictionList->pPredictions[wLook].bPredictionCount) { break; } } { const ET9U16 wMinStay = __DLM_ContextCount / 20; if (wLook + wMinStay > __DLM_ContextCount) { wLook = __DLM_ContextCount - wMinStay; } wMoveTo = wLook; nMoveCount = wPredictionIndex - wMoveTo; } } /* move */ { const __DLM_PredictionList sTmp = pPredictionList->pPredictions[wPredictionIndex]; _ET9ByteMove((ET9U8*)&pPredictionList->pPredictions[wMoveTo + 1], (ET9U8*)&pPredictionList->pPredictions[wMoveTo], (ET9U32)(nMoveCount * sizeof(__DLM_PredictionList))); pPredictionList->pPredictions[wMoveTo] = sTmp; } /* update prediction index */ WLOG2(fprintf(pLogFile2, "__UpdateContext, moved prediction to %u\n", wMoveTo);) wPredictionIndex = wMoveTo; /* notify change */ __PredictionOrderChanged(pLingInfo, nContextIndex); } } /* done */ return wPredictionIndex; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static void ET9LOCALCALL __UpdateContexts(ET9AWLingInfo * const pLingInfo, const ET9U16 wWordIndex, const ET9U32 dwLdbNum, const ET9BOOL bUpdateUse, ET9SimpleWord const * const pLocalContext, const ET9UINT nLocalContextLen) { const ET9UINT nContextLen = __UpdateContextData(pLingInfo, dwLdbNum, pLocalContext, nLocalContextLen); ET9UINT nContextIndex; WLOG2(fprintf(pLogFile2, "__UpdateContexts, wWordIndex %5u, bUpdateUse %u, nLocalContextLen %u\n", wWordIndex, bUpdateUse, nLocalContextLen);) if (wWordIndex == __DLM_Index_NULL) { return; } ET9Assert(nContextLen >= 1); ET9Assert(wWordIndex < __DLM_WordCount); for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths && nContextIndex < nContextLen; ++nContextIndex) { __UpdateContext(pLingInfo, nContextIndex, wWordIndex, bUpdateUse); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __AddOneWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen, const ET9BOOL bUpdateUse, const _ET9AW_DLM_WordQuality eQuality, const ET9BOOL bInhibitNewWord, ET9SimpleWord const * const pLocalContext, const ET9UINT nLocalContextLen) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9Assert(eQuality > _ET9AW_DLM_WordQuality_First && eQuality < _ET9AW_DLM_WordQuality_Last); WLOG2(fprintf(pLogFile2, "__AddOneWord, bUpdateUse %c, eQuality %s, bInhibitNewWord %c, string ", (bUpdateUse ? 'Y' : 'N'), __DLM_WORDQ_TOSTRING(eQuality), (bInhibitNewWord ? 'Y' : 'N'));) WLOG2String(pLogFile2, "", psBuf, wBufLen, 0, 1); WLOG2(fprintf(pLogFile2, "\n");) if (!pDLM) { return ET9STATUS_NONE; } if (!wBufLen || wBufLen > ET9MAXWORDSIZE) { return ET9STATUS_BAD_PARAM; } { ET9UINT nCount; ET9SYMB const *psSymb; psSymb = psBuf; for (nCount = wBufLen; nCount; --nCount, ++psSymb) { const ET9SymbClass eClass = _ET9_GetSymbolClass(*psSymb); if (!*psSymb || eClass == ET9_WhiteSymbClass || eClass == ET9_UnassSymbClass) { ET9Assert(0); return ET9STATUS_INVALID_TEXT; } } } ++pDLM->dwUpdateCounter; __AssureUpdateSpace(pLingInfo); { ET9U32 dwLdbNum; ET9U16 wWordIndex; dwLdbNum = 0; wWordIndex = __UpdateWord(pLingInfo, psBuf, wBufLen, bUpdateUse, eQuality, bInhibitNewWord, &dwLdbNum); __UpdateContexts(pLingInfo, wWordIndex, dwLdbNum, bUpdateUse, pLocalContext, nLocalContextLen); } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __ResetCaches(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9UINT nIndex; for (nIndex = 0; nIndex < ET9NLM_CONTEXT_COUNT; ++nIndex) { _DLM_PredictionCacheItem * const pCache = &pLingCmnInfo->Private.__DLM_Private.pPredictionCache[nIndex]; pCache->wIndex = __DLM_Index_NULL; pCache->dwIndexHash = 0; pCache->dwIndexState = 0; pCache->dwCurrentState = pCache->dwIndexState + 1; } } /* ************************************************************************************************************** */ /* * PUBLIC API ************************************************************************************************* */ /* ************************************************************************************************************** */ /*---------------------------------------------------------------------------*/ /** * Gets the data size needed for the DLM. * * @return Data size. */ ET9U32 ET9FARCALL ET9AWDLMGetDataSize(void) { return (ET9U32)sizeof(_ET9DLM_info); } /*---------------------------------------------------------------------------*/ /** * Sets up the DLM. * Either using a previously saved DLM or a 'fresh' memory buffer. * Will reset the provided DLM if it looks corrupted. * * @param pLingInfo Pointer to alpha information structure. * @param pDLMInfo Pointer to DLM buffer. * @param dwDataSize Size (in bytes) of provided DLM buffer. * * @return ET9STATUS_NONE on success, otherwise return error code. */ ET9STATUS ET9FARCALL ET9AWDLMInit(ET9AWLingInfo * const pLingInfo, _ET9DLM_info ET9FARDATA * const pDLMInfo, const ET9U32 dwDataSize) { ET9STATUS eStatus; #ifdef _ET9_DLM_REVERSE_HASH { __ReverseHash_Restore(); } #endif WLOG2(fprintf(pLogFile2, "ET9AWDLMInit\n");) WLOG2(fprintf(pLogFile2, "\n");) WLOG2(fprintf(pLogFile2, "_ET9DLM_info = %4.2f MB\n", (float)sizeof(_ET9DLM_info) / (1024*1024));) WLOG2(fprintf(pLogFile2, " __DLM_WordStore = %6d\n", sizeof(__DLM_WordStore));) WLOG2(fprintf(pLogFile2, " __DLM_PredictionsList = %6d\n", sizeof(__DLM_PredictionsList));) WLOG2(fprintf(pLogFile2, "\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if ((pDLMInfo && !dwDataSize) || (!pDLMInfo && dwDataSize)) { return ET9STATUS_INVALID_MEMORY; } if (pDLMInfo && dwDataSize != ET9AWDLMGetDataSize()) { return ET9STATUS_INVALID_SIZE; } if (pDLMInfo && pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_ALREADY_INITIALIZED; } if (pDLMInfo && (pLingInfo->pLingCmnInfo->pRUDBInfo || pLingInfo->pLingCmnInfo->pCDBInfo)) { return ET9STATUS_BAD_CONFIG; } #ifndef ET9_USE_FLOAT_FREQS if (pDLMInfo) { return ET9STATUS_ERROR; } #endif pLingInfo->pLingCmnInfo->pDLMInfo = pDLMInfo; pLingInfo->pLingCmnInfo->Private.bStateDLMEnabled = pDLMInfo ? 1 : 0; if (!pDLMInfo) { return ET9STATUS_NONE; } if (pDLMInfo->bVersion != __DLM_Version || pDLMInfo->wInitOk != ET9GOODSETUP || pDLMInfo->dwDataSize != dwDataSize|| pDLMInfo->wDataWordCount != __DLM_WordCount|| pDLMInfo->wDataPoolCharCount != __DLM_PoolCharCount|| pDLMInfo->wDataContextCount != __DLM_ContextCount|| pDLMInfo->wDataListChunkCount != __DLM_ListChunkCount) { WLOG2(fprintf(pLogFile2, " corrupt - reset\n");) return ET9AWDLMReset(pLingInfo); } __ResetCaches(pLingInfo); #ifdef ET9_DEBUG /* validate prediction content (word repeat etc) */ { _ET9DLM_info ET9FARDATA * const pDLM = pLingInfo->pLingCmnInfo->pDLMInfo; ET9UINT nContextIndex; for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths; ++nContextIndex) { __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; ET9U16 wPredictionIndex; for (wPredictionIndex = 0; wPredictionIndex < pPredictionList->wUsedPredictionCount; ++wPredictionIndex) { __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; __ValidatePrediction(pPredictionList, pPrediction); } } } /* validate list chunk use */ { _ET9DLM_info ET9FARDATA * const pDLM = pLingInfo->pLingCmnInfo->pDLMInfo; ET9UINT nContextIndex; for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths; ++nContextIndex) { ET9UINT nUsedListChunkCount = 0; __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; ET9U16 wPredictionIndex; for (wPredictionIndex = 0; wPredictionIndex < pPredictionList->wUsedPredictionCount; ++wPredictionIndex) { __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; if (pPrediction->bPredictionCount) { const ET9UINT nListChunkCount = (pPrediction->bPredictionCount / __DLM_ChunkWordCount) + ((pPrediction->bPredictionCount % __DLM_ChunkWordCount) ? 1 : 0); ET9Assert(nListChunkCount <= __DLM_MaxListChunks); nUsedListChunkCount += nListChunkCount; { ET9UINT nCount; __DLM_PredictionListChunk const * pCurrChunk; pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (nCount = nListChunkCount; nCount; --nCount, pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]) { if (nCount == 1) { ET9Assert(pCurrChunk->wNextChunkIndex == __DLM_Index_NULL); } else { ET9Assert(pCurrChunk->wNextChunkIndex < __DLM_ListChunkCount); } } } } } ET9Assert(nUsedListChunkCount == pPredictionList->wUsedListChunkCount); } } #endif return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * Set the DLM to an initial clean state. * Removes all content from the DLM. * * @param pLingInfo Pointer to alpha information structure. * * @return ET9STATUS_NONE on success, else error status. */ ET9STATUS ET9FARCALL ET9AWDLMReset(ET9AWLingInfo * const pLingInfo) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMReset\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } #ifdef _ET9_DLM_REVERSE_HASH { __ReverseHash_Reset(); } #endif { const ET9U32 dwDataSize = ET9AWDLMGetDataSize(); _ET9DLM_info ET9FARDATA * const pDLM = pLingInfo->pLingCmnInfo->pDLMInfo; const ET9U32 dwUpdateCounter = !pDLM->wInitOk ? 0 : (pDLM->dwUpdateCounter + 1); ET9UINT nIndex; ET9UINT nContext; _ET9ClearMem((ET9U8*)pDLM, dwDataSize); pDLM->bVersion = __DLM_Version; pDLM->wInitOk = ET9GOODSETUP; pDLM->dwDataSize = dwDataSize; pDLM->wDataWordCount = __DLM_WordCount; pDLM->wDataPoolCharCount = __DLM_PoolCharCount; pDLM->wDataContextCount = __DLM_ContextCount; pDLM->wDataListChunkCount = __DLM_ListChunkCount; pDLM->dwUpdateCounter = dwUpdateCounter; /* init indexes to unused */ pDLM->sWordStore.wFreeCharCount = __DLM_PoolCharCount; for (nIndex = 0; nIndex < __DLM_WordCount; ++nIndex) { pDLM->sWordStore.pwWordAccess[nIndex] = __DLM_Index_Unused; pDLM->sWordStore.pWordItems[nIndex].wCharIndex = __DLM_Index_Unused; } for (nContext = 0; nContext < __DLM_ContextLengths; ++nContext) { __DLM_PredictionsList * const pPredictionList = &pDLM->pPredictionsLists[nContext]; for (nIndex = 0; nIndex < __DLM_ContextCount; ++nIndex) { pPredictionList->pPredictions[nIndex].wFirstChunkIndex = __DLM_Index_Unused; } for (nIndex = 0; nIndex < __DLM_ListChunkCount; ++nIndex) { pPredictionList->pPredictionListChunks[nIndex].wNextChunkIndex = __DLM_Index_Unused; } } } __ResetCaches(pLingInfo); #ifdef _ET9_DLM_AUTO_CONTENT { ET9UINT nIndex; for (nIndex = 0; nIndex < 2; ++nIndex) { FILE *pf; char pcFileName[] = "xx_english_0000.txt"; pcFileName[14] = (char)('0' + nIndex); pf = fopen(pcFileName, "rb"); if (pf) { static ET9SYMB psContent[5000000]; ET9U32 dwContentLength = 0; while (!feof(pf) && dwContentLength < sizeof(psContent) / sizeof(ET9SYMB)) { psContent[dwContentLength++] = (ET9U8)fgetc(pf); } fclose(pf); ET9AWDLMScanBuf(pLingInfo, psContent, dwContentLength, ET9PLIDEnglish); } } } #endif return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * Adds a user word to the DLM - a context independent word. * * @param pLingInfo Pointer to alpha information structure. * @param psBuf Pointer to word being added. * @param wBufLen Length (in symbols) of word being added. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWDLMAddWord(ET9AWLingInfo * const pLingInfo, ET9SYMB * const psBuf, const ET9U16 wBufLen) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMAddWord\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return ET9STATUS_CORRUPT_DB; } return _ET9AW_DLM_AddUserWord(pLingInfo, psBuf, wBufLen); } /*---------------------------------------------------------------------------*/ /** * Delete DLM word. * Deletes a given word from the DLM. * * @param pLingInfo Pointer to alpha information structure. * @param psBuf Pointer to word to match for deletion. * @param wBufLen Length (in symbols) of word pointed to by psBuf. * * @return ET9STATUS_NONE on success, otherwise return error code. */ ET9STATUS ET9FARCALL ET9AWDLMDeleteWord(ET9AWLingInfo * const pLingInfo, ET9SYMB * const psBuf, const ET9U16 wBufLen) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMDeleteWord\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return ET9STATUS_CORRUPT_DB; } return _ET9AW_DLM_DeleteWord(pLingInfo, psBuf, wBufLen); } /*---------------------------------------------------------------------------*/ /** * Gets the number of currently defined DLM words. * * @param pLingInfo Pointer to alpha information structure. * @param pwWordCount Pointer to store DLM word count in. * * @return ET9STATUS_NONE on success, otherwise return error code. */ ET9STATUS ET9FARCALL ET9AWDLMGetWordCount(ET9AWLingInfo * const pLingInfo, ET9U16 * const pwWordCount) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMGetWordCount\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return ET9STATUS_CORRUPT_DB; } if (!pwWordCount) { return ET9STATUS_INVALID_MEMORY; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U16 wFirstLangId = (ET9U16)pLingCmnInfo->dwFirstLdbNum; const ET9U16 wSecondLangId = (ET9U16)(ET9AW_GetBilingualSupported(pLingInfo) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum); ET9U16 wWordCount = 0; ET9UINT nIndex; for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; if (pWord->sbQuality == __DLM_StaticCQuality || pWord->sbQuality == __DLM_StaticXQuality || pWord->sbQuality <= __DLM_GetWords_CutOffLevel) { continue; } if (pWord->wLanguageId == __DLM_AnyLanguage || pWord->wLanguageId == wFirstLangId || pWord->wLanguageId == wSecondLangId) { ++wWordCount; } } WLOG2(fprintf(pLogFile2, " wWordCount %u\n", wWordCount);) *pwWordCount = wWordCount; } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * Get DLM Word. * Designed to be used repeatedly to retrieve all of the words contained in the DLM.

* * The first time the routine is called, the value contained in *pwWordLen * should be 0.

* * If an update has been made to the DLM the function restarts the process at the beginning of the DLM * (and returns a status of ET9STATUS_WORD_NOT_FOUND along with the first word).

* * When processing 'forward', the routine will return the first DLM word * and proceed forward through the DLM. When processing in 'reverse', the routine * starts with the last DLM word and then procees backwards * through the DLM.
* If the API is called again after traversing all words, either forward * or in reverse, the routine will return the ET9STATUS_NO_MATCHING_WORDS status. * * @param pLingInfo Pointer to alphabetic information structure. * @param psWordBuf Pointer to buffer for loading word. * @param wWordBufLen Size (in symbols) of the input buffer (should be atleast ET9MAXWORDSIZE). * @param pwWordLen Pointer to load current word length into (should contain the previously retrieved word length on subsequent calls). * @param bForward Non zero to get first/next word, zero to get previous word. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code.
* On success: *psWordBuf contains the (first/next) DLM word, * *pwWordLen contains the DLM word length */ ET9STATUS ET9FARCALL ET9AWDLMGetWord(ET9AWLingInfo * const pLingInfo, ET9SYMB * const psWordBuf, const ET9U16 wWordBufLen, ET9U16 * const pwWordLen, const ET9U8 bForward) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMGetWord\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return ET9STATUS_CORRUPT_DB; } if (!psWordBuf || !pwWordLen) { return ET9STATUS_INVALID_MEMORY; } if (wWordBufLen < ET9MAXWORDSIZE) { return ET9STATUS_BUFFER_TOO_SMALL; } if (*pwWordLen > ET9MAXUDBWORDSIZE) { return ET9STATUS_BAD_PARAM; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U16 wFirstLangId = (ET9U16)pLingCmnInfo->dwFirstLdbNum; const ET9U16 wSecondLangId = (ET9U16)(ET9AW_GetBilingualSupported(pLingInfo) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum); const ET9BOOL bRestart = (*pwWordLen && pLingCmnInfo->Private.__DLM_Private.dwCurrGetWordState != pDLM->dwUpdateCounter) ? 1 : 0; const ET9BOOL bInitialItem = (!*pwWordLen || bRestart) ? 1 : 0; if (!pDLM->sWordStore.wWordCount || (!bInitialItem && pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex >= pDLM->sWordStore.wWordCount)) { return ET9STATUS_NO_MATCHING_WORDS; } if (bInitialItem) { pLingCmnInfo->Private.__DLM_Private.dwCurrGetWordState = pDLM->dwUpdateCounter; pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex = bForward ? 0 : (pDLM->sWordStore.wWordCount - 1); } else { if (bForward) { ++pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex; } else { --pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex; } } for (;;) { if (pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex >= pDLM->sWordStore.wWordCount) { return ET9STATUS_NO_MATCHING_WORDS; } { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex]; if (pWord->sbQuality == __DLM_StaticCQuality || pWord->sbQuality == __DLM_StaticXQuality || pWord->sbQuality <= __DLM_GetWords_CutOffLevel) { } else if (pWord->wLanguageId == __DLM_AnyLanguage || pWord->wLanguageId == wFirstLangId || pWord->wLanguageId == wSecondLangId) { *pwWordLen = pWord->bWordLength; _ET9SymCopy(psWordBuf, &pDLM->sWordStore.psPoolChars[pWord->wCharIndex], pWord->bWordLength); break; } } if (bForward) { ++pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex; } else { --pLingCmnInfo->Private.__DLM_Private.nCurrGetWordIndex; } } if (bRestart) { return ET9STATUS_WORD_NOT_FOUND; } return ET9STATUS_NONE; } } /*---------------------------------------------------------------------------*/ /** * Determine if a given word exists in the DLM (a user word, not static). * Find a word by passing a word string and the word length. * * @param pLingInfo Pointer to alpha information structure. * @param psBuf Pointer to word to search for. * @param wBufLen Length of buf pointed to by psBuf. * * @return ET9STATUS_NONE if word is found; ET9STATUS_NO_MATCHING_WORDS if not. */ ET9STATUS ET9FARCALL ET9AWDLMFindWord(ET9AWLingInfo * const pLingInfo, ET9SYMB * const psBuf, const ET9U16 wBufLen) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMFindWord\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return ET9STATUS_CORRUPT_DB; } if (!psBuf) { return ET9STATUS_INVALID_MEMORY; } if (wBufLen < 2 || wBufLen > ET9MAXUDBWORDSIZE) { return ET9STATUS_BAD_PARAM; } __LogDLM(pLingInfo, pLogFile2, 0); { const ET9BOOL bFound = _ET9AW_DLM_FindWord(pLingInfo, psBuf, wBufLen, 0, 1); return bFound ? ET9STATUS_NONE : ET9STATUS_NO_MATCHING_WORDS; } } /*---------------------------------------------------------------------------*/ /** * Scans buffer for LM content. * * @param pLingInfo Pointer to alpha information structure. * @param psBuf Pointer to buffer to be parsed. * @param dwBufLen Length of buffer pointed to by psBuf (in symbols). * @param dwLdbNum Language id - ET9PLIDNone (zero) uses the currently active language. * @param bHighQuality Whether the data scanned is high quality or not (will affect quarantining etc). * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWDLMScanBuf(ET9AWLingInfo * const pLingInfo, ET9SYMB * const psBuf, const ET9U32 dwBufLen, const ET9U32 dwLdbNum, const ET9BOOL bHighQuality) { ET9STATUS eStatus; ET9U32 dwAddedWordCount = 0; WLOG2(fprintf(pLogFile2, "ET9AWDLMScanBuf, dwBufLen %u, dwLdbNum %08x, bHighQuality %u\n", dwBufLen, dwLdbNum, bHighQuality);) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return ET9STATUS_CORRUPT_DB; } if (!psBuf) { return ET9STATUS_INVALID_MEMORY; } if (dwBufLen < 1) { return ET9STATUS_BAD_PARAM; } if (dwLdbNum) { if (!_ET9_LanguageSpecific_IsDLMLanguage(dwLdbNum)) { WLOG2(fprintf(pLogFile2, "ET9AWDLMScanBuf, skip, not a DLM language (%08x)\n", dwLdbNum);) return ET9STATUS_NONE; } } else if (!_ET9_LanguageSpecific_IsDLMLanguage(pLingInfo->pLingCmnInfo->dwFirstLdbNum)) { WLOG2(fprintf(pLogFile2, "ET9AWDLMScanBuf, skip, not a DLM language (%08x)\n", pLingInfo->pLingCmnInfo->dwFirstLdbNum);) return ET9STATUS_NONE; } { const ET9U8 bOldSwitchLanguage = pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage; const ET9U32 dwOldFirstLdbNum = pLingInfo->pLingCmnInfo->dwFirstLdbNum; const ET9U32 dwOldSecondLdbNum = pLingInfo->pLingCmnInfo->dwSecondLdbNum; const ET9BOOL bOldCDBEnabled = pLingInfo->pLingCmnInfo->Private.bStateCDBEnabled; /* assure proper settings */ if ((dwLdbNum & ET9PLIDMASK) != ET9SLIDNone) { pLingInfo->pLingCmnInfo->dwFirstLdbNum = dwLdbNum; if ((pLingInfo->pLingCmnInfo->dwFirstLdbNum & ET9SLIDMASK) == ET9SLIDNone) { pLingInfo->pLingCmnInfo->dwFirstLdbNum += ET9SLIDDEFAULT; } pLingInfo->pLingCmnInfo->dwSecondLdbNum = 0; pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage = 0; } { _ET9AW_DLM_WordQuality eWordQuality = ((bHighQuality && (pLingInfo->pLingCmnInfo->dwFirstLdbNum & ET9PLIDMASK)) ? _ET9AW_DLM_WordQuality_ScanHigh : _ET9AW_DLM_WordQuality_ScanLow); ET9U32 dwWordStartPos; /* disable the CDB */ pLingInfo->pLingCmnInfo->Private.bStateCDBEnabled = 0; /* loop over buf for words */ for (dwWordStartPos = 0; dwWordStartPos < dwBufLen; ) { if (_ET9_IsWhiteSpace(psBuf[dwWordStartPos])) { ++dwWordStartPos; continue; } { ET9U32 dwWordEndPos; for (dwWordEndPos = dwWordStartPos; dwWordEndPos < dwBufLen; ++dwWordEndPos) { if (dwWordEndPos + 1 >= dwBufLen) { break; } if (_ET9_IsWhiteSpace(psBuf[dwWordEndPos + 1])) { break; } } /* found word */ if (dwWordEndPos - dwWordStartPos + 1 <= 0xFFFF) { const ET9U16 wWordLen = (ET9U16)(dwWordEndPos - dwWordStartPos + 1); const ET9U32 dwContextStartPos = (dwWordStartPos < 1000) ? 0 : (dwWordStartPos - 1000); const ET9UINT nContextLen = (ET9UINT)(dwWordStartPos - dwContextStartPos); if (!ET9AWFillContextBuffer(pLingInfo, &psBuf[dwContextStartPos], nContextLen)) { _ET9AW_DLM_AddWord(pLingInfo, &psBuf[dwWordStartPos], wWordLen, 1, eWordQuality, 0); ++dwAddedWordCount; } } /* continue after word */ dwWordStartPos = dwWordEndPos + 1; } } } pLingInfo->pLingCmnInfo->dwFirstLdbNum = dwOldFirstLdbNum; pLingInfo->pLingCmnInfo->dwSecondLdbNum = dwOldSecondLdbNum; pLingInfo->pLingCmnInfo->Private.bStateCDBEnabled = bOldCDBEnabled; pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage = bOldSwitchLanguage; } ET9AWFillContextBuffer(pLingInfo, NULL, 0); WLOG2(fprintf(pLogFile2, "ET9AWDLMScanBuf, added %u words\n", dwAddedWordCount);) #ifdef _ET9_DLM_REVERSE_HASH { __ReverseHash_Persist(); } #endif __LogDLM(pLingInfo, pLogFile2, 1); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * Gets the number of bytes needed by the export function. * (This amount of memory is what should be passed into the “export” function given the current DLM.) * * @param pLingInfo Pointer to alpha information structure. * * @return Data size. */ ET9U32 ET9FARCALL ET9AWDLMGetExportMaxDataSize(ET9AWLingInfo * const pLingInfo) { ET9STATUS eStatus; eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return 0; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return 0; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return 0; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9U32 dwCurrOffset = 0; /* header */ dwCurrOffset += 1; /* bVersion */ dwCurrOffset += 4; /* dwUpdateCounter */ /* words */ { ET9UINT nIndex; dwCurrOffset += 2; /* pDLM->sWordStore.wWordCount */ for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; dwCurrOffset += 2; /* pWord->wUseCount */ dwCurrOffset += 2; /* pWord->wLanguageId */ dwCurrOffset += 1; /* pWord->bWordLength */ dwCurrOffset += 1; /* pWord->sbQuality */ dwCurrOffset += 1; /* pWord->bIsCapitalized */ dwCurrOffset += 2 * pWord->bWordLength; /* pDLM->sWordStore.psPoolChars */ } } /* predictions */ { ET9UINT nContextIndex; dwCurrOffset += 1; /* __DLM_ContextLengths */ for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths; ++nContextIndex) { __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; ET9UINT nPredictionIndex; dwCurrOffset += 2; /* pPredictionList->wUsedPredictionCount */ for (nPredictionIndex = 0; nPredictionIndex < pPredictionList->wUsedPredictionCount; ++nPredictionIndex) { __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[nPredictionIndex]; dwCurrOffset += 4; /* pPrediction->dwHashValue */ dwCurrOffset += 1; /* pPrediction->bPredictionCount */ dwCurrOffset += (2 + 2) * (!pPrediction->bPredictionCount ? 1 : pPrediction->bPredictionCount); /* pPrediction -> items */ } } } /* checksum */ dwCurrOffset += 4; /* make it even K's and add an extra */ { const ET9U32 dwKs = dwCurrOffset / 1024 + ((dwCurrOffset % 1024) ? 1 : 0) + 1; const ET9U32 dwSize = dwKs * 1024; ET9Assert(dwCurrOffset <= dwSize); return dwSize; } } } /*---------------------------------------------------------------------------*/ /** * Will serialize the DLM content in a way that it can be used to transfer content between different devices. * E.g. different memory architecture, endianness and different core versions. * * @param pLingInfo Pointer to alpha information structure. * @param pbBuf Pointer to buffer that will receive the data written. * @param dwBufLen Length of buffer pointed to by psBuf (in bytes). * @param pdwBufUsed Will contain the actual number of bytes used, the number that should be used when importing it. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWDLMExport(ET9AWLingInfo * const pLingInfo, ET9U8 * const pbBuf, const ET9U32 dwBufLen, ET9U32 * const pdwBufUsed) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMExport\n");) if (pdwBufUsed) { *pdwBufUsed = 0; } eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (pLingInfo->pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return ET9STATUS_CORRUPT_DB; } if (!pbBuf) { return ET9STATUS_INVALID_MEMORY; } if (!pdwBufUsed) { return ET9STATUS_INVALID_MEMORY; } if (dwBufLen < ET9AWDLMGetExportMaxDataSize(pLingInfo)) { return ET9STATUS_BAD_PARAM; } __LogDLM(pLingInfo, pLogFile2, 1); { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9U32 dwCurrOffset = 0; /* header */ dwCurrOffset = _ET9WriteByte(pbBuf, dwCurrOffset, pDLM->bVersion); dwCurrOffset = _ET9WriteWord4(pbBuf, dwCurrOffset, pDLM->dwUpdateCounter); /* words */ { ET9UINT nIndex; dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, pDLM->sWordStore.wWordCount); for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, pWord->wUseCount); dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, pWord->wLanguageId); dwCurrOffset = _ET9WriteByte(pbBuf, dwCurrOffset, pWord->bWordLength); dwCurrOffset = _ET9WriteByte(pbBuf, dwCurrOffset, pWord->sbQuality); dwCurrOffset = _ET9WriteByte(pbBuf, dwCurrOffset, pWord->bIsCapitalized); { ET9SYMB const * const psString = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; ET9UINT nCharIndex; for (nCharIndex = 0; nCharIndex < pWord->bWordLength; ++nCharIndex) { dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, psString[nCharIndex]); } } } } /* predictions */ { ET9UINT nContextIndex; dwCurrOffset = _ET9WriteByte(pbBuf, dwCurrOffset, __DLM_ContextLengths); for (nContextIndex = 0; nContextIndex < __DLM_ContextLengths; ++nContextIndex) { __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; ET9UINT nPredictionIndex; dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, pPredictionList->wUsedPredictionCount); for (nPredictionIndex = 0; nPredictionIndex < pPredictionList->wUsedPredictionCount; ++nPredictionIndex) { __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[nPredictionIndex]; dwCurrOffset = _ET9WriteWord4(pbBuf, dwCurrOffset, pPrediction->dwHashValue); dwCurrOffset = _ET9WriteByte(pbBuf, dwCurrOffset, pPrediction->bPredictionCount ? pPrediction->bPredictionCount : 1); if (!pPrediction->bPredictionCount) { dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, pDLM->sWordStore.pwWordAccess[pPrediction->wFirstChunkIndex]); dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, 1); } else { ET9U8 bWordIndex; __DLM_PredictionListChunk const * pCurrChunk; pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; } dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, pDLM->sWordStore.pwWordAccess[pCurrChunk->pItems[bChunkWordIndex].wWordIndex]); dwCurrOffset = _ET9WriteWord2(pbBuf, dwCurrOffset, pCurrChunk->pItems[bChunkWordIndex].wUseCount); } } } } } /* checksum */ { const ET9U32 dwChecksum = _ET9ByteCheckSum(pbBuf, dwCurrOffset); dwCurrOffset = _ET9WriteWord4(pbBuf, dwCurrOffset, dwChecksum); } /* done */ ET9Assert(dwCurrOffset <= dwBufLen); *pdwBufUsed = dwCurrOffset; /* zero out the rest */ for (; dwCurrOffset < dwBufLen; ++dwCurrOffset) { pbBuf[dwCurrOffset] = 0; } } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * Will reset the DLM and import serialized content. * If the current configuration is smaller than the exporting one and perhaps won’t fit all content, * then as much as can fit will be retained. * * @param pLingInfo Pointer to alpha information structure. * @param pbBuf Pointer to buffer that holds data to be read. * @param dwBufLen Length of buffer pointed to by psBuf (in bytes). * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWDLMImport(ET9AWLingInfo * const pLingInfo, ET9U8 const * const pbBuf, const ET9U32 dwBufLen) { ET9STATUS eStatus; WLOG2(fprintf(pLogFile2, "ET9AWDLMImport\n");) eStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (eStatus) { return eStatus; } if (!pLingInfo->pLingCmnInfo->pDLMInfo) { return ET9STATUS_NO_DLM; } if (!pbBuf) { return ET9STATUS_INVALID_MEMORY; } if (dwBufLen < 1) { return ET9STATUS_BAD_PARAM; } eStatus = ET9AWDLMReset(pLingInfo); if (eStatus) { return eStatus; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9U32 dwCurrOffset = 0; /* header */ { ET9U8 bVersion; dwCurrOffset = _ET9ReadByte(pbBuf, dwCurrOffset, &bVersion); if (bVersion != pDLM->bVersion) { return ET9STATUS_DLM_VERSION_ERROR; } { ET9U32 dwUpdateCounter; dwCurrOffset = _ET9ReadWord4(pbBuf, dwCurrOffset, &dwUpdateCounter); pDLM->dwUpdateCounter = dwUpdateCounter; } } /* words */ { ET9U16 wWordCount; ET9UINT nIndex; dwCurrOffset = _ET9ReadWord2(pbBuf, dwCurrOffset, &wWordCount); for (nIndex = 0; nIndex < wWordCount; ++nIndex) { const ET9U16 wWordIndex = pDLM->sWordStore.wWordCount; __DLM_WordItem * const pWord = (pDLM->sWordStore.wWordCount < __DLM_WordCount) ? &pDLM->sWordStore.pWordItems[wWordIndex] : NULL; ET9U16 wUseCount; ET9U16 wLanguageId; ET9U8 bWordLength; ET9U8 bQuality; ET9U8 bIsCapitalized; dwCurrOffset = _ET9ReadWord2(pbBuf, dwCurrOffset, &wUseCount); dwCurrOffset = _ET9ReadWord2(pbBuf, dwCurrOffset, &wLanguageId); dwCurrOffset = _ET9ReadByte(pbBuf, dwCurrOffset, &bWordLength); dwCurrOffset = _ET9ReadByte(pbBuf, dwCurrOffset, &bQuality); dwCurrOffset = _ET9ReadByte(pbBuf, dwCurrOffset, &bIsCapitalized); { const ET9U16 wCharIndex = __DLM_PoolCharCount - pDLM->sWordStore.wFreeCharCount; ET9SYMB * const psString = (bWordLength <= pDLM->sWordStore.wFreeCharCount) ? &pDLM->sWordStore.psPoolChars[wCharIndex] : NULL; ET9UINT nCharIndex; for (nCharIndex = 0; nCharIndex < bWordLength; ++nCharIndex) { ET9SYMB sSymb; dwCurrOffset = _ET9ReadWord2(pbBuf, dwCurrOffset, &sSymb); if (psString) { psString[nCharIndex] = sSymb; } } if (pWord && psString && nIndex == wWordIndex) { pWord->dwStringHash = __CalculateWordHashValue(psString, bWordLength); pWord->wCharIndex = wCharIndex; pWord->wUseCount = wUseCount; pWord->wAccessIndex = wWordIndex; pWord->wLanguageId = wLanguageId; pWord->bWordLength = bWordLength; pWord->sbQuality = (ET9S8)bQuality; pWord->bIsCapitalized = (ET9BOOL)bIsCapitalized; pDLM->sWordStore.pwWordAccess[pDLM->sWordStore.wWordCount] = wWordIndex; pDLM->sWordStore.wFreeCharCount = (ET9U16)(pDLM->sWordStore.wFreeCharCount - bWordLength); ++pDLM->sWordStore.wWordCount; } } } } /* predictions */ { ET9U8 bContextLengths; ET9UINT nContextIndex; dwCurrOffset = _ET9ReadByte(pbBuf, dwCurrOffset, &bContextLengths); for (nContextIndex = 0; nContextIndex < bContextLengths; ++nContextIndex) { __DLM_PredictionsList * const pPredictionList = (nContextIndex < __DLM_ContextLengths) ? &pDLM->pPredictionsLists[nContextIndex] : NULL; ET9U16 wPredictionCount; ET9UINT nPredictionIndex; dwCurrOffset = _ET9ReadWord2(pbBuf, dwCurrOffset, &wPredictionCount); for (nPredictionIndex = 0; nPredictionIndex < wPredictionCount; ++nPredictionIndex) { __DLM_PredictionList * const pPrediction = (pPredictionList && pPredictionList->wUsedPredictionCount < __DLM_ContextCount) ? &pPredictionList->pPredictions[pPredictionList->wUsedPredictionCount] : NULL; ET9U32 dwHashValue; ET9U8 bPredictionCount; const ET9UINT nMaxPredictionCount = __DLM_MaxListChunks * __DLM_ChunkWordCount; ET9UINT nPredictionItemCount = 0; __DLM_PredictionListItem pPredictionItems[__DLM_MaxListChunks * __DLM_ChunkWordCount]; ET9U8 bWordIndex; dwCurrOffset = _ET9ReadWord4(pbBuf, dwCurrOffset, &dwHashValue); dwCurrOffset = _ET9ReadByte(pbBuf, dwCurrOffset, &bPredictionCount); for (bWordIndex = 0; bWordIndex < bPredictionCount; ++bWordIndex) { ET9U16 wWordIndex; ET9U16 wUseCount; dwCurrOffset = _ET9ReadWord2(pbBuf, dwCurrOffset, &wWordIndex); dwCurrOffset = _ET9ReadWord2(pbBuf, dwCurrOffset, &wUseCount); if (nPredictionItemCount < nMaxPredictionCount && wWordIndex < pDLM->sWordStore.wWordCount) { pPredictionItems[nPredictionItemCount].wWordIndex = wWordIndex; pPredictionItems[nPredictionItemCount].wUseCount = wUseCount; ++nPredictionItemCount; } } { const ET9BOOL bIsSingleUseDirectWord = (nPredictionItemCount == 1 && pPredictionItems[0].wUseCount == 1) ? 1 : 0; if (pPrediction && nPredictionItemCount && (pPredictionList->wUsedListChunkCount < __DLM_ListChunkCount || bIsSingleUseDirectWord)) { ++pPredictionList->wUsedPredictionCount; pPrediction->dwHashValue = dwHashValue; pPrediction->bPredictionCount = bIsSingleUseDirectWord ? 0 : (ET9U8)nPredictionItemCount; if (bIsSingleUseDirectWord) { pPrediction->wFirstChunkIndex = pPredictionItems[0].wWordIndex; } else { ET9U8 bWordIndex; __DLM_PredictionListChunk *pCurrChunk; ET9Assert(pPredictionList->wUsedListChunkCount < __DLM_ListChunkCount); pPrediction->wFirstChunkIndex = pPredictionList->wUsedListChunkCount; pCurrChunk = &pPredictionList->pPredictionListChunks[pPredictionList->wUsedListChunkCount++]; pCurrChunk->wNextChunkIndex = __DLM_Index_NULL; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { pCurrChunk->wNextChunkIndex = (pPredictionList->wUsedListChunkCount < __DLM_ListChunkCount) ? pPredictionList->wUsedListChunkCount : __DLM_Index_NULL; pCurrChunk = (pCurrChunk->wNextChunkIndex != __DLM_Index_NULL) ? &pPredictionList->pPredictionListChunks[pPredictionList->wUsedListChunkCount++] : NULL; if (pCurrChunk) { pCurrChunk->wNextChunkIndex = __DLM_Index_NULL; } } if (pCurrChunk) { pCurrChunk->pItems[bChunkWordIndex] = pPredictionItems[bWordIndex]; } } } } } } } } /* checksum */ { ET9U32 dwImportCheckSum; const ET9U32 dwChecksum = _ET9ByteCheckSum(pbBuf, dwCurrOffset); dwCurrOffset = _ET9ReadWord4(pbBuf, dwCurrOffset, &dwImportCheckSum); if (dwChecksum != dwImportCheckSum || dwCurrOffset != dwBufLen) { (void)ET9AWDLMReset(pLingInfo); return ET9STATUS_CORRUPT_DB; } } } __LogDLM(pLingInfo, pLogFile2, 1); return ET9STATUS_NONE; } /* ************************************************************************************************************** */ /* * PRIVATE API ************************************************************************************************ */ /* ************************************************************************************************************** */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AW_DLM_AddWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen, const ET9BOOL bUpdateUse, const _ET9AW_DLM_WordQuality eQuality, const ET9BOOL bInhibitNewWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; __ProfileStart; ET9Assert(eQuality > _ET9AW_DLM_WordQuality_First && eQuality < _ET9AW_DLM_WordQuality_Last); WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_AddWord, bUpdateUse %c, eQuality %s, bInhibitNewWord %c, string ", (bUpdateUse ? 'Y' : 'N'), __DLM_WORDQ_TOSTRING(eQuality), (bInhibitNewWord ? 'Y' : 'N'));) WLOG2String(pLogFile2, "", psBuf, wBufLen, 0, 1); WLOG2(fprintf(pLogFile2, "\n");) if (!pDLM) { return ET9STATUS_NONE; } if (!psBuf) { return ET9STATUS_INVALID_MEMORY; } if (!wBufLen || wBufLen > ET9MAXWORDSIZE) { return ET9STATUS_BAD_PARAM; } { ET9UINT nLocalContextLen = 0; ET9SimpleWord pLocalContext[__DLM_ContextLengths]; ET9UINT nWordStart; for (nWordStart = 0; nWordStart < wBufLen; ++nWordStart) { ET9UINT nWordEnd; if (!psBuf[nWordStart] || _ET9_IsWhiteSpace(psBuf[nWordStart])) { continue; } for (nWordEnd = nWordStart; nWordEnd < wBufLen; ++nWordEnd) { const ET9UINT nChkIndex = nWordEnd + 1; if (nChkIndex >= wBufLen || !psBuf[nChkIndex] || _ET9_IsWhiteSpace(psBuf[nChkIndex])) { break; } } { ET9STATUS eStatus; const ET9U16 wWordLen = (ET9U16)(nWordEnd - nWordStart + 1); eStatus =__AddOneWord(pLingInfo, &psBuf[nWordStart], wWordLen, bUpdateUse, eQuality, bInhibitNewWord, pLocalContext, nLocalContextLen); if (eStatus) { return eStatus; } /* when not exhausted */ if (nWordEnd + 1 < wBufLen) { /* potentially shift context down */ if (nLocalContextLen) { ET9UINT nIndex; for (nIndex = nLocalContextLen - 1; nIndex; --nIndex) { pLocalContext[nIndex] = pLocalContext[nIndex - 1]; } } /* add current word as context */ _ET9SymCopy(pLocalContext[0].sString, &psBuf[nWordStart], wWordLen); pLocalContext[0].wLen = wWordLen; pLocalContext[0].wCompLen = 0; if (nLocalContextLen < __DLM_ContextLengths) { ++nLocalContextLen; } } } /* move to the beginning of the word (will get an extra bump in the loop) */ nWordStart = nWordEnd + 1; } } __ProfileEnd(tAW_DLM_AddWord); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9STATUS ET9FARCALL _ET9AW_DLM_AddUserWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_AddUserWord, string ");) WLOG2String(pLogFile2, "", psBuf, wBufLen, 0, 1); WLOG2(fprintf(pLogFile2, "\n");) if (!pDLM) { return ET9STATUS_NONE; } if (!psBuf) { return ET9STATUS_INVALID_MEMORY; } if (wBufLen < 2 || wBufLen > ET9MAXWORDSIZE) { return ET9STATUS_BAD_PARAM; } if (_ET9FindSpacesAndUnknown(psBuf, wBufLen)) { return ET9STATUS_INVALID_TEXT; } ++pDLM->dwUpdateCounter; __AssureUpdateSpace(pLingInfo); { ET9U32 dwLdbNum; dwLdbNum = __DLM_AnyLanguage; (void)__UpdateWord(pLingInfo, psBuf, wBufLen, 0, _ET9AW_DLM_WordQuality_UserHigh, 0, &dwLdbNum); } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AW_DLM_DeleteWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9UINT nWordCount; ET9UINT nStaticCount = 0; ET9UINT nDeleteCount = 0; ET9SYMB psBufOC[ET9MAXWORDSIZE]; if (!pDLM) { return ET9STATUS_NONE; } if (!psBuf) { return ET9STATUS_INVALID_MEMORY; } if (!wBufLen || wBufLen > ET9MAXWORDSIZE) { return ET9STATUS_BAD_PARAM; } if (_ET9FindSpacesAndUnknown(psBuf, wBufLen)) { return ET9STATUS_INVALID_TEXT; } WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_DeleteWord, string ");) WLOG2String(pLogFile2, "", psBuf, wBufLen, 0, 1); WLOG2(fprintf(pLogFile2, "\n");) ++pDLM->dwUpdateCounter; /* first check to see if the exact word exists as a non exact static */ { const ET9U32 dwBufHash = __CalculateWordHashValue(psBuf, wBufLen); for (nWordCount = pDLM->sWordStore.wWordCount; nWordCount; --nWordCount) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nWordCount - 1]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); if (pWord->dwStringHash != dwBufHash) { continue; } if (pWord->bWordLength != wBufLen) { continue; } if (pWord->sbQuality <= __DLM_UserKillQuality) { continue; } if (pWord->sbQuality == __DLM_StaticXQuality) { continue; } { ET9UINT nCount; ET9SYMB const * psSymbWord; ET9SYMB const * psSymb; psSymb = psBuf; psSymbWord = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = wBufLen; nCount; --nCount, ++psSymbWord, ++psSymb) { if (*psSymbWord != *psSymb) { break; } } if (!nCount) { WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_DeleteWord, found @ %u (exact), wAccessIndex %u, sbQuality %d\n", nWordCount - 1, pWord->wAccessIndex, pWord->sbQuality);) ++nDeleteCount; __MarkWordUnused(pLingInfo, pWord->wAccessIndex); } } } if (nDeleteCount) { /* stop here when at least one exact word was found */ __UpdatePredictionsForMissingWords(pLingInfo); return ET9STATUS_NONE; } } /* create other case version */ { ET9UINT nCount; ET9SYMB const * psSymbSrc; ET9SYMB * psSymbOC; psSymbSrc = psBuf; psSymbOC = psBufOC; for (nCount = wBufLen; nCount; --nCount, ++psSymbSrc, ++psSymbOC) { *psSymbOC = _ET9SymToOther(*psSymbSrc, pLingInfo->pLingCmnInfo->dwFirstLdbNum); } } /* mark matching words unused */ { const ET9U32 dwBufHash = __CalculateWordHashValue(psBuf, wBufLen); for (nWordCount = pDLM->sWordStore.wWordCount; nWordCount; --nWordCount) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nWordCount - 1]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); if (pWord->dwStringHash != dwBufHash) { continue; } if (pWord->bWordLength != wBufLen) { continue; } { ET9UINT nCount; ET9SYMB const * psSymbWord; ET9SYMB const * psSymb; ET9SYMB const * psSymbOC; psSymb = psBuf; psSymbOC = psBufOC; psSymbWord = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = wBufLen; nCount; --nCount, ++psSymbWord, ++psSymb, ++psSymbOC) { if (*psSymbWord != *psSymb && *psSymbWord != *psSymbOC) { break; } } if (!nCount) { WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_DeleteWord, found @ %u, wAccessIndex %u, sbQuality %d\n", nWordCount - 1, pWord->wAccessIndex, pWord->sbQuality);) /* even previous "kill" words will be removed and re-inserted to capture the latest kill casing (very unlikely case ever) */ if (pWord->sbQuality == __DLM_StaticXQuality || pWord->sbQuality <= __DLM_UserKillQuality) { ++nStaticCount; } ++nDeleteCount; __MarkWordUnused(pLingInfo, pWord->wAccessIndex); } } } } /* update all predictions that used the words */ if (nDeleteCount) { __UpdatePredictionsForMissingWords(pLingInfo); } /* possibly add an active delete */ if (nDeleteCount == nStaticCount) { /* either no words found or all are static/kill */ __AssureUpdateSpace(pLingInfo); __AddNewWord(pLingInfo, psBuf, wBufLen, __DLM_AnyLanguage, __DLM_UserKillQuality); } /* done */ return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ ET9BOOL ET9FARCALL _ET9AW_DLM_FindWord(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen, const ET9BOOL bAllowStatic, const ET9BOOL bCaseSensitive) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9U16 wWordIndex; __ProfileStart; WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_FindWord\n");) if (!pDLM || !ET9DLMENABLED(pLingCmnInfo)) { return 0; } if (!psBuf) { return ET9STATUS_INVALID_MEMORY; } if (!wBufLen || wBufLen > ET9MAXWORDSIZE) { return 0; } { wWordIndex = __FindWord(pLingInfo, psBuf, wBufLen, __DLM_AnyLanguage, bCaseSensitive); if (wWordIndex == __DLM_Index_NULL) { } else { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[wWordIndex]]; if (pWord->sbQuality > __DLM_FindWord_CutOffLevel && (bAllowStatic || (pWord->sbQuality != __DLM_StaticCQuality && pWord->sbQuality != __DLM_StaticXQuality))) { return 1; } } } { wWordIndex = __FindWord(pLingInfo, psBuf, wBufLen, (ET9U16)pLingCmnInfo->dwFirstLdbNum, bCaseSensitive); if (wWordIndex == __DLM_Index_NULL) { } else { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[wWordIndex]]; if (pWord->sbQuality > __DLM_FindWord_CutOffLevel && (bAllowStatic || (pWord->sbQuality != __DLM_StaticCQuality && pWord->sbQuality != __DLM_StaticXQuality))) { return 1; } } } if (ET9AW_GetBilingualSupported(pLingInfo)) { wWordIndex = __FindWord(pLingInfo, psBuf, wBufLen, (ET9U16)pLingCmnInfo->dwSecondLdbNum, bCaseSensitive); if (wWordIndex == __DLM_Index_NULL) { } else { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[wWordIndex]]; if (pWord->sbQuality > __DLM_FindWord_CutOffLevel && (bAllowStatic || (pWord->sbQuality != __DLM_StaticCQuality && pWord->sbQuality != __DLM_StaticXQuality))) { return 1; } } } __ProfileEnd(tAW_DLM_FindWord); return 0; } /*---------------------------------------------------------------------------*/ typedef struct __AddCandidateContext_s { ET9U16 wIndex; ET9U16 wLength; ET9_FREQ_DESIGNATION bFreqIndicator; ET9U16 wFirstLanguageId; ET9U16 wSecondLanguageId; ET9U8 bBothLangIndex; ET9UINT nAddCount; ET9UINT nMatchCount; ET9BOOL bCheckUnigramCaps; ET9AWPrivWordInfo sLocalWord; } __AddCandidateContext; /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * * * * * */ ET9INLINE static void ET9LOCALCALL __MatchAndAdd(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator, ET9UINT * const pnMatchCount, ET9UINT * const pnAddCount) { /* match if not NWP */ if (wLength) { ET9U8 bMatch; ET9STATUS eStatus; ++*pnMatchCount; eStatus = _ET9AWSelLstWordMatch(pLingInfo, pWord, wIndex, wLength, &bMatch, bFreqIndicator); if (eStatus || !bMatch) { return; } } /* actual add */ ++*pnAddCount; (void)_ET9AWSelLstAdd(pLingInfo, pWord, wLength, bFreqIndicator); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9INLINE static void ET9LOCALCALL __AddCandidate(ET9AWLingInfo * const pLingInfo, const ET9UINT nWordIndex, const ET9U8 bOrder, const ET9FREQPART xWordFreq, __AddCandidateContext * const pContext) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9FLOAT fScoreFactor = (ET9FLOAT)ET9_DB_MAX_FREQ; const ET9UINT nWordItemIndex = pDLM->sWordStore.pwWordAccess[nWordIndex]; __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nWordItemIndex]; ET9BOOL bForceHidden = 0; ET9BOOL bIsOutOfLanguage = 0; ET9U8 bLangIndex; ET9Assert(!(pWord->bWordLength == 1 && _ET9_IsPunctChar(pDLM->sWordStore.psPoolChars[pWord->wCharIndex])) || pWord->sbQuality <= __DLM_UserKillQuality || pWord->sbQuality >= __DLM_UserAddQuality); /* assign language index, skip if unwanted language */ if (pWord->wLanguageId == __DLM_AnyLanguage) { bLangIndex = pContext->bBothLangIndex; } else if (pWord->wLanguageId == pContext->wFirstLanguageId) { bLangIndex = ET9AWFIRST_LANGUAGE; } else if (pWord->wLanguageId == pContext->wSecondLanguageId) { bLangIndex = ET9AWSECOND_LANGUAGE; } else if (pWord->sbQuality == __DLM_NormalQuality) { bLangIndex = pContext->bBothLangIndex; bIsOutOfLanguage = 1; ET9Assert(!bOrder); } else { return; } ET9Assert((xWordFreq >= 0.0f && xWordFreq <= 1.0f) || bIsOutOfLanguage); /* skip words with the "wrong" length */ if (pContext->wLength && pWord->bWordLength < pLingCmnInfo->Private.wCurrMinSourceLength) { return; } /* skip length one words when not NWP */ if (pWord->bWordLength == 1 && pContext->wLength) { return; } /* skip capitalized unigram words when input isn't shifted (non user explicit words) */ if (!bOrder && pContext->bCheckUnigramCaps && pWord->bIsCapitalized && pWord->sbQuality < __DLM_UserAddQuality && pWord->sbQuality > __DLM_UserKillQuality) { if (pWord->sbQuality == __DLM_StaticXQuality) { } else if (pWord->sbQuality == __DLM_HiddenQuality) { } else if (pWord->sbQuality == __DLM_NormalQuality) { bForceHidden = 1; } else { return; } } /* transfer info to word struct */ _ET9SymCopy(&pContext->sLocalWord.Base.sWord[0], &pDLM->sWordStore.psPoolChars[pWord->wCharIndex], pWord->bWordLength); pContext->sLocalWord.Base.wWordLen = pWord->bWordLength; pContext->sLocalWord.Base.wWordCompLen = 0; pContext->sLocalWord.Base.bLangIndex = 0xcc; pContext->sLocalWord.Body.bWordSrc = ET9WORDSRC_DLM; pContext->sLocalWord.Body.bNLMOrder = bOrder; pContext->sLocalWord.Body.bCollectionPrio = bOrder; pContext->sLocalWord.Body.bLangIndexScoring = bLangIndex; pContext->sLocalWord.Body.bIsWeak = bIsOutOfLanguage; pContext->sLocalWord.Body.bIsBlack = (pWord->sbQuality <= __DLM_UserKillQuality) ? 1 : 0; pContext->sLocalWord.Body.bIsHidden = (pWord->sbQuality == __DLM_HiddenQuality || bForceHidden) ? 1 : 0; pContext->sLocalWord.Body.bIsQuarantine = (pWord->sbQuality < __DLM_NormalQuality && pWord->sbQuality > __DLM_HiddenQuality) ? 1 : 0; pContext->sLocalWord.Body.xWordFreq = (pContext->sLocalWord.Body.bIsBlack || pContext->sLocalWord.Body.bIsWeak || pContext->sLocalWord.Body.bIsQuarantine) ? (ET9FREQPART)1 : (ET9FREQPART)(xWordFreq * fScoreFactor); ET9Assert(pContext->sLocalWord.Body.xWordFreq >= 0.0f && pContext->sLocalWord.Body.xWordFreq <= fScoreFactor); #ifdef ET9_DEBUG pContext->sLocalWord.Body.sScoreContext = pLingInfo->pLingCmnInfo->Private.sCurrContext; #endif if (!pContext->sLocalWord.Body.bIsBlack) { __MatchAndAdd(pLingInfo, &pContext->sLocalWord, pContext->wIndex, pContext->wLength, pContext->bFreqIndicator, &pContext->nMatchCount, &pContext->nAddCount); } else { ET9AWPrivWordInfo sWord; /* original case */ sWord = pContext->sLocalWord; __MatchAndAdd(pLingInfo, &sWord, pContext->wIndex, pContext->wLength, pContext->bFreqIndicator, &pContext->nMatchCount, &pContext->nAddCount); /* lower case version, if original had case */ sWord = pContext->sLocalWord; { ET9UINT nCount; ET9SYMB *psSymb; ET9BOOL bHasCaseInfo; bHasCaseInfo = 0; psSymb = sWord.Base.sWord; for (nCount = sWord.Base.wWordLen; nCount; --nCount, ++psSymb) { const ET9SYMB sLC = _ET9SymToLower(*psSymb, pLingInfo->pLingCmnInfo->dwFirstLdbNum); if (sLC != *psSymb) { bHasCaseInfo = 1; } } if (bHasCaseInfo) { __MatchAndAdd(pLingInfo, &sWord, pContext->wIndex, pContext->wLength, pContext->bFreqIndicator, &pContext->nMatchCount, &pContext->nAddCount); } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __PassScreening(ET9AWLingInfo * const pLingInfo, const ET9UINT nWordIndex, __AddCandidateContext * const pContext) { #if 1 WLOG2(fprintf(pLogFile2, "__PassScreening, nWordIndex %u\n", nWordIndex);) if (!pContext->wLength) { return 1; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9UINT nWordItemIndex = pDLM->sWordStore.pwWordAccess[nWordIndex]; __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nWordItemIndex]; const ET9SYMB sFirstSymb = pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; WLOG2(fprintf(pLogFile2, " bWordLength %u, sFirstSymb %c (%04x)\n", pWord->bWordLength, (char)sFirstSymb, sFirstSymb);) /* black words all pass */ if (pWord->sbQuality <= __DLM_UserKillQuality) { WLOG2(fprintf(pLogFile2, " pass, black word\n");) return 1; } /* screen word length */ if (pWord->bWordLength < pLingCmnInfo->Private.wCurrMinSourceLength) { WLOG2(fprintf(pLogFile2, " rejected, bad length, wCurrMinSourceLength %u\n", pLingCmnInfo->Private.wCurrMinSourceLength);) return 0; } /* if the word starts on a free character, just let it through (makes things more complicated) */ if (_ET9_IsFree(sFirstSymb)) { WLOG2(fprintf(pLogFile2, " pass, starts on free char\n");) return 1; } /* screening depends on if it's trace or not */ if (pLingCmnInfo->Private.bTraceBuild) { /* for trace the first char must be in the start region */ const ET9U8 bSymbFreq = _ET9_GetSymbFreq(pLingCmnInfo, 0, sFirstSymb, NULL, NULL); WLOG2(fprintf(pLogFile2, " %s\n", (bSymbFreq ? "pass, trace, initial match" : "rejected, trace, no initial match"));) return (bSymbFreq) ? 1 : 0; } else { /* for tap the first char must be some place in the beginning depending on available edit distance */ const ET9U8 bCurrMaxEditDistance = pLingCmnInfo->Private.bCurrMaxEditDistance; ET9UINT nIndex; for (nIndex = 0; nIndex <= bCurrMaxEditDistance; ++nIndex) { if (_ET9_GetSymbFreq(pLingCmnInfo, nIndex, sFirstSymb, NULL, NULL)) { WLOG2(fprintf(pLogFile2, " pass, tap, index %u match\n", nIndex);) return 1; } } WLOG2(fprintf(pLogFile2, " rejected, tap, no match withing %u\n", bCurrMaxEditDistance);) return 0; } } #else ET9_UNUSED(pLingInfo); ET9_UNUSED(nWordIndex); ET9_UNUSED(pContext); return 1; #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ static void ET9LOCALCALL __FetchPredictionWordList(ET9AWLingInfo * const pLingInfo, __DLM_PredictionsList const * const pPredictionList, __DLM_PredictionList const * const pPrediction, ET9UINT * const pnCount, __DLM_PredictionListItem * const pItems, ET9U32 * const pdwUseCountSum, const ET9BOOL bApplyRecencyAdjustment, __AddCandidateContext * const pContext) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U16 wFirstLanguageId = pContext->wFirstLanguageId; const ET9U16 wSecondLanguageId = pContext->wSecondLanguageId; *pnCount = 0; *pdwUseCountSum = 0; if (!pPrediction->bPredictionCount) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[pPrediction->wFirstChunkIndex]]; if (!pContext->wLength && pWord->sbQuality > __DLM_UserKillQuality && pWord->sbQuality < __DLM_NormalQuality) { /* skip NWP words in quarantine */ } else if (pWord->wLanguageId == __DLM_AnyLanguage || pWord->wLanguageId == wFirstLanguageId || pWord->wLanguageId == wSecondLanguageId) { pItems[*pnCount].wUseCount = 1; pItems[*pnCount].wWordIndex = pPrediction->wFirstChunkIndex; *pdwUseCountSum += pItems[*pnCount].wUseCount; ++*pnCount; } } else { ET9U8 bWordIndex; __DLM_PredictionListChunk const * pCurrChunk; pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; } { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[pCurrChunk->pItems[bChunkWordIndex].wWordIndex]]; if (!pContext->wLength && pWord->sbQuality > __DLM_UserKillQuality && pWord->sbQuality < __DLM_NormalQuality) { /* skip NWP words in quarantine */ } else if (pWord->wLanguageId == __DLM_AnyLanguage || pWord->wLanguageId == wFirstLanguageId || pWord->wLanguageId == wSecondLanguageId) { pItems[*pnCount] = pCurrChunk->pItems[bChunkWordIndex]; *pdwUseCountSum += pItems[*pnCount].wUseCount; ++*pnCount; } } } } #if 0 if (bApplyRecencyAdjustment && *pnCount > 1) { ET9U32 dwAdjustPool = *pdwUseCountSum / *pnCount; ET9UINT nIndex; if (dwAdjustPool < 2) { dwAdjustPool = 2; } for (nIndex = 0; nIndex < *pnCount; ++nIndex) { ET9U32 dwAdjustment; dwAdjustPool /= 2; if (!dwAdjustPool) { break; } dwAdjustment = dwAdjustPool; if (pItems[nIndex].wUseCount + dwAdjustment > 0xFFFF) { dwAdjustment = 0xFFFF - pItems[nIndex].wUseCount; } pItems[nIndex].wUseCount = (ET9U16)(pItems[nIndex].wUseCount + dwAdjustment); *pdwUseCountSum += dwAdjustment; } } #else ET9_UNUSED(bApplyRecencyAdjustment); #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AW_DLM_WordsSearch(ET9AWLingInfo * const pLingInfo, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9FLOAT fUnseenBackOffWt = 1.00f; const ET9U32 dwMinPredictionUseCount = 5; const ET9FLOAT fWordPadItemUseCount = 1.0f; __AddCandidateContext sAddContext; ET9FLOAT fBackOffWt = 1.0f; __ProfileStart; if (!pDLM || !ET9DLMENABLED(pLingCmnInfo)) { return ET9STATUS_NONE; } if (!wLength && !ET9NEXTWORDPREDICTION_MODE(pLingCmnInfo)) { return ET9STATUS_NONE; } __LogDLM(pLingInfo, pLogFile2, 0); #ifdef _ET9_DLM_REVERSE_HASH { __ReverseHash_Persist(); } #endif WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_WordsSearch, wIndex %d, wLength %d\n", wIndex, wLength);) /* init context */ sAddContext.nAddCount = 0; sAddContext.nMatchCount = 0; sAddContext.wIndex = wIndex; sAddContext.wLength = wLength; sAddContext.bFreqIndicator = bFreqIndicator; sAddContext.wFirstLanguageId = (ET9U16)pLingCmnInfo->dwFirstLdbNum; sAddContext.wSecondLanguageId = (ET9U16)(ET9AW_GetBilingualSupported(pLingInfo) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum); sAddContext.bBothLangIndex = (ET9U8)((ET9AW_GetBilingualSupported(pLingInfo)) ? ET9AWBOTH_LANGUAGES : ET9AWFIRST_LANGUAGE); sAddContext.bCheckUnigramCaps = (wLength && !pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo[wIndex].eShiftState) ? 1 : 0; WLOG2(fprintf(pLogFile2, " context, wFirstLanguageId %04x, wSecondLanguageId %04x\n", sAddContext.wFirstLanguageId, sAddContext.wSecondLanguageId);) _InitPrivWordInfo(&sAddContext.sLocalWord); /* predictions */ if (!wIndex && ET9CONTEXTBASEDPREDICTION(pLingCmnInfo)) { const ET9UINT nContextMaxLen = __UpdateContextData(pLingInfo, pLingInfo->pLingCmnInfo->dwFirstLdbNum, NULL, 0); ET9UINT nContextLen; for (nContextLen = __DLM_ContextLengths; nContextLen; --nContextLen) { if (nContextLen > nContextMaxLen) { fBackOffWt *= fUnseenBackOffWt; } else { const ET9UINT nContextIndex = nContextLen - 1; const ET9U32 dwHasValue = __CalculateContextHashValue(pLingInfo, nContextLen); __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; const ET9U16 wPredictionIndex = __FindPrediction(pLingInfo, nContextIndex, dwHasValue); if (wPredictionIndex != __DLM_Index_NULL) { const ET9U8 bOrder = (ET9U8)(nContextLen + 1); __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; __DLM_PredictionListItem pItems[__DLM_MaxListChunks * __DLM_ChunkWordCount]; ET9U32 dwCountSum; ET9UINT nWordCount; __FetchPredictionWordList(pLingInfo, pPredictionList, pPrediction, &nWordCount, pItems, &dwCountSum, 1, &sAddContext); if (nWordCount) { const ET9U32 dwWordPadCount = (dwCountSum >= dwMinPredictionUseCount) ? 0 : (dwMinPredictionUseCount - dwCountSum); const ET9FLOAT fWordPadCountSum = dwWordPadCount * fWordPadItemUseCount; const ET9FLOAT fCountSum = (ET9FLOAT)(dwCountSum + fWordPadCountSum); ET9FLOAT fTmpBackOffWt = 0; ET9UINT nWordIndex; for (nWordIndex = 0; nWordIndex < nWordCount; ++nWordIndex) { const ET9FLOAT fCount = pItems[nWordIndex].wUseCount; const ET9FLOAT fWordProb = ((fCount / (fCount + 1)) * (fCount / fCountSum)) * fBackOffWt; fTmpBackOffWt += ((1 / (fCount + 1)) * (fCount / fCountSum)); if (__PassScreening(pLingInfo, pItems[nWordIndex].wWordIndex, &sAddContext)) { __AddCandidate(pLingInfo, pItems[nWordIndex].wWordIndex, bOrder, (ET9FREQPART)fWordProb, &sAddContext); } } if (dwWordPadCount) { fTmpBackOffWt += dwWordPadCount * ((1 / (fWordPadItemUseCount + 1)) * (fWordPadItemUseCount / fCountSum)); } if (dwCountSum >= dwMinPredictionUseCount) { fBackOffWt *= fTmpBackOffWt; } else { fBackOffWt *= fUnseenBackOffWt; } } else { fBackOffWt *= fUnseenBackOffWt; } } else { fBackOffWt *= fUnseenBackOffWt; } } } } /* unigram */ if (wLength) { ET9U32 dwCountSum = 0; ET9UINT nWordCount = 0; ET9UINT nIndex; for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; if (pWord->wLanguageId == __DLM_AnyLanguage || pWord->wLanguageId == sAddContext.wFirstLanguageId || pWord->wLanguageId == sAddContext.wSecondLanguageId) { ++nWordCount; if (pWord->sbQuality > __DLM_UserKillQuality) { dwCountSum += pWord->wUseCount; } } } /* all words, even out-of-language */ { const ET9U32 dwWordPadCount = (dwCountSum >= dwMinPredictionUseCount) ? 0 : (dwMinPredictionUseCount - dwCountSum); const ET9FLOAT fWordPadCountSum = dwWordPadCount * fWordPadItemUseCount; const ET9FLOAT fCountSum = (ET9FLOAT)(dwCountSum + fWordPadCountSum); for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; const ET9FLOAT fCount = pWord->wUseCount; const ET9FLOAT fWordProb = ((fCount / (fCount + 1)) * (fCount / fCountSum)) * fBackOffWt; if (__PassScreening(pLingInfo, pWord->wAccessIndex, &sAddContext)) { __AddCandidate(pLingInfo, pWord->wAccessIndex, 0, (ET9FREQPART)fWordProb, &sAddContext); } } } } else if (pDLM->sWordStore.bHasKillWords) { ET9UINT nIndex; for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; if (pWord->sbQuality <= __DLM_UserKillQuality) { __AddCandidate(pLingInfo, pWord->wAccessIndex, 0, 0, &sAddContext); } } } WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_WordsSearch, added %u words, matched %u words\n", sAddContext.nAddCount, sAddContext.nMatchCount);) __ProfileEnd(tAW_DLM_WordSearch); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9BOOL ET9LOCALCALL __HasPredictions(ET9AWLingInfo * const pLingInfo, __DLM_PredictionsList const * const pPredictionList, __DLM_PredictionList const * const pPrediction) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U16 wFirstLangId = (ET9U16)pLingCmnInfo->dwFirstLdbNum; const ET9U16 wSecondLangId = (ET9U16)(ET9AW_GetBilingualSupported(pLingInfo) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum); const ET9U32 dwMinUseCount = 2; ET9U32 dwUseCount = 0; if (!pPrediction->bPredictionCount) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[pPrediction->wFirstChunkIndex]]; if (pWord->wLanguageId == __DLM_AnyLanguage || pWord->wLanguageId == wFirstLangId || pWord->wLanguageId == wSecondLangId) { dwUseCount += 1; } } else { ET9U8 bWordIndex; __DLM_PredictionListChunk const * pCurrChunk; pCurrChunk = &pPredictionList->pPredictionListChunks[pPrediction->wFirstChunkIndex]; for (bWordIndex = 0; bWordIndex < pPrediction->bPredictionCount; ++bWordIndex) { const ET9U8 bChunkWordIndex = bWordIndex % __DLM_ChunkWordCount; if (bWordIndex && !bChunkWordIndex) { pCurrChunk = &pPredictionList->pPredictionListChunks[pCurrChunk->wNextChunkIndex]; } { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[pDLM->sWordStore.pwWordAccess[pCurrChunk->pItems[bChunkWordIndex].wWordIndex]]; if (pWord->wLanguageId == __DLM_AnyLanguage || pWord->wLanguageId == wFirstLangId || pWord->wLanguageId == wSecondLangId) { dwUseCount += pCurrChunk->pItems[bChunkWordIndex].wUseCount; if (dwUseCount >= dwMinUseCount) { break; } } } } } return (dwUseCount >= dwMinUseCount) ? 1 : 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9UINT ET9LOCALCALL __GetBigramPredictionCount(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9UINT nMaxCountOfInterest = 1000; ET9BOOL bDoCheck = 0; __ProfileStart; WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_HasContentConfidence\n");) if (!pDLM || !ET9DLMENABLED(pLingCmnInfo)) { return 0; } if (pLingCmnInfo->dwFirstLdbNum != pLingCmnInfo->Private.__DLM_Private.dwConfFirstLdbNum || pLingCmnInfo->dwSecondLdbNum != pLingCmnInfo->Private.__DLM_Private.dwConfSecondLdbNum) { WLOG2(fprintf(pLogFile2, " language changed\n");) bDoCheck = 1; } if (!bDoCheck && pLingCmnInfo->Private.__DLM_Private.nCurrPredictionCount >= nMaxCountOfInterest) { WLOG2(fprintf(pLogFile2, " returning same as last time, above max interest\n");) return pLingCmnInfo->Private.__DLM_Private.nCurrPredictionCount; } if (!bDoCheck && pLingCmnInfo->Private.__DLM_Private.dwConfUpdateCounter + 10 < pDLM->dwUpdateCounter) { WLOG2(fprintf(pLogFile2, " re-check - enough changed\n");) bDoCheck = 1; } if (!bDoCheck) { WLOG2(fprintf(pLogFile2, " returning same as last time\n");) return pLingCmnInfo->Private.__DLM_Private.nCurrPredictionCount; } { ET9UINT nActivePredictionCount = 0; { __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[0]; ET9U16 wPredictionIndex; for (wPredictionIndex = 0; wPredictionIndex < pPredictionList->wUsedPredictionCount; ++wPredictionIndex) { __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; if (__HasPredictions(pLingInfo, pPredictionList, pPrediction)) { ++nActivePredictionCount; if (nActivePredictionCount >= nMaxCountOfInterest) { break; } } } } WLOG2(fprintf(pLogFile2, " found %u predictions (%u)\n", nActivePredictionCount, nMaxCountOfInterest);) pLingCmnInfo->Private.__DLM_Private.nCurrPredictionCount = nActivePredictionCount; pLingCmnInfo->Private.__DLM_Private.dwConfUpdateCounter = pDLM->dwUpdateCounter; pLingCmnInfo->Private.__DLM_Private.dwConfFirstLdbNum = pLingCmnInfo->dwFirstLdbNum; pLingCmnInfo->Private.__DLM_Private.dwConfSecondLdbNum = pLingCmnInfo->dwSecondLdbNum; } __ProfileEnd(tAW_DLM_Confidence); return pLingCmnInfo->Private.__DLM_Private.nCurrPredictionCount; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9BOOL ET9FARCALL _ET9AW_DLM_HasALMConfidence(ET9AWLingInfo * const pLingInfo) { if (!ET9CONTEXTBASEDPREDICTION(pLingInfo->pLingCmnInfo)) { WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_HasALMConfidence, CBP is off\n");) return 0; } { const ET9UINT nTargetPredictionCount = 100; const ET9UINT nCurrCount = __GetBigramPredictionCount(pLingInfo); WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_HasALMConfidence, nCurrCount %u\n", nCurrCount);) return (nCurrCount >= nTargetPredictionCount) ? 1 : 0; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9FLOAT ET9FARCALL _ET9AW_DLM_GetConfidence(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9UINT nCurrCount = __GetBigramPredictionCount(pLingInfo); WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_GetConfidence, nCurrCount %u, NWP %c\n", nCurrCount, (pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs ? 'N' : 'Y'));) if (!pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs) { return (nCurrCount >= 1000) ? 0.55f : (nCurrCount >= 100) ? 0.55f : 0.55f; } if (pLingCmnInfo->Private.bUsingALM) { return (nCurrCount >= 1000) ? 0.32f : (nCurrCount >= 100) ? 0.30f : 0.25f; } else if (pLingCmnInfo->Private.bUsingLM) { return (nCurrCount >= 1000) ? 0.50f : (nCurrCount >= 100) ? 0.40f : 0.30f; } else if (pLingCmnInfo->Private.bUsingMGD) { return (nCurrCount >= 1000) ? 0.50f : (nCurrCount >= 100) ? 0.40f : 0.30f; } else if (ET9CONTEXTBASEDPREDICTION(pLingInfo->pLingCmnInfo)) { return (nCurrCount >= 1000) ? 0.95f : (nCurrCount >= 100) ? 0.85f : 0.75f; } else { return (nCurrCount >= 1000) ? 0.95f : (nCurrCount >= 100) ? 0.85f : 0.75f; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ void ET9FARCALL _ET9AW_DLM_ProcessSelListEntries(ET9AWLingInfo * const pLingInfo, const ET9UINT nSelectIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9UINT nRemovedCount = 0; ET9UINT nWordIndex; __ProfileStart; if (!pDLM) { return; } __LogDLM(pLingInfo, pLogFile2, 0); WLOG2(fprintf(pLogFile2, "_ET9AW_DLM_ProcessSelListEntries\n");) for (nWordIndex = 0; nWordIndex < pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords; ++nWordIndex) { /* if this was not the selected word */ if (nWordIndex != nSelectIndex) { ET9AWPrivWordInfo const * const pCandidateWord = &pLingCmnInfo->Private.sWordC.pCurrC->pWordList[pLingCmnInfo->Private.sWordC.pCurrC->pnWordList[nWordIndex]]; /* if it is a quarantine word, is a terminal and is NOT a spell correction... */ if (pCandidateWord->Body.bIsQuarantine && !pCandidateWord->Base.bIsSpellCorr && !pCandidateWord->Base.wWordCompLen) { const ET9U16 wFirstLangId = (ET9U16)pLingCmnInfo->dwFirstLdbNum; const ET9U16 wSecondLangId = (ET9U16)(ET9AW_GetBilingualSupported(pLingInfo) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum); const ET9U32 dwCandidateHash = __CalculateWordHashValue(pCandidateWord->Base.sWord, pCandidateWord->Base.wWordLen); ET9SYMB psBufOther[ET9MAXWORDSIZE]; ET9UINT nIndex; WLOG2(fprintf(pLogFile2, " found quarantine candidate @ %u\n", nWordIndex);) for (nIndex = 0; nIndex < pCandidateWord->Base.wWordLen && nIndex < ET9MAXWORDSIZE; ++nIndex) { psBufOther[nIndex] = _ET9SymToOther(pCandidateWord->Base.sWord[nIndex], pLingCmnInfo->dwFirstLdbNum); } for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; if (pWord->dwStringHash != dwCandidateHash) { continue; } if (pWord->bWordLength != pCandidateWord->Base.wWordLen) { continue; } if (pWord->wUseCount > 1) { continue; } if (pWord->sbQuality <= __DLM_HiddenQuality || pWord->sbQuality >= __DLM_NormalQuality) { continue; } if (pWord->wLanguageId != __DLM_AnyLanguage && pWord->wLanguageId != wFirstLangId && pWord->wLanguageId != wSecondLangId) { continue; } { ET9UINT nCount; ET9SYMB const * psSymb1; ET9SYMB const * psSymb2; ET9SYMB const * psSymb3; psSymb1 = pCandidateWord->Base.sWord; psSymb2 = psBufOther; psSymb3 = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = pWord->bWordLength; nCount; --nCount, ++psSymb1, ++psSymb2, ++psSymb3) { if (*psSymb1 != *psSymb3 && *psSymb2 != *psSymb3) { break; } } if (!nCount) { WLOG2(fprintf(pLogFile2, " found quarantine word @ %u (%d)\n", nIndex, pWord->sbQuality);) --pWord->sbQuality; if (pWord->sbQuality <= __DLM_CutLowQuality) { WLOG2(fprintf(pLogFile2, " removing quarantine\n");) ++nRemovedCount; __MarkWordUnused(pLingInfo, pWord->wAccessIndex); --nIndex; /* repeat index */ } } } } } } } if (nRemovedCount) { __UpdatePredictionsForMissingWords(pLingInfo); } __ProfileEnd(tAW_DLM_ProcessEntries); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * */ ET9BOOL ET9FARCALL _ET9AW_Is_DLM_Entries(ET9AWLingInfo * const pLingInfo, ET9SimpleWord const * const pFirstWord, ET9SimpleWord const * const pSecondWord, ET9FREQPART * const pxWordFreq, ET9U8 * const pbNLMOrder, const ET9BOOL bCheckAlternate, ET9SimpleWord * const pAlternateWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; ET9BOOL bAlternateSuggestion = 0; ET9FREQPART xAlternateFreq = 0; ET9U8 bAlternateOrder = 0; const ET9FLOAT fUnseenBackOffWt = 1.00f; const ET9U32 dwMinPredictionUseCount = 5; __AddCandidateContext sAddContext; ET9FLOAT fBackOffWt = 1.0f; ET9Assert(!bCheckAlternate || pAlternateWord); if (!pDLM || !ET9DLMENABLED(pLingCmnInfo) || !ET9CONTEXTBASEDPREDICTION(pLingCmnInfo)) { return 0; } /* init context */ sAddContext.wLength = 0; /* will exclude all quarantine words */ sAddContext.wFirstLanguageId = (ET9U16)pLingCmnInfo->dwFirstLdbNum; sAddContext.wSecondLanguageId = (ET9U16)(ET9AW_GetBilingualSupported(pLingInfo) ? pLingCmnInfo->dwSecondLdbNum : pLingCmnInfo->dwFirstLdbNum); /* predictions */ { const ET9U32 dwSecondWordStringHash = __CalculateWordHashValue(pSecondWord->sString, pSecondWord->wLen); const ET9UINT nContextMaxLen = __UpdateContextData(pLingInfo, pLingInfo->pLingCmnInfo->dwFirstLdbNum, pFirstWord, 1); ET9UINT nContextLen; for (nContextLen = __DLM_ContextLengths; nContextLen; --nContextLen) { if (nContextLen > nContextMaxLen) { fBackOffWt *= fUnseenBackOffWt; } else { const ET9UINT nContextIndex = nContextLen - 1; const ET9U32 dwHasValue = __CalculateContextHashValue(pLingInfo, nContextLen); __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nContextIndex]; const ET9U16 wPredictionIndex = __FindPrediction(pLingInfo, nContextIndex, dwHasValue); if (wPredictionIndex != __DLM_Index_NULL) { const ET9U8 bOrder = (ET9U8)(nContextLen + 1); __DLM_PredictionList const * const pPrediction = &pPredictionList->pPredictions[wPredictionIndex]; __DLM_PredictionListItem pItems[__DLM_MaxListChunks * __DLM_ChunkWordCount]; ET9U32 dwCountSum; ET9UINT nWordCount; __FetchPredictionWordList(pLingInfo, pPredictionList, pPrediction, &nWordCount, pItems, &dwCountSum, 1, &sAddContext); if (nWordCount) { const ET9U32 dwWordPadCount = (dwCountSum >= dwMinPredictionUseCount) ? 0 : (dwMinPredictionUseCount - dwCountSum); const ET9FLOAT fWordPadItemUseCount = 1.0f; const ET9FLOAT fWordPadCountSum = dwWordPadCount * fWordPadItemUseCount; const ET9FLOAT fCountSum = (ET9FLOAT)(dwCountSum + fWordPadCountSum); ET9FLOAT fTmpBackOffWt = 0; ET9UINT nWordIndex; for (nWordIndex = 0; nWordIndex < nWordCount; ++nWordIndex) { const ET9FLOAT fScoreFactor = (ET9FLOAT)ET9_DB_MAX_FREQ; const ET9FLOAT fCount = pItems[nWordIndex].wUseCount; const ET9FLOAT fWordProb = ((fCount / (fCount + 1)) * (fCount / fCountSum)) * fBackOffWt; const ET9UINT nWordItemIndex = pDLM->sWordStore.pwWordAccess[pItems[nWordIndex].wWordIndex]; __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nWordItemIndex]; fTmpBackOffWt += ((1 / (fCount + 1)) * (fCount / fCountSum)); if (pWord->dwStringHash != dwSecondWordStringHash || pWord->bWordLength != pSecondWord->wLen) { continue; } *pxWordFreq = (pWord->sbQuality <= __DLM_UserKillQuality) ? (ET9FREQPART)1 : (ET9FREQPART)(fWordProb * fScoreFactor); *pbNLMOrder = bOrder; { ET9UINT nCount; ET9SYMB const * psSymb; ET9SYMB const * psSymbWord; psSymb = pSecondWord->sString; psSymbWord = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = pSecondWord->wLen; nCount; --nCount, ++psSymb, ++psSymbWord) { if (*psSymb != *psSymbWord) { break; } } if (!nCount) { if (bCheckAlternate) { pAlternateWord->wLen = 0; } return 1; } } if (bCheckAlternate && (!bAlternateSuggestion || *pxWordFreq > xAlternateFreq)) { xAlternateFreq = (pWord->sbQuality <= __DLM_UserKillQuality) ? (ET9FREQPART)1 : (ET9FREQPART)(fWordProb * fScoreFactor); bAlternateOrder = bOrder; _ET9SymCopy(pAlternateWord->sString, &pDLM->sWordStore.psPoolChars[pWord->wCharIndex], pWord->bWordLength); pAlternateWord->wLen = pWord->bWordLength; pAlternateWord->wCompLen = 0; bAlternateSuggestion = 1; } } if (dwWordPadCount) { fTmpBackOffWt += dwWordPadCount * ((1 / (fWordPadItemUseCount + 1)) * (fWordPadItemUseCount / fCountSum)); } if (dwCountSum >= dwMinPredictionUseCount) { fBackOffWt *= fTmpBackOffWt; } else { fBackOffWt *= fUnseenBackOffWt; } } else { fBackOffWt *= fUnseenBackOffWt; } } else { fBackOffWt *= fUnseenBackOffWt; } } } } if (bCheckAlternate && bAlternateSuggestion) { *pxWordFreq = xAlternateFreq; *pbNLMOrder = bAlternateOrder; return 1; } return 0; } #ifdef ET9_QA_ACCESS /*---------------------------------------------------------------------------*/ /* -IDR- * */ ET9BOOL ET9FARCALL _ET9_QA_DLM_GetWordProperties(ET9AWLingInfo * const pLingInfo, ET9SYMB const * const psBuf, const ET9U16 wBufLen, const ET9U16 wLanguageId, ET9U16 * const pwUseCount, ET9S8 * const psbQuality) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; const ET9U32 dwBufHash = __CalculateWordHashValue(psBuf, wBufLen); ET9UINT nIndex; if (_ET9AWSys_BasicValidityCheck(pLingInfo) || !pLingCmnInfo->pDLMInfo || pLingCmnInfo->pDLMInfo->wInitOk != ET9GOODSETUP) { return 0; } for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; if (pWord->dwStringHash != dwBufHash) { continue; } if (pWord->bWordLength != wBufLen) { continue; } if (pWord->wLanguageId != wLanguageId) { continue; } { ET9UINT nCount; ET9SYMB const * psSymb; ET9SYMB const * psSymbWord; psSymb = psBuf; psSymbWord = &pDLM->sWordStore.psPoolChars[pWord->wCharIndex]; for (nCount = wBufLen; nCount; --nCount, ++psSymb, ++psSymbWord) { if (*psSymb != *psSymbWord) { break; } } if (!nCount) { if (pwUseCount) { *pwUseCount = pWord->wUseCount; } if (psbQuality) { *psbQuality = pWord->sbQuality; } return 1; } } } return 0; } #endif /* ET9_QA_ACCESS */ #ifdef ET9_ACTIVATE_DLM_REPORTING #ifndef _WIN32 #error ET9_ACTIVATE_DLM_REPORTING is for win32 only #endif #include /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9S32 ET9FARCALL _ET9AW_DLM_ReportContent(ET9AWLingInfo * const pLingInfo, ET9U8 * const pbBuf, const ET9S32 snBufSize) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; _ET9DLM_info ET9FARDATA * const pDLM = pLingCmnInfo->pDLMInfo; wchar_t *psBufPos = (wchar_t*)pbBuf; ET9S32 snBufLen = snBufSize / sizeof(wchar_t); ET9S32 snLen = 0; if (!pDLM) { swprintf(psBufPos, snBufLen, L"No DLM\n\n"); snLen = wcslen(psBufPos); } else { ET9UINT nIndex; swprintf(psBufPos, snBufLen - snLen, L"DLM, size %u, Update-Counter %u\n\n", pDLM->dwDataSize, pDLM->dwUpdateCounter); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; { ET9UINT nCharUseCount; nCharUseCount = 0; for (nIndex = 0; nIndex < __DLM_PoolCharCount; ++nIndex) { if (pDLM->sWordStore.psPoolChars[nIndex]) { ++nCharUseCount; } } swprintf(psBufPos, snBufLen - snLen, L"Unigram word list, WordCount %5u (%6.2f%%), CharCount %5u (%6.2f%%)\n\n", pDLM->sWordStore.wWordCount, (float)pDLM->sWordStore.wWordCount / __DLM_WordCount * 100.0f, nCharUseCount, (float)nCharUseCount / __DLM_PoolCharCount * 100.0f); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; for (nIndex = 0; nIndex < __DLM_ContextLengths; ++nIndex) { __DLM_PredictionsList const * const pPredictionList = &pDLM->pPredictionsLists[nIndex]; swprintf(psBufPos, snBufLen - snLen, L"%u-gram, UsedPredictionCount %5u (%6.2f%%), UsedListChunkCount %5u (%6.2f%%)\n\n", nIndex + 2, pPredictionList->wUsedPredictionCount, (float)pPredictionList->wUsedPredictionCount / __DLM_ContextCount * 100.0f, pPredictionList->wUsedListChunkCount, (float)pPredictionList->wUsedListChunkCount / __DLM_ListChunkCount * 100.0f); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; } swprintf(psBufPos, snBufLen - snLen, L"\n"); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; } swprintf(psBufPos, snBufLen - snLen, L"-------------------- Unigram word list\n\n"); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; swprintf(psBufPos, snBufLen - snLen, L" Word\tLanguageId\tQuality\t\tUseCount\tString\n\n"); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; for (nIndex = 0; nIndex < pDLM->sWordStore.wWordCount; ++nIndex) { ET9UINT snIndex; wchar_t psQuality[32]; __DLM_WordItem const * const pWord = &pDLM->sWordStore.pWordItems[nIndex]; ET9Assert(pWord->wCharIndex != __DLM_Index_Unused); ET9Assert(pDLM->sWordStore.pwWordAccess[pWord->wAccessIndex] == nIndex); if (pWord->sbQuality <= __DLM_InitialLowQuality && pWord->sbQuality >= __DLM_CutLowQuality) { swprintf(psQuality, 32, L"Q%02d\t\t", __DLM_InitialLowQuality + pWord->sbQuality + 1); } else { switch (pWord->sbQuality) { case __DLM_UserAddQuality: wcscpy(psQuality, L"UserAdd\t"); break; case __DLM_StaticXQuality: wcscpy(psQuality, L"StaticX\t"); break; case __DLM_StaticCQuality: wcscpy(psQuality, L"StaticC\t"); break; case __DLM_NormalQuality: wcscpy(psQuality, L"Normal\t"); break; case __DLM_HiddenQuality: wcscpy(psQuality, L"Hidden\t"); break; case __DLM_UserKillQuality: wcscpy(psQuality, L"UserKill"); break; default: wcscpy(psQuality, L"\t\t"); break; } } swprintf(psBufPos, snBufLen - snLen, L"%5u\t\t %04x\t\t\t%s\t%5u\t\t ", nIndex, pWord->wLanguageId, psQuality, pWord->wUseCount); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; for (snIndex = 0; snIndex < pWord->bWordLength; ++snIndex) { ET9SYMB sSymb = pDLM->sWordStore.psPoolChars[pWord->wCharIndex + snIndex]; if (sSymb >= 0x20 && sSymb <= 0xFF) { swprintf(psBufPos, snBufLen - snLen, L"%c", (unsigned char)sSymb); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; } else { swprintf(psBufPos, snBufLen - snLen, L"<%x>", (int)sSymb); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; } } swprintf(psBufPos, snBufLen - snLen, L"\n"); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; } swprintf(psBufPos, snBufLen - snLen, L"\n\n"); snLen += wcslen(psBufPos); psBufPos = (wchar_t*)pbBuf + snLen; } return snLen; } #endif /* ET9_ACTIVATE_DLM_REPORTING */ #endif /* ET9_ALPHABETIC_MODULE */ /*! @} */ /* ----------------------------------< eof >--------------------------------- */