/******************************************************************************* ;******************************************************************************* ;** ** ;** COPYRIGHT 2001-2012 NUANCE COMMUNICATIONS ** ;** ** ;** NUANCE COMMUNICATIONS PROPRIETARY INFORMATION ** ;** ** ;** This software is supplied under the terms of a license agreement ** ;** or non-disclosure agreement with Nuance Communications and may not ** ;** be copied or disclosed except in accordance with the terms of that ** ;** agreement. ** ;** ** ;******************************************************************************* ;** ** ;** FileName: et9aldb.c ** ;** ** ;** Description: ET9 Alphabetic LDB Module (Intervals) ** ;** ** ;******************************************************************************* ;******************************************************************************/ /*! \addtogroup et9aldb LDB for alphabetic * LDB for alphabetic XT9. * @{ */ #include "et9api.h" #ifdef ET9_ALPHABETIC_MODULE #include "et9imu.h" #include "et9asys.h" #include "et9aldb.h" #include "et9amisc.h" #include "et9sym.h" #include "et9aspc.h" #include "et9aasdb.h" #include "et9alsasdb.h" #include "et9asym.h" #ifdef ET9_DEBUGLOG3 #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG3 ACTIVATED ***") #endif #include #ifdef XT9_ANDROID_BUILD static const char pczzzET9ALDB[] = "/sdcard/data/zzzET9ALDB.txt"; #else static const char pczzzET9ALDB[] = "zzzET9ALDB.txt"; #endif #define WLOG3(q) { if (pLogFile3 == NULL) { pLogFile3 = fopen(pczzzET9ALDB, "w"); } q fflush(pLogFile3); } static FILE *pLogFile3 = NULL; #else #define WLOG3(q) #endif #ifdef ET9_DEBUGLOG3B #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG3B ACTIVATED ***") #endif #define WLOG3B(q) WLOG3(q) #else #define WLOG3B(q) #endif #ifdef ET9_DEBUGLOG7 #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG7 ACTIVATED ***") #endif #include #include #ifdef XT9_ANDROID_BUILD static const char pczzzET9ALDB_LM[] = "/sdcard/data/zzzET9ALDB-LM.txt"; #else static const char pczzzET9ALDB_LM[] = "zzzET9ALDB-LM.txt"; #endif #define WLOG7(q) { if (pLogFile7 == NULL) { pLogFile7 = fopen(pczzzET9ALDB_LM, "w"); } {q} fflush(pLogFile7); } static FILE *pLogFile7 = NULL; #else #define WLOG7(q) #endif #ifdef ET9_DIRECT_LDB_ACCESS #ifdef _WIN32 #pragma message ("*** USING DIRECT LDB ACCESS (FAST) ***") #endif #endif #ifdef __ET9_MGD_COMPLETION #error invalid switch __ET9_MGD_COMPLETION #endif #if 0 /* LM debug*/ #include typedef struct valueItem_s { unsigned char bOrder; float fValue; unsigned int nCount; } valueItem; #endif /******************************************************************************* ** ** G L O B A L S A N D L O C A L S T A T I C S ** ** ET9 does not make use of any dynamic global or local static variables!! ** It is acceptable to make use of constant globals or local statics. ** If you need persistent dynamic memory in the ET9 core, it should be ** allocated in the ET9AWLingPrivate data structure and fogged through the definitions ** found in the et9asystm.h file. ** ******************************************************************************/ #define ASPC (pLingCmnInfo->Private.ASpc) /* < -IDR- */ #define ALDB (pLingCmnInfo->Private.ALdb) /* < -IDR- */ #define ALDBL (pLingInfo->Private.ALdb) /* < -IDR- */ #define ALDB_INTERVAL_MULTI_BYTE_CODE 0xFF /* < -IDR- */ #define ALDB_INTERVAL_END_CHAR_VAL 0x16FD /* < -IDR- */ #define ALDB_INTERVAL_JUMP_CHAR_VAL 0x16FE /* < -IDR- */ #define ALDB_INTERVAL_EXTEND_CHAR_VAL 0x16FF /* < -IDR- */ #define ET9HASHLDBBUFFSIZE 0x100 /* < -IDR- */ #define __fEulerNumberE 2.718281828f /* < -IDR- */ #define __ET9AWMGD_NOFREQ 0xFF /* < -IDR- */ #define __ET9AWMGD_CLASS_PER_STEM 40 /* < -IDR- / */ #define __ET9AWMGD_PATH_ADD 2 /* < -IDR- */ #define __ET9AWMGD_PATH_SUBTRACT 2 /* < -IDR- */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / */ #define __IsPrivateSymbol(wSymbol) \ ((wSymbol) == 0 || \ (wSymbol) == ALDB_INTERVAL_END_CHAR_VAL || \ (wSymbol) == ALDB_INTERVAL_JUMP_CHAR_VAL || \ (wSymbol) == ALDB_INTERVAL_EXTEND_CHAR_VAL) \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ #define __ET9ByteDistance(p1, p2) ((ET9U32)((ET9U8*)(p1) - (ET9U8*)(p2))) #ifdef ET9_DEBUGLOG3 static ET9U32 dwAttemptCount; static ET9U32 dwPassedCount; static ET9U32 dwRejectedCount; static ET9U32 dwLimitedCount; static ET9U32 dwLengthCalcCount; static ET9U32 dwLengthsCount[1000]; static ET9U32 dwJumpsCount[1000]; #define __IncScreenAttempt() { ++dwAttemptCount; } #define __IncScreenPassed() { ++dwPassedCount; } #define __IncScreenRejected() { ++dwRejectedCount; } #define __IncScreenLimited() { ++dwLimitedCount; } #define __IncScreenLengthCalc() { ++dwLengthCalcCount; } #define __IncScreenLength(x) { ++dwLengthsCount[(x) < 1000 ? (x) : 999]; } #define __IncScreenJump(x) { ++dwJumpsCount[(x) < 1000 ? (x) : 999]; } #define __ResetScreenCounters() \ { \ dwAttemptCount = 0; \ dwPassedCount = 0; \ dwRejectedCount = 0; \ dwLimitedCount = 0; \ dwLengthCalcCount = 0; \ memset(dwLengthsCount, 0, sizeof(dwLengthsCount)); \ memset(dwJumpsCount, 0, sizeof(dwJumpsCount)); \ } \ #define __LogScreenCounters() \ { \ int i; \ fprintf(pLogFile3, \ "__LogScreenCounters, Tot %d, pass %d, reject %d, limited %d, lengths %d\n", \ dwAttemptCount, dwPassedCount, dwRejectedCount, dwLimitedCount, dwLengthCalcCount); \ for (i = 0; i < 1000; ++i) { \ if (dwLengthsCount[i]) { \ fprintf(pLogFile3, "Length run %3d : %5d\n", i, dwLengthsCount[i]); \ } \ } \ for (i = 0; i < 1000; ++i) { \ if (dwJumpsCount[i]) { \ fprintf(pLogFile3, "Executed jump %3d : %5d\n", i, dwJumpsCount[i]); \ } \ } \ } \ #else /* ET9_DEBUGLOG3 */ #define __IncScreenAttempt() #define __IncScreenPassed() #define __IncScreenRejected() #define __IncScreenLimited() #define __IncScreenLengthCalc() #define __IncScreenLength(x) #define __IncScreenJump(x) #define __LogScreenCounters() #define __ResetScreenCounters() #endif /* ET9_DEBUGLOG3 */ #ifdef ET9_DEBUGLOG3 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL WLOG3Word(FILE *pF, char *pcComment, ET9AWPrivWordInfo *pWord) { ET9U16 wIndex; ET9Assert(pF); ET9Assert(pcComment); ET9Assert(pWord); fprintf(pF, "%s: ", pcComment); for (wIndex = 0; wIndex < pWord->Base.wWordLen; ++wIndex) { ET9SYMB sSymb = pWord->Base.sWord[wIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F) { fprintf(pF, "%c", (char)sSymb); } else { fprintf(pF, "<%x>", (int)sSymb); } } fprintf(pF, " (%c : %d,%d : %d,%d : %d)\n", (char)(pWord->Base.bIsTerm ? 'T' : 'S'), pWord->Base.wWordLen, pWord->Base.wWordCompLen, pWord->Body.bEditDistSpc, pWord->Body.bEditDistStem, pWord->Body.bWordSrc); fflush(pF); } #else /* ET9_DEBUGLOG3 */ #define WLOG3Word(pF,pcComment,pWord) #endif /* ET9_DEBUGLOG3 */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWLdbBasicValidityCheck(ET9AWLingInfo * const pLingInfo, const ET9BOOL bCheckLDBOk) { if (pLingInfo == NULL) { return ET9STATUS_INVALID_MEMORY; } if (pLingInfo->Private.wInfoInitOK != ET9GOODSETUP) { return ET9STATUS_NO_INIT; } if (pLingInfo->pLingCmnInfo == NULL) { return ET9STATUS_INVALID_MEMORY; } if (pLingInfo->pLingCmnInfo->Private.wInfoInitOK != ET9GOODSETUP) { return ET9STATUS_NO_INIT; } if (bCheckLDBOk && pLingInfo->Private.wLDBInitOK != ET9GOODSETUP) { return ET9STATUS_NO_DB_INIT; } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9INLINE static ET9STATUS ET9LOCALCALL __ET9ReadLDBData (ET9AWLingInfo * const pLingInfo, const ET9U32 dwOffset, const ET9U32 dwNumberOfBytesToRead, ET9U8 * const pbDst) { ET9Assert(pLingInfo); ET9Assert(pbDst); ET9Assert(dwNumberOfBytesToRead); #ifdef ET9_DIRECT_LDB_ACCESS #ifdef ET9_DEBUGLOG7 if (dwOffset >= ALDBL.dwLdbDataSize || dwOffset + dwNumberOfBytesToRead > ALDBL.dwLdbDataSize) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; WLOG7(fprintf(pLogFile7, "__ET9ReadLDBData, dwOffset %u, dwNumberOfBytesToRead %u, dwLdbDataSize %u, LDB %08x, LM %u, ALM %u\n", dwOffset, dwNumberOfBytesToRead, ALDBL.dwLdbDataSize, pLingCmnInfo->dwLdbNum, (int)pLingCmnInfo->Private.ALdbLM.bSupported, (int)pLingCmnInfo->Private.ALdbNLM.bSupported);) } #endif /* ET9_DEBUGLOG7 */ ET9Assert(dwOffset < ALDBL.dwLdbDataSize); ET9Assert(dwOffset + dwNumberOfBytesToRead <= ALDBL.dwLdbDataSize); _ET9ByteCopy(pbDst, &ALDBL.pLdbData[dwOffset], dwNumberOfBytesToRead); #else { ET9U32 dwNumberOfBytesRead; ET9Assert(pLingInfo->ET9AWLdbReadData != NULL); if (pLingInfo->ET9AWLdbReadData(pLingInfo, dwOffset, dwNumberOfBytesToRead, pbDst, &dwNumberOfBytesRead) || (dwNumberOfBytesRead != dwNumberOfBytesToRead)) { return ET9STATUS_READ_DB_FAIL; } } #endif /* ET9_SINGLE_LDB_READ_BUFFER */ return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ ET9INLINE static ET9STATUS ET9LOCALCALL __ET9ReadLDBWord(ET9AWLingInfo * const pLingInfo, const ET9U32 dwOffset, ET9U16 * const pwData) { ET9U8 byWord[2]; ET9Assert(pLingInfo); ET9Assert(pwData); if (__ET9ReadLDBData(pLingInfo, dwOffset, 2, byWord)) { return ET9STATUS_READ_DB_FAIL; } *pwData = (ET9U16) ((ET9U16)(byWord[0] << 8) | byWord[1]); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9U8 ET9FARCALL _ET9ReadLDBByte(ET9AWLingInfo * const pLingInfo, const ET9U32 dwOffset) { ET9U8 bValue; ET9Assert(pLingInfo); if (__ET9ReadLDBData(pLingInfo, dwOffset, 1, &bValue)) { bValue = 0; } return bValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9U16 ET9FARCALL _ET9ReadLDBWord2(ET9AWLingInfo * const pLingInfo, const ET9U32 dwOffset) { ET9U8 byWord[2]; ET9U16 wValue; ET9Assert(pLingInfo); if (__ET9ReadLDBData(pLingInfo, dwOffset, 2, byWord)) { wValue = 0; } else { wValue = (ET9U16)((ET9U16)(byWord[0] << 8) | byWord[1]); } return wValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9U32 ET9FARCALL _ET9ReadLDBWord3(ET9AWLingInfo * const pLingInfo, const ET9U32 dwOffset) { ET9U8 byWord[3]; ET9U32 dwValue; ET9Assert(pLingInfo); if (__ET9ReadLDBData(pLingInfo, dwOffset, 3, byWord)) { dwValue = 0; } else { dwValue = (ET9U32)((ET9U32)(byWord[0] << 16) | (ET9U32)(byWord[1] << 8) | byWord[2]); } return dwValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9FLOAT ET9LOCALCALL __ReadLDBFloat(ET9AWLingInfo * const pLingInfo, const ET9U32 dwOffset, const ET9U8 bPrecision) { ET9U8 bSign; ET9U16 wIntegerPart; ET9U16 wDecimalPart; ET9FLOAT fValue; bSign = _ET9ReadLDBByte(pLingInfo, dwOffset); wIntegerPart = _ET9ReadLDBWord2(pLingInfo, dwOffset + 1); wDecimalPart = _ET9ReadLDBWord2(pLingInfo, dwOffset + 3); fValue = (ET9FLOAT)wIntegerPart + (ET9FLOAT)wDecimalPart / _ET9pow_f(10.0f, (ET9FLOAT)bPrecision); if (bSign) { fValue *= -1.0f; } return fValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWLdbCheckOEMID (ET9AWLingInfo * const pLingInfo) { ET9STATUS wStatus; ET9U16 wOEMID = 0; ET9Assert(pLingInfo); wStatus = __ET9ReadLDBWord(pLingInfo, ET9LDBOFFSET_OEMID, &wOEMID); if (!wStatus) { /* Compare the computed OEM ID with the one stored in LDB */ wStatus = (ET9STATUS)((wOEMID == ET9OEMID) ? ET9STATUS_NONE : ET9STATUS_WRONG_OEMID); } return wStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWLdbCheckCompat (ET9AWLingInfo * const pLingInfo) { ET9STATUS wStatus; ET9U16 wLdbCompatID; ET9U16 wLdbCompatOffset; ET9Assert(pLingInfo); /* Get the compatibility ID from LDB */ wStatus = __ET9ReadLDBWord(pLingInfo, ET9LDBOFFSET_COMPATID, &wLdbCompatID); if (!wStatus) { if (wLdbCompatID < ET9COMPATIDLDBXBASEALPH) { wStatus = ET9STATUS_DB_CORE_INCOMP; } else { /* Compute LDB Compatibility index offset */ wLdbCompatOffset = (ET9U16)(wLdbCompatID - ET9COMPATIDLDBXBASEALPH); if (wLdbCompatOffset > ET9MAXCOMPATIDLDBXOFFSET) { wStatus = ET9STATUS_DB_CORE_INCOMP; } else { /* * If the compat idx from LDB is equal to the compat idx base defined in core, * the offset is 0. Therefore, they are compatible. Otherwise, compare the offset. */ if (wLdbCompatOffset) { /* Convert the offset to bit mask. */ wLdbCompatOffset = ET9MASKOFFSET(wLdbCompatOffset); if (!(ET9COMPATIDLDBXOFFSETALPH & wLdbCompatOffset)) { wStatus = ET9STATUS_DB_CORE_INCOMP; } } } } } return wStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * * * */ static void ET9LOCALCALL __ET9AWLdbHashChunk(ET9AWLingInfo * const pLingInfo, ET9U32 * const pdwHashValue, ET9U32 dwPos, ET9U32 dwSize) { #ifdef ET9_DIRECT_LDB_ACCESS ET9U8 *pStr; ET9U32 dwCount; ET9U32 dwHashValue; ET9Assert(pLingInfo); ET9Assert(pdwHashValue); ET9Assert(dwSize); ET9Assert(ALDBL.dwLdbDataSize); ET9Assert(ALDBL.pLdbData != NULL); dwCount = dwSize; if (dwCount >= ALDBL.dwLdbDataSize - dwPos) { dwCount = ALDBL.dwLdbDataSize - dwPos; } /* hash data chunk */ pStr = &ALDBL.pLdbData[dwPos]; dwHashValue = *pdwHashValue; while (dwCount--) { dwHashValue = *(pStr++) + (65599 * dwHashValue); } *pdwHashValue = dwHashValue; #else /* ET9_DIRECT_LDB_ACCESS */ ET9STATUS wStatus; ET9U8 *pStr; ET9U32 dwNum; ET9U32 dwHashValue; ET9U32 dwReadSize; ET9U32 dwNumberOfBytesRead; ET9U8 byHashLDBBuff[ET9HASHLDBBUFFSIZE]; ET9Assert(pLingInfo); ET9Assert(pdwHashValue); ET9Assert(dwSize); while (dwSize) { if (ET9HASHLDBBUFFSIZE > dwSize) { dwReadSize = dwSize; } else { dwReadSize = ET9HASHLDBBUFFSIZE; } wStatus = pLingInfo->ET9AWLdbReadData(pLingInfo, dwPos, dwReadSize, byHashLDBBuff, &dwNumberOfBytesRead); /* Check for end-of-file (or other failure). */ if (wStatus || !dwNumberOfBytesRead) { break; } /* Hash LDB data */ pStr = byHashLDBBuff; dwNum = dwNumberOfBytesRead; dwHashValue = *pdwHashValue; while (dwNum--) { dwHashValue = *(pStr++) + (65599 * dwHashValue); } *pdwHashValue = dwHashValue; dwSize -= dwNumberOfBytesRead; dwPos += dwNumberOfBytesRead; } #endif /* ET9_DIRECT_LDB_ACCESS */ } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U16 ET9LOCALCALL __ET9ExtendedSymbolToCode(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9SYMB sChar) { ET9U16 wIndex; ET9SYMB *psChar; for (wIndex = ALDB.header.wCharacterDecodeCount, psChar = ALDB.header.psCharacterDecodeTable; wIndex; --wIndex, ++psChar) { if (*psChar == sChar) { return (ET9U16)(ALDB.header.wCharacterDecodeCount - wIndex); } } return ALDB_CHAR_CODE_NONE; } /* ************************************************************************* *********************** COMPARE ***************************************** ************************************************************************* */ /*---------------------------------------------------------------------------*/ /* -IDR- * */ #define __SetCodeIsFree(wCode) \ __ET9BitSet(ALDB.compare.pbCodeIsFree, wCode) /*---------------------------------------------------------------------------*/ /* -IDR- * */ #define __IsCodeFree(wCode) \ (__ET9BitCheck(ALDB.compare.pbCodeIsFree, wCode) != 0) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static ET9U16 ET9LOCALCALL __SymbolToCode(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9SYMB sSymb) { if (sSymb < ALDB_HEADER_MAX_DIRECT_ENCODE) { return ALDB.header.pwCharacterEncodeTable[sSymb]; } else if (!ALDB.header.wCharacterEncodeExtendCount || sSymb < ALDB.header.sCharacterEncodeExtendFirstChar || sSymb > ALDB.header.sCharacterEncodeExtendLastChar) { return ALDB_CHAR_CODE_NONE; } else { return __ET9ExtendedSymbolToCode(pLingCmnInfo, sSymb); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ #define __GetPosLock(wPos) \ (ALDB.compare.pbLocked[wPos]) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ #define __SetPosLock(wPos, bLock) \ { \ if (wPos < ALDB_COMPARE_MAX_POS) { \ ALDB.compare.pbLocked[wPos] = bLock; \ } \ WLOG3(fprintf(pLogFile3, "__SetPosLock, wPos = %2d, bLock = %s\n", \ wPos, \ (bLock == ET9_LOCKED_SYMB_NONE ? "NONE" : \ bLock == ET9_LOCKED_SYMB_VALUE ? "LOCKED_SYMB_VALUE" : \ bLock == ET9_LOCKED_SYMB_VALUE_AND_POS ? "LOCKED_SYMB_VALUE_AND_POS" : \ "\?\?\?"));) \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ #define __IsCodeActive(wPos, wCode) \ __ET9BitCheck(ALDB.compare.ppbActive[wPos], wCode) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ #define __GetActiveSectionSpc(bLengthIndex) \ (&ASPC.u.sCmpData.pppbActiveSpc[bLengthIndex]) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ #define __GetActiveSectionFlexSpc(bIndexKind) \ (&ASPC.u.sCmpDataFlex.pppbActiveSpc[bIndexKind]) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ #define __IsCodeActiveSpc(ppbActiveSpc, wPos, wCode) \ __ET9BitCheck((*ppbActiveSpc)[wPos], wCode) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ #define __IsCodeExact(wPos, wCode) \ __ET9BitCheck(ALDB.compare.ppbExact[wPos], wCode) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ #define __IsCodeLimited(wPos, wCode) \ (ET9BOOL)__ET9BitCheck(ALDB.compare.ppbLimited[wPos], wCode) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * */ #define __ActivateSymbol(wPos, wSymbol, bSymFreq, bSymDist, bExact, bLimited) \ { \ if (wPos < ALDB_COMPARE_MAX_POS) { \ const ET9U16 wCode = (ET9U16)__SymbolToCode(pLingCmnInfo, wSymbol); \ if (wCode != ALDB_CHAR_CODE_NONE) { \ __ET9BitSet(ALDB.compare.ppbActive[wPos], wCode); \ if (bExact) { \ __ET9BitSet(ALDB.compare.ppbExact[wPos], wCode); \ } \ if (bLimited) { \ __ET9BitSet(ALDB.compare.ppbLimited[wPos], wCode); \ } \ if (wPos < ALDB_COMPARE_MAX_FREQ_POS) { \ ALDB.compare.ppbFreq[wPos][wCode] = bSymFreq; \ } \ if (wPos < ALDB_COMPARE_MAX_DIST_POS) { \ ALDB.compare.ppbDist[wPos][wCode] = bSymDist; \ } \ } \ } \ WLOG3(fprintf(pLogFile3, \ "__ActivateSymbol, wPos = %2d, wSymbol = %4x, bSymFreq = %3d, bSymDist = %3d, bExact = %d\n", \ wPos, wSymbol, bSymFreq, bSymDist, bExact);) \ ET9Assert((wSymbol || wPos == ALDB.compare.wLength || pLingCmnInfo->Private.ALdbMGD.bSupported) && bSymFreq); \ } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ #define __DeactivateCode(wPos, wCode) \ { \ if (wPos < ALDB_COMPARE_MAX_POS) { \ if (wCode != ALDB_CHAR_CODE_NONE) { \ __ET9BitClear(ALDB.compare.ppbActive[wPos], wCode); \ __ET9BitClear(ALDB.compare.ppbExact[wPos], wCode); \ if (wPos < ALDB_COMPARE_MAX_FREQ_POS) { \ ALDB.compare.ppbFreq[wPos][wCode] = 0; \ } \ } \ } \ WLOG3(fprintf(pLogFile3, "__DeactivateSymbol, wPos = %2d, wCode = %d\n", wPos, wCode);) \ } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ #define __GetCodeFreq(wPos, wCode) \ ( \ (wPos < ALDB_COMPARE_MAX_FREQ_POS) \ ? \ (ALDB.compare.ppbFreq[wPos][wCode]) \ : \ (__IsCodeActive(wPos, wCode) ? 1 : 0) \ ) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ #define __GetSymbDistance(wPos, wCode) \ ( \ (wPos < ALDB_COMPARE_MAX_DIST_POS) \ ? \ (ALDB.compare.ppbDist[wPos][wCode]) \ : \ 0 \ ) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9STATUS ET9LOCALCALL __AssureActiveLDB(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum) { if (!ET9LDBENABLED(pLingInfo->pLingCmnInfo)) { return ET9STATUS_NO_INIT; } if (dwLdbNum != pLingInfo->pLingCmnInfo->dwLdbNum || pLingInfo->Private.wLDBInitOK != ET9GOODSETUP) { return _ET9AWLdb_SetActiveLanguage(pLingInfo, dwLdbNum); } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9BOOL ET9FARCALL _ET9AWLdbIsSymbolUsed (ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9SYMB sSymb) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; /* pretend that digits always are in... */ if (sSymb >= '0' && sSymb <= '9') { return 1; } /* assure LDB cache */ if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { if (sSymb >= 'a' && sSymb <= 'z') { return 1; } else { return 0; } } /* actual LDB check */ { const ET9U16 wCode = __SymbolToCode(pLingCmnInfo, sSymb); ET9BOOL bResult = (wCode == ALDB_CHAR_CODE_NONE) ? 0 : 1; return bResult; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * * * * */ static void ET9LOCALCALL __ET9CompareActivatePos (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U16 wIndex, ET9U8 * const pbLocked, const ET9U16 wPos) { ET9U8 bBaseIndex; ET9SymbInfo * const pSymbInfo = &pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo[wIndex]; WLOG3(fprintf(pLogFile3, "__ET9CompareActivatePos, wIndex = %2d, wPos = %2d\n", wIndex, wPos);) *pbLocked |= pSymbInfo->bLocked; if (*pbLocked) { const ET9U8 bSymFreq = 1; const ET9U8 bSymDist = 0; const ET9U8 bExact = 1; const ET9SYMB sLockedSymb = pSymbInfo->sLockedSymb; const ET9BOOL bLimited = 0; __SetPosLock(wPos, ET9_LOCKED_SYMB_VALUE_AND_POS); __ActivateSymbol(wPos, sLockedSymb, bSymFreq, bSymDist, bExact, bLimited); __ActivateSymbol(wPos, _ET9SymToOther(sLockedSymb, pLingCmnInfo->dwLdbNum), bSymFreq, bSymDist, bExact, bLimited); } else { ET9DataPerBaseSym *pDPBS; pDPBS = pSymbInfo->DataPerBaseSym; for (bBaseIndex = 0; bBaseIndex < pSymbInfo->bNumBaseSyms; ++bBaseIndex, ++pDPBS) { ET9U8 bExact; const ET9BOOL bLimited = pDPBS->bLimited; if (bLimited && pLingCmnInfo->Private.bCurrLimitedRegionality) { continue; } if (bLimited && wIndex && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[wIndex - 1] && pLingCmnInfo->Private.sBuildInfo.pbFlushPos[wIndex - 1] <= pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs && _ET9_IsPunctOrNumeric(pSymbInfo->DataPerBaseSym[0].sChar[0])) { /* disallow limited symbols for punct/numeric buildaround points after a flush */ continue; } if (pSymbInfo->eInputType == ET9DISCRETEKEY || pSymbInfo->eInputType == ET9MULTITAPKEY || pSymbInfo->eInputType == ET9CUSTOMSET || pSymbInfo->bSymbType == ET9KTSMARTPUNCT) { bExact = 1; } else { bExact = !bBaseIndex; if (bBaseIndex >= pLingCmnInfo->Private.bCurrMaxRegionality) { WLOG3(fprintf(pLogFile3, " bCurrMaxRegionality %u - skip\n", pLingCmnInfo->Private.bCurrMaxRegionality);) break; } } { ET9U8 bSymFreq; ET9U8 bSymDist; ET9U8 bSymsIndex; ET9SYMB *pLower; ET9SYMB *pUpper; bSymFreq = pDPBS->bSymFreq; bSymDist = pDPBS->bSymDist; if (!bSymFreq) { bSymFreq = 1; } pLower = pDPBS->sChar; pUpper = pDPBS->sUpperCaseChar; for (bSymsIndex = 0; bSymsIndex < pDPBS->bNumSymsToMatch; ++bSymsIndex, ++pLower, ++pUpper) { __ActivateSymbol(wPos, *pLower, bSymFreq, bSymDist, bExact, bLimited); __ActivateSymbol(wPos, *pUpper, bSymFreq, bSymDist, bExact, bLimited); } } } __SetPosLock(wPos, ET9_SPC_IS_LOCKED_POS(pSymbInfo)); } } #ifdef ET9ASPC_DEV_SCREEN /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __ET9CompareActivateQualityPos (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nSymbIndex, const ET9UINT nQualityIndex) { ET9U8 wBaseIndex; ET9DataPerBaseSym *pDPBS; ET9SymbInfo * const pSymbInfo = &pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo[nSymbIndex]; ET9ASPCFlexCompareData * const pCMP = &pLingCmnInfo->Private.ASpc.u.sCmpDataFlex; WLOG3(fprintf(pLogFile3, "__ET9CompareActivateQualityPos, nSymbIndex = %2d, nQualityIndex = %2d\n", nSymbIndex, nQualityIndex);) pDPBS = pSymbInfo->DataPerBaseSym; for (wBaseIndex = 0; wBaseIndex < pSymbInfo->bNumBaseSyms; ++wBaseIndex, ++pDPBS) { ET9U8 wSymsIndex; ET9SYMB *pLower = pDPBS->sChar; ET9SYMB *pUpper = pDPBS->sUpperCaseChar; for (wSymsIndex = 0; wSymsIndex < pDPBS->bNumSymsToMatch; ++wSymsIndex, ++pLower, ++pUpper) { { const ET9UINT nCode = __SymbolToCode(pLingCmnInfo, *pLower); if (nCode != ALDB_CHAR_CODE_NONE) { ET9ASPCSupportRelations * const pRelations = &pCMP->pCharCodeRelations[nCode]; if (!pRelations->bCount || pRelations->pbIndex[pRelations->bCount - 1] != nQualityIndex) { WLOG3(fprintf(pLogFile3, " + nCode %2u (%c)\n", nCode, (char)ALDB.header.psCharacterDecodeTable[nCode]);) pRelations->pbIndex[pRelations->bCount++] = (ET9U8)nQualityIndex; } else { WLOG3(fprintf(pLogFile3, " x nCode %2u (%c)\n", nCode, (char)ALDB.header.psCharacterDecodeTable[nCode]);) } } } { const ET9UINT nCode = __SymbolToCode(pLingCmnInfo, *pUpper); if (nCode != ALDB_CHAR_CODE_NONE) { ET9ASPCSupportRelations * const pRelations = &pCMP->pCharCodeRelations[nCode]; if (!pRelations->bCount || pRelations->pbIndex[pRelations->bCount - 1] != nQualityIndex) { WLOG3(fprintf(pLogFile3, " + nCode %2u (%c)\n", nCode, (char)ALDB.header.psCharacterDecodeTable[nCode]);) pRelations->pbIndex[pRelations->bCount++] = (ET9U8)nQualityIndex; } else { WLOG3(fprintf(pLogFile3, " x nCode %2u (%c)\n", nCode, (char)ALDB.header.psCharacterDecodeTable[nCode]);) } } } } } } #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * / * * */ static void ET9LOCALCALL __ET9CompareActivateList (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U16 wIndex, const ET9U16 wLength) { ET9U8 bLocked = 0; ET9U16 wCmpIndex; ET9Assert(pLingCmnInfo != NULL); WLOG3(fprintf(pLogFile3, "__ET9CompareActivateList, wIndex = %2d, wLength = %2d\n", wIndex, wLength);) if (!wLength) { return; } for (wCmpIndex = (ET9U16)(wLength - 1); ; --wCmpIndex) { __ET9CompareActivatePos(pLingCmnInfo, (ET9U16)(wCmpIndex + wIndex), &bLocked, wCmpIndex); if (!wCmpIndex) { break; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ static void ET9LOCALCALL __ET9CompareActivateSpcInfoStd (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U8 bSpcMode, ET9BOOL * const pbExactCompare) { const ET9ASPCSEARCHFILTER eSearchFilter = _ET9AWGetSpcSearchFilter(pLingCmnInfo); const ET9BOOL bExactSpcMode = (ET9U8)(ET9_SPC_GET_MODE(bSpcMode) == ET9ASPCMODE_EXACT); const ET9BOOL bSpcFilterOne = ET9_SPC_SEARCH_FILTER_IS_ONE(eSearchFilter); const ET9BOOL bSpcFilterTwo = ET9_SPC_SEARCH_FILTER_IS_TWO(eSearchFilter); const ET9BOOL bSearchFilterExact = ET9_SPC_SEARCH_FILTER_IS_EXACT(eSearchFilter); const ET9U16 wInputLength = ALDB.compare.wLength; const ET9S16 swMaxDist = (ET9S16)pLingCmnInfo->Private.bCurrMaxEditDistance; const ET9U16 wDistLengths = swMaxDist * 2 + 1; ET9U16 wLengthIndex; ET9BOOL bExactIsActive = 0; if (!ALDB.compare.bSpcActive) { *pbExactCompare = bExactIsActive; return; } ALDB.compare.bSpcMaxEdits = (ET9U8)swMaxDist; ALDB.compare.bSpcLengthOffset = (ET9U8)pLingCmnInfo->Private.wCurrMinSourceLength; ALDB.compare.bPosLo = (ET9U8)(ALDB.compare.wLength - swMaxDist); ALDB.compare.bPosHi = (ET9U8)(ALDB.compare.wLength + swMaxDist - 1); if (ALDB.compare.bPosHi >= ALDB.header.bPosCount) { ALDB.compare.bPosHi = (ET9U8)(ALDB.header.bPosCount - 1); } WLOG3(fprintf(pLogFile3, "__ET9CompareActivateSpcPosInfo, wInputLength = %d, wDistLengths = %d\n", wInputLength, wDistLengths);) _ET9ClearMem((ET9U8*)ASPC.u.sCmpData.pppbActiveSpc, sizeof(ASPC.u.sCmpData.pppbActiveSpc)); for (wLengthIndex = 0; wLengthIndex < wDistLengths; ++wLengthIndex) { ET9S16 swCmpLen; ET9S16 swStartPos; ET9S16 swCmpWidth; ET9S16 swBaseWidth; ET9S16 swCurrStartPos; ET9S16 swIndex; ET9S16 swPosIndex; swBaseWidth = (wLengthIndex + 1) / 2; swStartPos = - swBaseWidth; swCmpWidth = (wLengthIndex % 2) + swMaxDist + 1; swCurrStartPos = swStartPos; swCmpLen = wInputLength + wLengthIndex - swMaxDist; if (swCmpLen > ET9MAXLDBWORDSIZE) { swCmpLen = ET9MAXLDBWORDSIZE; } /* don't futz with array pointers */ for (swPosIndex = 0; swPosIndex < swCmpLen; ++swPosIndex, ++swCurrStartPos) { for (swIndex = 0; swIndex < swCmpWidth; ++swIndex) { ET9S16 swPos = swCurrStartPos + swIndex; if (__GetPosLock(swPosIndex) == ET9_LOCKED_SYMB_VALUE_AND_POS && swPos != swPosIndex) { continue; } else if (swPos < 0) { continue; } else if (swPos >= wInputLength) { break; } else if (!swPosIndex && bSpcFilterOne && swPos > 0 && !wLengthIndex) { break; } else if (!swPosIndex && bSpcFilterTwo && swPos > 1 && !wLengthIndex) { break; } else if (((swMaxDist == 2 && wLengthIndex == 1) || (swMaxDist == 3 && (wLengthIndex == 1 || wLengthIndex == 3))) && swPosIndex + 1 == swCmpLen && !swIndex) { continue; } else if (((swMaxDist == 2 && wLengthIndex == 3) || (swMaxDist == 3 && (wLengthIndex == 3 || wLengthIndex == 5))) && swPos + 1 == wInputLength && swIndex + 1 == swCmpWidth) { continue; } { ET9U16 wCount; ET9BOOL bAddExact; ET9U8 *pbExact = ALDB.compare.ppbExact[swPos]; ET9U8 *pbActive = ALDB.compare.ppbActive[swPos]; ET9U8 *pbActiveSpc = ASPC.u.sCmpData.pppbActiveSpc[wLengthIndex][swPosIndex]; ET9Assert(wLengthIndex <= 7); ET9Assert(swPos >= 0 && swPos < wInputLength); ET9Assert(swPosIndex >= 0 && swPosIndex < swCmpLen); if (!swPosIndex && !wLengthIndex && (bSpcFilterOne || bSpcFilterTwo)) { bAddExact = bSearchFilterExact; } else { bAddExact = bExactSpcMode; } for (wCount = ALDB_COMPARE_MAX_CODE_BYTES; wCount; --wCount, ++pbExact, ++pbActive, ++pbActiveSpc) { if (bAddExact) { *pbActiveSpc |= *pbExact; if (*pbExact != *pbActive) { bExactIsActive = 1; } } else { *pbActiveSpc |= *pbActive; } } } } } WLOG3({ ET9U16 wCodeIndex; fprintf(pLogFile3, "wLengthIndex = %d (bExactIsActive = %d)\n", wLengthIndex, bExactIsActive); for (swPosIndex = 0; swPosIndex < swCmpLen; ++swPosIndex) { fprintf(pLogFile3, "..pos %02d: ", swPosIndex); for (wCodeIndex = 0; wCodeIndex < ALDB.header.wCharacterDecodeCount; ++wCodeIndex) { if (__IsCodeActiveSpc(__GetActiveSectionSpc(wLengthIndex), swPosIndex, wCodeIndex)) { ET9SYMB sSymb = ALDB.header.psCharacterDecodeTable[wCodeIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F) { fprintf(pLogFile3, "%c", (char)sSymb); } else { fprintf(pLogFile3, "<%x>", sSymb); } } } fprintf(pLogFile3, "\n"); } }) } *pbExactCompare = bExactIsActive; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * * */ static void ET9LOCALCALL __ET9CompareActivateSpcInfoFlex (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9UINT nInputOffset) { const ET9U16 wPos = 0; ET9ALdbSymbPosInfo * const pRegSection = __GetActiveSectionFlexSpc(0); ET9ALdbSymbPosInfo * const pExtSection = __GetActiveSectionFlexSpc(1); ET9U16 wCode; _ET9ClearMem((ET9U8*)ASPC.u.sCmpDataFlex.pppbActiveSpc, sizeof(ASPC.u.sCmpDataFlex.pppbActiveSpc)); _ET9ByteCopy((ET9U8*)pRegSection, (ET9U8*)ALDB.compare.ppbActive, sizeof(ET9ALdbSymbPosInfo)); _ET9ByteCopy((ET9U8*)pExtSection, (ET9U8*)ALDB.compare.ppbExact, sizeof(ET9ALdbSymbPosInfo)); for (wCode = 0; wCode < ALDB.header.wCharacterDecodeCount; ++wCode) { if (__IsCodeFree(wCode)) { __ET9BitSet((*pRegSection)[wPos], wCode); __ET9BitSet((*pExtSection)[wPos], wCode); } } #ifdef ET9ASPC_DEV_SCREEN { ET9ASPCFlexCompareData * const pCMP = &pLingCmnInfo->Private.ASpc.u.sCmpDataFlex; const ET9UINT nInputLength = ALDB.compare.wLength; ET9U8 * const pbIsQualityKey = pCMP->pbIsQualityKey; const ET9UINT nActiveQualityCount = pCMP->nQualityCount - 1; ET9UINT nIndex; ET9UINT nActiveIndex; ET9Assert(pCMP->nQualityCount); pCMP->nActiveQualityCount = nActiveQualityCount; pCMP->nSupportedQualityCount = 0; for (nIndex = 0; nIndex < ALDB_HEADER_MAX_CHAR_CODES; ++nIndex) { pCMP->pbCharCounters[nIndex] = 0; pCMP->pCharCodeRelations[nIndex].bCount = 0; } nActiveIndex = 0; for (nIndex = 1; nIndex < nInputLength; ++nIndex) { if (!pbIsQualityKey[nIndex + 1]) { continue; } pCMP->pbQualityCounters[nActiveIndex] = 0; __ET9CompareActivateQualityPos(pLingCmnInfo, nIndex + nInputOffset, nActiveIndex); ++nActiveIndex; } ET9Assert(nActiveIndex == nActiveQualityCount); } #else ET9_UNUSED(nInputOffset); #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * / * * * */ static void ET9LOCALCALL __ET9CompareStart (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U16 wIndex, const ET9U16 wLength, const ET9BOOL bUsingFlex) { const ET9U8 bSpcMode = pLingCmnInfo->Private.bCurrSpcMode; const ET9U16 wMaxWordLength = __ET9Min(pLingCmnInfo->Private.wMaxWordLength, ET9MAXLDBWORDSIZE); const ET9U16 wUsefulLength = (wLength <= ALDB_COMPARE_MAX_POS) ? wLength : ALDB_COMPARE_MAX_POS; const ET9ASPCSEARCHFILTER eSearchFilter = _ET9AWGetSpcSearchFilter(pLingCmnInfo); ET9Assert(pLingCmnInfo != NULL); WLOG3(fprintf(pLogFile3, "__ET9CompareStart, wIndex = %2d, wLength = %2d, wUsefulLength = %2d, bSpcMode = %d\n", wIndex, wLength, wUsefulLength, bSpcMode);) ALDB.compare.wLength = wUsefulLength; ALDB.compare.bSpcActive = ET9_SPC_IS_ACTIVE(bSpcMode); ALDB.compare.bFirstPosSetOpt = (wMaxWordLength == 1) ? 1 : 0; ALDB.compare.bSpcFilteredCompare = ALDB.compare.bSpcActive && !ET9_SPC_SEARCH_FILTER_IS_UNFILTERED(eSearchFilter); ALDB.compare.bSpcExactFilter = ALDB.compare.bSpcActive && ET9_SPC_SEARCH_FILTER_IS_EXACT(eSearchFilter); ALDB.compare.bSpcExactFilterTrace = ALDB.compare.bSpcActive && ET9_SPC_SEARCH_FILTER_IS_EXACT(ASPC.eSearchFilterTrace); ALDB.compare.bSpcExactPrunedTrace = 0; ALDB.compare.pppbSpcFlexSection = __GetActiveSectionFlexSpc(0); WLOG3(fprintf(pLogFile3, "..wMaxWordLength = %2d, bFirstPosSetOpt = %d\n", wMaxWordLength, ALDB.compare.bFirstPosSetOpt);) /* clear all compare info, leave completion positions "matching" (to prevent length checking) */ _ET9ClearMem((ET9U8*)ALDB.compare.ppbActive, __ET9ByteDistance(&ALDB.compare.ppbActive[wUsefulLength], ALDB.compare.ppbActive)); if (wUsefulLength < ALDB_COMPARE_MAX_POS) { _ET9ByteSet((ET9U8*)&ALDB.compare.ppbActive[wUsefulLength], __ET9ByteDistance(&ALDB.compare.ppbActive[ALDB_COMPARE_MAX_POS], &ALDB.compare.ppbActive[wUsefulLength]), 0xFF ); } _ET9ClearMem((ET9U8*)ALDB.compare.ppbExact, sizeof(ALDB.compare.ppbExact)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbLimited, sizeof(ALDB.compare.ppbLimited)); _ET9ClearMem((ET9U8*)ALDB.compare.pbLocked, sizeof(ALDB.compare.pbLocked)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbFreq, sizeof(ALDB.compare.ppbFreq)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbDist, sizeof(ALDB.compare.ppbDist)); __ET9CompareActivateList(pLingCmnInfo, wIndex, wUsefulLength); ALDB.compare.bActiveCmpLength = (ET9U8)wUsefulLength; if (bUsingFlex) { __ET9CompareActivateSpcInfoFlex(pLingCmnInfo, wIndex); } else { __ET9CompareActivateSpcInfoStd(pLingCmnInfo, bSpcMode, &ALDB.compare.bSpcExactCompare); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9CompareStartWord (ET9AWLingInfo * const pLingInfo, const ET9SYMB * const psChars, const ET9U16 wLength, const ET9BOOL bPartialMatchOK, const ET9BOOL bReverse) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U16 wIndex; const ET9U8 bSymFreq = 1; /* for __ActivateSymbol readability and warnings. */ const ET9U8 bSymDist = 0; /* for __ActivateSymbol readability and warnings. */ const ET9U8 bExact = 1; /* for __ActivateSymbol readability and warnings. */ const ET9U8 bLimited = 0; /* for __ActivateSymbol readability and warnings. */ ET9Assert(pLingInfo != NULL); ET9Assert(psChars); WLOG3(fprintf(pLogFile3, "__ET9CompareStartWord\n");) /* setup things first - needed if aborting early */ ALDB.compare.wLength = wLength; if (ALDB.compare.wLength > ET9MAXLDBWORDSIZE) { ALDB.compare.wLength = ET9MAXLDBWORDSIZE; } ALDB.compare.bSpcActive = 0; ALDB.compare.bFirstPosSetOpt = 0; ALDB.compare.bActiveCmpLength = (ET9U8)ALDB.compare.wLength; /* Extend our compare to include the trailing null if we're not already at max word length. */ if (ALDB.compare.wLength < ALDB.header.bPosCount && wLength) { ++ALDB.compare.bActiveCmpLength; } /* valid compare? */ if (wLength > ALDB.header.bPosCount) { WLOG3(fprintf(pLogFile3, "__ET9CompareStartWord, too long word!\n");) return ET9STATUS_ERROR; } /* init compare info */ _ET9ClearMem((ET9U8*)ALDB.compare.ppbActive, __ET9ByteDistance(&ALDB.compare.ppbActive[ALDB.compare.bActiveCmpLength], ALDB.compare.ppbActive)); if (ALDB.compare.bActiveCmpLength < ALDB_COMPARE_MAX_POS) { _ET9ByteSet((ET9U8*)&ALDB.compare.ppbActive[ALDB.compare.bActiveCmpLength], __ET9ByteDistance(&ALDB.compare.ppbActive[ALDB_COMPARE_MAX_POS], &ALDB.compare.ppbActive[ALDB.compare.bActiveCmpLength]), 0xFF ); } _ET9ClearMem((ET9U8*)ALDB.compare.ppbExact, sizeof(ALDB.compare.ppbExact)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbLimited, sizeof(ALDB.compare.ppbLimited)); _ET9ClearMem((ET9U8*)ALDB.compare.pbLocked, sizeof(ALDB.compare.pbLocked)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbFreq, sizeof(ALDB.compare.ppbFreq)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbDist, sizeof(ALDB.compare.ppbDist)); /* Note the symbols */ for (wIndex = 0; wIndex < wLength; ++wIndex) { const ET9SYMB sSymb = psChars[bReverse ? wLength - wIndex - 1 : wIndex]; if (pLingInfo->Private.pConvertSymb != NULL) { ET9SYMB sSymbOther = _ET9SymToOther(sSymb, pLingCmnInfo->dwLdbNum); ET9U16 wDecodeCount = ALDB.header.wCharacterDecodeCount; ET9SYMB *pDecodeSymb = ALDB.header.psCharacterDecodeTable; WLOG3(fprintf(pLogFile3, "__ET9CompareStartWord, pos %02d, word symb %04x (%04x)\n", wIndex, sSymb, sSymbOther);) for (; wDecodeCount; --wDecodeCount, ++pDecodeSymb) { ET9STATUS wStatus; ET9SYMB sConvertSymb = *pDecodeSymb; if (__IsPrivateSymbol(sConvertSymb)) { continue; } wStatus = pLingInfo->Private.pConvertSymb(pLingInfo->Private.pConvertSymbInfo, &sConvertSymb); if (!wStatus && (sConvertSymb == sSymb || sConvertSymb == sSymbOther)) { __ActivateSymbol(wIndex, *pDecodeSymb, bSymFreq, bSymDist, bExact, bLimited); __ActivateSymbol(wIndex, _ET9SymToOther(*pDecodeSymb, pLingCmnInfo->dwLdbNum), bSymFreq, bSymDist, bExact, bLimited); if (wIndex > 0 && bPartialMatchOK) { __ActivateSymbol(wIndex, 0, bSymFreq, bSymDist, bExact, bLimited); } } } } else { __ActivateSymbol(wIndex, sSymb, bSymFreq, bSymFreq, bExact, bLimited); __ActivateSymbol(wIndex, _ET9SymToOther(sSymb, pLingCmnInfo->dwLdbNum), bSymFreq, bSymDist, bExact, bLimited); if (wIndex > 0 && bPartialMatchOK) { __ActivateSymbol(wIndex, 0, bSymFreq, bSymDist, bExact, bLimited); } } } /* termination */ if (ALDB.compare.wLength < ALDB.header.bPosCount && wLength) { __ActivateSymbol(ALDB.compare.wLength, 0, bSymFreq, bSymDist, bExact, bLimited); } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9CompareStartAmbigWord (ET9AWLingInfo * const pLingInfo, const ET9SYMB * const psChars, const ET9U16 wStartSymbIndex, const ET9U16 wLength, const ET9BOOL bPartialMatchOK, const ET9BOOL bReverse) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U16 wIndex; const ET9U8 bSymFreq = 1; /* for __ActivateSymbol readability and warnings. */ const ET9U8 bSymDist = 0; /* for __ActivateSymbol readability and warnings. */ const ET9U8 bExact = 1; /* for __ActivateSymbol readability and warnings. */ const ET9U8 bLimited = 0; /* for __ActivateSymbol readability and warnings. */ ET9Assert(pLingInfo != NULL); ET9Assert(psChars); WLOG3(fprintf(pLogFile3, "__ET9CompareStartWord\n");) /* setup things first - needed if aborting early */ ALDB.compare.wLength = wLength; if (ALDB.compare.wLength > ET9MAXLDBWORDSIZE) { ALDB.compare.wLength = ET9MAXLDBWORDSIZE; } ALDB.compare.bSpcActive = 0; ALDB.compare.bFirstPosSetOpt = 0; ALDB.compare.bActiveCmpLength = (ET9U8)ALDB.compare.wLength; /* Extend our compare to include the trailing null if we're not already at max word length. */ if (ALDB.compare.wLength < ALDB.header.bPosCount && wLength) { ++ALDB.compare.bActiveCmpLength; } /* valid compare? */ if (wLength > ALDB.header.bPosCount) { WLOG3(fprintf(pLogFile3, "__ET9CompareStartWord, too long word!\n");) return ET9STATUS_ERROR; } /* init compare info */ _ET9ClearMem((ET9U8*)ALDB.compare.ppbActive, __ET9ByteDistance(&ALDB.compare.ppbActive[ALDB.compare.bActiveCmpLength], ALDB.compare.ppbActive)); if (ALDB.compare.bActiveCmpLength < ALDB_COMPARE_MAX_POS) { _ET9ByteSet((ET9U8*)&ALDB.compare.ppbActive[ALDB.compare.bActiveCmpLength], __ET9ByteDistance(&ALDB.compare.ppbActive[ALDB_COMPARE_MAX_POS], &ALDB.compare.ppbActive[ALDB.compare.bActiveCmpLength]), 0xFF ); } _ET9ClearMem((ET9U8*)ALDB.compare.ppbExact, sizeof(ALDB.compare.ppbExact)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbLimited, sizeof(ALDB.compare.ppbLimited)); _ET9ClearMem((ET9U8*)ALDB.compare.pbLocked, sizeof(ALDB.compare.pbLocked)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbFreq, sizeof(ALDB.compare.ppbFreq)); _ET9ClearMem((ET9U8*)ALDB.compare.ppbDist, sizeof(ALDB.compare.ppbDist)); /* Note the symbols */ for (wIndex = 0; wIndex < wLength; ++wIndex) { const ET9SYMB sSymb = psChars[bReverse ? wLength - wIndex - 1 : wIndex]; if (pLingInfo->Private.pConvertSymb != NULL) { ET9SYMB sSymbOther = _ET9SymToOther(sSymb, pLingCmnInfo->dwLdbNum); ET9U16 wDecodeCount = ALDB.header.wCharacterDecodeCount; ET9SYMB *pDecodeSymb = ALDB.header.psCharacterDecodeTable; WLOG3(fprintf(pLogFile3, "__ET9CompareStartWord, pos %02d, word symb %04x (%04x)\n", wIndex, sSymb, sSymbOther);) for (; wDecodeCount; --wDecodeCount, ++pDecodeSymb) { ET9STATUS wStatus; ET9SYMB sConvertSymb = *pDecodeSymb; if (__IsPrivateSymbol(sConvertSymb)) { continue; } wStatus = pLingInfo->Private.pConvertSymb(pLingInfo->Private.pConvertSymbInfo, &sConvertSymb); if (!wStatus && (sConvertSymb == sSymb || sConvertSymb == sSymbOther)) { __ActivateSymbol(wIndex, *pDecodeSymb, bSymFreq, bSymDist, bExact, bLimited); __ActivateSymbol(wIndex, _ET9SymToOther(*pDecodeSymb, pLingCmnInfo->dwLdbNum), bSymFreq, bSymDist, bExact, bLimited); if (wIndex > 0 && bPartialMatchOK) { __ActivateSymbol(wIndex, 0, bSymFreq, bSymDist, bExact, bLimited); } } } } else { #if 1 { ET9WordSymbInfo const * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; const ET9INT nNumBaseSymbs = pWordSymbInfo->SymbsInfo[wStartSymbIndex+wIndex].bNumBaseSyms; ET9INT nBaseSymbIndex; for (nBaseSymbIndex = 0; nBaseSymbIndex < nNumBaseSymbs; ++nBaseSymbIndex) { ET9SymbInfo const * const pSymbsInfo = &pWordSymbInfo->SymbsInfo[wStartSymbIndex+wIndex]; const ET9INT nNumAltSymbs = pSymbsInfo->DataPerBaseSym[nBaseSymbIndex].bNumSymsToMatch; ET9INT nAltSymbIndex; for(nAltSymbIndex = 0; nAltSymbIndex < nNumAltSymbs; ++nAltSymbIndex) { const ET9SYMB sSymb = pSymbsInfo->DataPerBaseSym[nBaseSymbIndex].sChar[nAltSymbIndex]; __ActivateSymbol(wIndex, sSymb, bSymFreq, bSymDist, bExact, bLimited); __ActivateSymbol(wIndex, _ET9SymToOther(sSymb, pLingCmnInfo->dwLdbNum), bSymFreq, bSymDist, bExact, bLimited); } } } #else __ActivateSymbol(wIndex, sSymb, bSymFreq, bSymDist, bExact, bLimited); __ActivateSymbol(wIndex, _ET9SymToOther(sSymb, pLingCmnInfo->dwLdbNum), bSymFreq, bSymDist, bExact, bLimited); if (wIndex > 0 && bPartialMatchOK) { __ActivateSymbol(wIndex, 0, bSymFreq, bSymDist, bExact, bLimited); } #endif } } /* termination */ if (ALDB.compare.wLength < ALDB.header.bPosCount && wLength) { __ActivateSymbol(ALDB.compare.wLength, 0, bSymFreq, bSymDist, bExact, bLimited); } return ET9STATUS_NONE; } /* ************************************************************************* *********************** SEARCH ****************************************** ************************************************************************* */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ #define __LdbGetCharCodeAtPos(wPos) (ALDB.pCursors[wPos].wCode) #ifdef ET9_DIRECT_LDB_ACCESS #define ET9FARDATA_LDB ET9FARDATA #define __InitDirectAccess() (pLingInfo->ET9AWLdbReadData(pLingInfo, &(ALDBL.pLdbData), &(ALDBL.dwLdbDataSize))) #define __LdbGetByte(pCursor) (*pbCurrData) #define __LdbGetIndex(pCursor) ((ET9U32)(pbCurrData - ALDBL.pLdbData)) #define __LdbSetIndex(pCursor,index) pbCurrData = (index) + ALDBL.pLdbData; #define __LdbIncIndex(pCursor) ++pbCurrData; #define __LdbDecIndex(pCursor) --pbCurrData; #else /* ET9_DIRECT_LDB_ACCESS */ #define ET9FARDATA_LDB #define __InitDirectAccess() (ET9STATUS_NONE) #define __LdbGetByte(pCursor) (*pbCurrData) #define __LdbGetIndex(pCursor) ((ET9U32)((pCursor)->dwCurrCacheStart + (pbCurrData - (pCursor)->pbCache))) #define __LdbUpdateCache(pCursor) \ { \ ET9U32 dwBytesRead; \ pLingInfo->ET9AWLdbReadData(pLingInfo, \ (pCursor)->dwCurrCacheStart, \ ALDB_CURSOR_DATA_CACHE_SIZE, \ (pCursor)->pbCache, \ &dwBytesRead); \ } #define __LdbSetIndex(pCursor,index) \ { \ (pCursor)->dwCurrCacheStart = (index); \ pbCurrData = (pCursor)->pbCache; __LdbUpdateCache(pCursor); \ } \ #define __LdbIncIndex(pCursor) \ { \ if (++pbCurrData == (pCursor)->pbCacheEnd) { \ __LdbSetIndex(pCursor, (pCursor)->dwCurrCacheStart + ALDB_CURSOR_DATA_CACHE_SIZE); \ }; \ } \ #define __LdbDecIndex(pCursor) \ { \ if (--pbCurrData < (pCursor)->pbCache) { \ __LdbSetIndex(pCursor, (pCursor)->dwCurrCacheStart - 1); \ }; \ } \ #endif /* ET9_DIRECT_LDB_ACCESS */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * */ #define __IsExhausted() (ALDB.search.bExhausted) /*---------------------------------------------------------------------------*/ /* -IDR- * * * * */ #define __GetWordIndex() (ALDB.search.dwCurrItem) /*---------------------------------------------------------------------------*/ /* -IDR- * */ #define __ET9HeaderConstsGetNextInterval \ const ET9U16 wCodeIntervalEnd = ALDB.header.wCodeIntervalEnd; \ const ET9U16 wCodeIntervalJump = ALDB.header.wCodeIntervalJump; \ const ET9U16 wCodeIntervalExtend = ALDB.header.wCodeIntervalExtend; \ ET9U8 * const pbOneByteCodes = ALDB.header.pbOneByteCodes; \ ET9U8 * const pbOneByteLengths = ALDB.header.pbOneByteLengths; \ ET9SYMB * const psTarget = ALDB.search.psTarget; \ ET9SYMB * const psCharacterDecodeTable = ALDB.header.psCharacterDecodeTable; \ /*---------------------------------------------------------------------------*/ /* -IDR- * * / * * * * * * * *** * * * * * * * */ #define __ET9SearchGetNextInterval(pLingInfo, pCursor, bPos, dwTargetPosIn) \ { \ register ET9U8 bCode; \ register ET9U16 wCode = 0xcccc; \ register ET9U32 dwStartPos = (pCursor)->dwStartPos; \ register ET9U32 dwEndPos = (pCursor)->dwEndPos; \ register ET9U32 dwJumpPos = (pCursor)->dwJumpPos; \ register ET9U8 ET9FARDATA_LDB *pbCurrData = (pCursor)->pbCurrData; \ \ const ET9U32 dwTargetPos = dwTargetPosIn; \ \ while (dwTargetPos >= dwEndPos || dwStartPos == dwEndPos) { \ \ /* Move start pos */ \ \ dwStartPos = dwEndPos; \ \ /* Check if jump is possible */ \ \ if (dwTargetPos >= dwJumpPos && dwStartPos + 1 < dwJumpPos) { \ \ dwStartPos = dwJumpPos; \ dwEndPos = dwStartPos; \ \ __LdbSetIndex(pCursor, (pCursor)->dwJumpAddress); \ \ continue; \ } \ \ /* Get interval code - could be single or multi encoded */ \ \ bCode = __LdbGetByte(pCursor); \ __LdbIncIndex(pCursor); \ \ /* Check if single, and possibly a double */ \ \ if (bCode != ALDB_INTERVAL_MULTI_BYTE_CODE) { \ \ wCode = pbOneByteCodes[bCode]; \ dwEndPos = dwStartPos + pbOneByteLengths[bCode]; \ \ bCode = __LdbGetByte(pCursor); \ \ if (wCode == pbOneByteCodes[bCode] && bCode != ALDB_INTERVAL_MULTI_BYTE_CODE) { \ __LdbIncIndex(pCursor); \ dwEndPos += pbOneByteLengths[bCode]; \ } \ \ continue; \ } \ \ /* Get multi code - could be jump, char with length or end */ \ \ wCode = __LdbGetByte(pCursor); \ __LdbIncIndex(pCursor); \ \ /* Check if jump info */ \ \ if (wCode == wCodeIntervalJump) { \ \ dwJumpPos = dwStartPos; \ \ dwJumpPos += ((ET9U32)__LdbGetByte(pCursor)) << 8; \ __LdbIncIndex(pCursor); \ \ dwJumpPos += __LdbGetByte(pCursor); \ __LdbIncIndex(pCursor); \ \ (pCursor)->dwJumpAddress = __LdbGetIndex(pCursor) - 4; \ \ (pCursor)->dwJumpAddress += ((ET9U32)__LdbGetByte(pCursor)) << 8; \ __LdbIncIndex(pCursor); \ \ (pCursor)->dwJumpAddress += __LdbGetByte(pCursor); \ __LdbIncIndex(pCursor); \ \ /* equal start/end implies loop again */ \ \ ET9Assert(dwStartPos == dwEndPos); \ \ continue; \ } \ \ /* Check if end */ \ \ if (wCode == wCodeIntervalEnd) { \ ALDB.search.bExhausted = 1; \ return; /* Note that since this is a macro, this returns the caller, not just this fn. */ \ } \ \ /* Direct interval... */ \ \ dwEndPos = dwStartPos + __LdbGetByte(pCursor); \ __LdbIncIndex(pCursor); \ \ /* Check length extension */ \ \ if (__LdbGetByte(pCursor) != ALDB_INTERVAL_MULTI_BYTE_CODE) { \ continue; \ } \ \ __LdbIncIndex(pCursor); \ \ if (__LdbGetByte(pCursor) != wCodeIntervalExtend) { \ \ __LdbDecIndex(pCursor); \ \ continue; \ } \ \ /* Found extension */ \ \ __LdbIncIndex(pCursor); \ \ dwEndPos += ((ET9U32)__LdbGetByte(pCursor)) << 8; \ __LdbIncIndex(pCursor); \ } \ \ /* Pick up the actual symbol as well... */ \ \ psTarget[bPos] = psCharacterDecodeTable[wCode]; \ \ /* store locals */ \ \ (pCursor)->wCode = wCode; \ (pCursor)->dwStartPos = dwStartPos; \ (pCursor)->dwEndPos = dwEndPos; \ (pCursor)->dwJumpPos = dwJumpPos; \ (pCursor)->pbCurrData = pbCurrData; \ \ ET9Assert(wCode != 0xcccc); \ ET9Assert(__IsExhausted() || dwStartPos <= dwTargetPos && dwTargetPos < dwEndPos); \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ #define __ET9SearchCatchupStream(pLingInfo, pCursor, bPos, dwTargetPosIn) \ { \ if ((pCursor)->dwEndPos <= dwTargetPosIn) { \ \ __ET9SearchGetNextInterval(pLingInfo, pCursor, bPos, dwTargetPosIn); \ } \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ #define __ET9SearchCatchupStreamCounters(pLingInfo, pCursor, bPos, dwTargetPosIn) \ { \ if ((pCursor)->dwEndPos <= dwTargetPosIn) { \ \ WLOG3(fprintf(pLogFile3, "__ET9SearchCatchupStreamCounters, bPos %2u, dwTargetPosIn %6u\n", bPos, dwTargetPosIn);) \ \ __DecCharCounter(pLingCmnInfo, pCursor->wCode); \ \ __ET9SearchGetNextInterval(pLingInfo, pCursor, bPos, dwTargetPosIn); \ \ __IncCharCounter(pLingCmnInfo, pCursor->wCode); \ } \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ #define __ET9SearchCatchupStreamWithBreak(pLingInfo, pCursor, bPos, dwTargetPosIn) \ { \ if (pCursor->dwEndPos <= dwTargetPosIn) { \ \ __ET9SearchGetNextInterval(pLingInfo, pCursor, bPos, dwTargetPosIn); \ } \ else { \ break; \ } \ } \ #ifdef ET9ASPC_DEV_SCREEN /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static void ET9LOCALCALL __IncCharCounter (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U16 wCode) { ET9ASPCFlexCompareData * const pCMP = &pLingCmnInfo->Private.ASpc.u.sCmpDataFlex; if (wCode == ALDB.header.wCodeZero) { return; } ET9Assert(wCode < ALDB_HEADER_MAX_CHAR_CODES); ET9Assert(pCMP->pbCharCounters[wCode] < ALDB.header.bPosCount); if (++pCMP->pbCharCounters[wCode] == 1) { ET9ASPCSupportRelations const * const pRelations = &pCMP->pCharCodeRelations[wCode]; ET9U8 bIndex; WLOG3(fprintf(pLogFile3, "__IncCharCounter, wCode %2u (%c) ON\n", wCode, (char)ALDB.header.psCharacterDecodeTable[wCode]);) for (bIndex = 0; bIndex < pRelations->bCount; ++bIndex) { if (++pCMP->pbQualityCounters[pRelations->pbIndex[bIndex]] == 1) { ++pCMP->nSupportedQualityCount; ET9Assert(pCMP->nSupportedQualityCount <= pCMP->nActiveQualityCount); WLOG3(fprintf(pLogFile3, " q-index %2u ON - new s-count %2u\n", pRelations->pbIndex[bIndex], pCMP->nSupportedQualityCount);) } } } } #else #define __IncCharCounter(pLingCmnInfo, wCode) #endif #ifdef ET9ASPC_DEV_SCREEN /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static void ET9LOCALCALL __DecCharCounter (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U16 wCode) { ET9ASPCFlexCompareData * const pCMP = &pLingCmnInfo->Private.ASpc.u.sCmpDataFlex; if (wCode == ALDB.header.wCodeZero) { return; } ET9Assert(wCode < ALDB_HEADER_MAX_CHAR_CODES); ET9Assert(pCMP->pbCharCounters[wCode] > 0); if (--pCMP->pbCharCounters[wCode] == 0) { ET9ASPCSupportRelations const * const pRelations = &pCMP->pCharCodeRelations[wCode]; ET9U8 bIndex; WLOG3(fprintf(pLogFile3, "__DecCharCounter, wCode %2u (%c) OFF\n", wCode, (char)ALDB.header.psCharacterDecodeTable[wCode]);) for (bIndex = 0; bIndex < pRelations->bCount; ++bIndex) { if (--pCMP->pbQualityCounters[pRelations->pbIndex[bIndex]] == 0) { --pCMP->nSupportedQualityCount; ET9Assert(pCMP->nSupportedQualityCount <= pCMP->nActiveQualityCount); WLOG3(fprintf(pLogFile3, " q-index %2u OFF - new s-count %2u\n", pRelations->pbIndex[bIndex], pCMP->nSupportedQualityCount);) } } } } #else #define __DecCharCounter(pLingCmnInfo, wCode) #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static void ET9LOCALCALL __ET9SearchGetNextIntervalSlow (ET9AWLingInfo * const pLingInfo, ET9ALdbCursorData * const pCursor, const ET9U16 wPos, const ET9U32 dwTargetPosIn, const ET9BOOL bUsingFlex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; __ET9HeaderConstsGetNextInterval WLOG3(fprintf(pLogFile3, "__ET9SearchGetNextIntervalSlow, wPos %2u, dwTargetPosIn %6u, bUsingFlex %u\n", wPos, dwTargetPosIn, bUsingFlex);) __ET9SearchGetNextInterval(pLingInfo, pCursor, wPos, dwTargetPosIn); if (bUsingFlex && wPos) { __IncCharCounter(pLingCmnInfo, pCursor->wCode); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * *** * * * * */ static void ET9LOCALCALL __ET9SearchGetNextStd (ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; __ET9HeaderConstsGetNextInterval const ET9U8 bPosCount = ALDB.header.bPosCount; const ET9U16 wCodeZero = ALDB.header.wCodeZero; const ET9U8 bRegCmpLength = ALDB.search.bRegCmpLength; ET9U8 * const pbRegPosCurrOrder = ALDB.search.pbRegPosCurrOrder; ET9ALdbCursorData * const pCursors = ALDB.pCursors; ET9U8 bPos; ET9U8 bPosIndex; ET9ALdbCursorData *pCursor; ET9U32 dwCurrItem = ALDB.search.dwCurrItem + 1; WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, dwCurrItem = %6d, spcActive = %d\n", dwCurrItem, ALDB.compare.bSpcActive);) /* validate */ ET9Assert(ALDB.search.bSpcNonZeroPos < ALDB.compare.bActiveCmpLength); /* should not already be exhausted */ ET9Assert(!__IsExhausted()); /* find a "match" - either the regular way or with spell correction */ if (ALDB.compare.bSpcActive && ALDB.compare.wLength > 2) { const ET9U8 bInputLength = (ET9U8)ALDB.compare.wLength; const ET9BOOL bSpcExactCompare = ALDB.compare.bSpcExactCompare; const ET9BOOL bSpcFilteredCompare = ALDB.compare.bSpcFilteredCompare; const ET9BOOL bSpcExactFilter = ALDB.compare.bSpcExactFilter; const ET9U8 bSpcMaxEdits = ALDB.compare.bSpcMaxEdits; const ET9U8 bSpcLengthOffset = ALDB.compare.bSpcLengthOffset; const ET9U8 bSpcNonZeroPos = ALDB.search.bSpcNonZeroPos; const ET9U8 bSpcControlPos = ALDB.search.bSpcControlPos; ET9BOOL bSpcCompare = ALDB.search.bSpcCompare; ET9U32 dwSpcControlEndPos = ALDB.search.dwSpcControlEndPos; ET9U32 dwRegNonMatchEndPos = ALDB.search.dwRegNonMatchEndPos; WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, bSpcCompare = %d, dwSpcControlEndPos = %d, non zero pos = %d, control pos = %d\n", bSpcCompare, dwSpcControlEndPos, bSpcNonZeroPos, bSpcControlPos);) /* repeat regular or spell correction search depending on word length properties */ for (;;) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, NEW REPEAT (%d)\n", dwCurrItem);) /* skip too short words */ pCursor = &pCursors[bSpcNonZeroPos]; __ET9SearchCatchupStream(pLingInfo, pCursor, bSpcNonZeroPos, dwCurrItem); /* return inside macro */ if (pCursor->wCode == wCodeZero) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, too short word, skipping %d\n", (pCursor->dwEndPos - dwCurrItem));) dwCurrItem = pCursor->dwEndPos; continue; } /* check if how-to-compare changed */ if (dwCurrItem >= dwSpcControlEndPos) { ET9U32 dwOldCurrItem = dwCurrItem; pCursor = &pCursors[bSpcControlPos]; /* catch up on interval data (test end to assure side effects from tails) */ for (;;) { __ET9SearchCatchupStreamWithBreak(pLingInfo, pCursor, bSpcControlPos, dwCurrItem); /* return inside macro */ /* make use of known non math */ if (dwRegNonMatchEndPos > dwCurrItem && pCursor->wCode != wCodeZero) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, using known non match (%d)\n", (dwRegNonMatchEndPos < pCursor->dwEndPos ? (dwRegNonMatchEndPos - dwCurrItem) : (pCursor->dwEndPos - dwCurrItem)));) if (dwRegNonMatchEndPos < pCursor->dwEndPos) { dwCurrItem = dwRegNonMatchEndPos; } else { dwCurrItem = pCursor->dwEndPos; } } } /* set compare info */ bSpcCompare = (ET9U8)(pCursor->wCode == wCodeZero); WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, new how-to-compare, spcCompare = %d, dwEndPos = %d (len %d)\n", bSpcCompare, pCursor->dwEndPos, (pCursor->dwEndPos - dwSpcControlEndPos));) dwSpcControlEndPos = pCursor->dwEndPos; /* need to recheck too short words? */ if (bSpcCompare && dwOldCurrItem != dwCurrItem) { continue; } } /* compare depending on spc/regular */ if (bSpcCompare) { /* spc check (while in this mode) */ ET9S8 sbPosIndex; ET9S8 sbAvailableEdits; ET9U32 dwPossibleJump; ET9ALdbSymbPosInfo *pbActiveSection; ET9U8 bCurrWordLength = ALDB.search.bCurrWordLength; ET9U32 dwWordLengthEndPos = ALDB.search.dwWordLengthEndPos; ET9U32 dwWordLengthActiveEdgeEndPos = dwWordLengthEndPos; ET9U32 dwWordLengthZeroEdgeEndPos = dwWordLengthEndPos; __IncScreenAttempt(); /* handle filter */ if (bSpcFilteredCompare) { const ET9U8 bFilterPos = 0; pCursor = &pCursors[bFilterPos]; __ET9SearchCatchupStream(pLingInfo, pCursor, bFilterPos, dwCurrItem); /* return inside macro */ /* check filter position */ if (!__IsCodeActiveSpc(__GetActiveSectionSpc(0), bFilterPos, pCursor->wCode)) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, filter pos not matching, dwEndPos = %d (%d)\n", pCursor->dwEndPos, dwCurrItem);) dwPossibleJump = pCursor->dwEndPos; /* if an exact filter and exact is significant, then we need a supporting (failing) classic position */ if (bSpcExactFilter && bSpcExactCompare) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, looking for a failing classic\n");) sbPosIndex = bInputLength - 1; pCursor = &pCursors[sbPosIndex]; for (; sbPosIndex >= 0; --sbPosIndex, --pCursor) { __ET9SearchCatchupStream(pLingInfo, pCursor, sbPosIndex, dwCurrItem); /* return inside macro */ if (!__IsCodeActive(sbPosIndex, pCursor->wCode)) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, found a failing classic at pos %d, dwEndPos = %d\n", sbPosIndex, pCursor->dwEndPos);) if (dwPossibleJump > pCursor->dwEndPos) { dwPossibleJump = pCursor->dwEndPos; } break; } } if (sbPosIndex >= 0) { __IncScreenRejected(); WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, spc filter rejection with support, skipping %d\n", (dwPossibleJump - dwCurrItem));) dwCurrItem = dwPossibleJump; continue; } } else { __IncScreenRejected(); WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, spc filter rejection, skipping %d\n", (dwPossibleJump - dwCurrItem));) dwCurrItem = dwPossibleJump; continue; } } /* __IsCodeActiveSpc */ } /* bSpcFilteredCompare */ /* determine the current word length */ if (dwCurrItem >= dwWordLengthEndPos) { ET9U8 bMid; ET9U8 bLo = ALDB.compare.bPosLo; ET9U8 bHi = ALDB.compare.bPosHi; while (bLo < bHi) { bMid = (ET9U8)((bLo + bHi) / 2); pCursor = &pCursors[bMid]; __ET9SearchCatchupStream(pLingInfo, pCursor, bMid, dwCurrItem); /* return inside macro */ if (pCursor->wCode == wCodeZero) { bHi = (ET9U8)(bMid - 1); } else { bLo = (ET9U8)(bMid + 1); } } pCursor = &pCursors[bLo]; __ET9SearchCatchupStream(pLingInfo, pCursor, bLo, dwCurrItem); /* return inside macro */ if (pCursor->wCode == wCodeZero) { bCurrWordLength = bLo; } else { bCurrWordLength = (ET9U8)(bLo + 1); } /* investigate how long the length run is (checking the edge streams) */ { ET9U8 bEdgePos = bCurrWordLength; __IncScreenLengthCalc(); pCursor = &pCursors[bEdgePos]; if (bEdgePos < bPosCount) { __ET9SearchCatchupStream(pLingInfo, pCursor, bEdgePos, dwCurrItem); /* return inside macro */ dwWordLengthZeroEdgeEndPos = pCursor->dwEndPos; } else { dwWordLengthZeroEdgeEndPos = ~((ET9U32)0); } WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, how long run, zero edge end = %d %+d (%d)\n", dwWordLengthZeroEdgeEndPos, (dwWordLengthZeroEdgeEndPos - dwCurrItem), dwCurrItem);) ET9Assert(bEdgePos > 0); --pCursor; --bEdgePos; __ET9SearchCatchupStream(pLingInfo, pCursor, bEdgePos, dwCurrItem); /* return inside macro */ dwWordLengthActiveEdgeEndPos = pCursor->dwEndPos; WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, how long run, active edge end = %d %+d (%d)\n", dwWordLengthActiveEdgeEndPos, (dwWordLengthActiveEdgeEndPos - dwCurrItem), dwCurrItem);) if (dwWordLengthActiveEdgeEndPos > dwWordLengthZeroEdgeEndPos) { dwWordLengthEndPos = dwWordLengthZeroEdgeEndPos; } else { dwWordLengthEndPos = dwWordLengthActiveEdgeEndPos; } WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, how long run, calculated end = %d %+d (%d)\n", dwWordLengthEndPos, (dwWordLengthEndPos - dwCurrItem), dwCurrItem);) } /* done with the length thing */ __IncScreenLength(dwWordLengthEndPos - dwCurrItem); WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, determined word length, bCurrWordLength = %d, remaining run = %d (%d)\n", bCurrWordLength, dwWordLengthEndPos - dwCurrItem, dwCurrItem);) /* store locals */ ALDB.search.bCurrWordLength = bCurrWordLength; ALDB.search.dwWordLengthEndPos = dwWordLengthEndPos; } else { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, recovered word length, bCurrWordLength = %d, remaining run = %d (%d)\n", bCurrWordLength, dwWordLengthEndPos - dwCurrItem, dwCurrItem);) } /* set up for compare */ dwPossibleJump = ~((ET9U32)0); sbAvailableEdits = bSpcMaxEdits; pbActiveSection = __GetActiveSectionSpc(bCurrWordLength - bSpcLengthOffset); WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, using section %d (%d)\n", (bCurrWordLength - bSpcLengthOffset), dwCurrItem);) /* words shorter than the input length take advantage of its "non matches" */ if (bCurrWordLength < bInputLength) { sbAvailableEdits -= (bInputLength - bCurrWordLength); } /* loop to find non matching spc word positions */ sbPosIndex = bCurrWordLength - 1; pCursor = &pCursors[sbPosIndex]; for (; sbPosIndex >= 0; --sbPosIndex, --pCursor) { __ET9SearchCatchupStream(pLingInfo, pCursor, sbPosIndex, dwCurrItem); /* return inside macro */ /* check against the spc calculated set */ if (!__IsCodeActiveSpc(pbActiveSection, sbPosIndex, pCursor->wCode)) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, spc pos %2d not matching, dwEndPos = %d %+2d (%d)\n", sbPosIndex, pCursor->dwEndPos, (pCursor->dwEndPos - dwCurrItem), dwCurrItem);) --sbAvailableEdits; if (dwPossibleJump > pCursor->dwEndPos) { dwPossibleJump = pCursor->dwEndPos; } /* found enough? */ if (sbAvailableEdits < 0) { break; } } } /* loop spc positions */ /* if necessary, find a supporting non matching classic position */ if (bSpcExactCompare && sbPosIndex >= 0) { ET9U32 dwPossibleClassicJump = 0; sbPosIndex = bCurrWordLength - 1; pCursor = &pCursors[sbPosIndex]; /* loop to find a non matching classic word position */ for (; sbPosIndex >= 0; --sbPosIndex, --pCursor) { __ET9SearchCatchupStream(pLingInfo, pCursor, sbPosIndex, dwCurrItem); /* return inside macro */ /* check against classic */ if (!__IsCodeActive(sbPosIndex, pCursor->wCode)) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, classic pos %2d not matching, dwEndPos = %d %+2d (%d)\n", sbPosIndex, pCursor->dwEndPos, (pCursor->dwEndPos - dwCurrItem), dwCurrItem);) if (dwPossibleClassicJump < pCursor->dwEndPos) { dwPossibleClassicJump = pCursor->dwEndPos; } if (dwPossibleClassicJump >= dwPossibleJump) { break; } } } /* loop classic positions */ if (dwPossibleClassicJump) { if (dwPossibleJump > dwPossibleClassicJump) { dwPossibleJump = dwPossibleClassicJump; } sbPosIndex = 0; } } /* break if pass (could return if we clean up before leaving) */ if (sbPosIndex < 0) { __IncScreenPassed(); bPosIndex = bCurrWordLength; break; } /* the compare is length dependent, so we can't jump longer than the length runs */ if (dwPossibleJump > dwWordLengthEndPos) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, length run limits the jump, compare %+d, length %+d\n", (dwPossibleJump - dwCurrItem), (dwWordLengthEndPos - dwCurrItem));) if (dwWordLengthActiveEdgeEndPos < dwWordLengthZeroEdgeEndPos) { ET9BOOL bExtended = 0; ET9U8 bAtiveEdgePos = bCurrWordLength - 1; WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, attempting a length extension, active edge %d, zero edge %d\n", dwWordLengthActiveEdgeEndPos, dwWordLengthZeroEdgeEndPos);) pCursor = &pCursors[bAtiveEdgePos]; /* while the active edge is non zero, shorter than the zero edge and the possible jump, extend it */ for (;;) { __ET9SearchCatchupStream(pLingInfo, pCursor, bAtiveEdgePos, dwWordLengthActiveEdgeEndPos); /* return inside macro */ if (pCursor->wCode == wCodeZero) { break; } bExtended = 1; dwWordLengthActiveEdgeEndPos = pCursor->dwEndPos; WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, found extension %d %+d\n", pCursor->dwEndPos, (pCursor->dwEndPos - dwCurrItem));) if (dwWordLengthActiveEdgeEndPos >= dwWordLengthZeroEdgeEndPos || dwWordLengthActiveEdgeEndPos >= dwPossibleJump) { break; } } if (!bExtended) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, no extension possible\n");) dwPossibleJump = dwWordLengthEndPos; } else { /* reevaluate the word length run end */ if (dwWordLengthActiveEdgeEndPos > dwWordLengthZeroEdgeEndPos) { dwWordLengthEndPos = dwWordLengthZeroEdgeEndPos; } else { dwWordLengthEndPos = dwWordLengthActiveEdgeEndPos; } WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, result, active edge %d %+d, zero edge %d %+d\n", dwWordLengthActiveEdgeEndPos, (dwWordLengthActiveEdgeEndPos - dwCurrItem), dwWordLengthZeroEdgeEndPos, (dwWordLengthZeroEdgeEndPos - dwCurrItem));) /* store locals */ ALDB.search.dwWordLengthEndPos = dwWordLengthEndPos; /* reevaluate the length based jump limitation */ if (dwPossibleJump > dwWordLengthEndPos) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, length run limits jump by %d\n", dwPossibleJump - dwWordLengthEndPos);) __IncScreenLimited(); dwPossibleJump = dwWordLengthEndPos; } } } else { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, length run limits jump by %d\n", dwPossibleJump - dwWordLengthEndPos);) __IncScreenLimited(); dwPossibleJump = dwWordLengthEndPos; } } WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, spc rejection, jumping %d\n", (dwPossibleJump - dwCurrItem));) __IncScreenRejected(); __IncScreenJump(dwPossibleJump - dwCurrItem); dwCurrItem = dwPossibleJump; } else { /* regular check (while in this mode) */ bPosIndex = 0; while (bPosIndex < bRegCmpLength) { bPos = pbRegPosCurrOrder[bPosIndex]; pCursor = &pCursors[bPos]; __ET9SearchCatchupStream(pLingInfo, pCursor, bPos, dwCurrItem); /* return inside macro */ if (__IsCodeActive(bPos, pCursor->wCode)) { ++bPosIndex; continue; } /* no match, skip this interval (max as far as this mode goes, save the rest if any) */ WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, spc in regular mode, skipping %d (lost %d)\n", ((dwSpcControlEndPos < pCursor->dwEndPos ? dwSpcControlEndPos : pCursor->dwEndPos) - dwCurrItem), (pCursor->dwEndPos > dwSpcControlEndPos ? (pCursor->dwEndPos - dwSpcControlEndPos) : 0));) if (dwSpcControlEndPos < pCursor->dwEndPos) { dwCurrItem = dwSpcControlEndPos; dwRegNonMatchEndPos = pCursor->dwEndPos; break; } else { dwCurrItem = pCursor->dwEndPos; bPosIndex = 0; continue; } } /* check if candidate passed */ if (bPosIndex == bRegCmpLength) { break; } } } /* repeat */ /* store locals */ ALDB.search.bSpcCompare = bSpcCompare; ALDB.search.dwSpcControlEndPos = dwSpcControlEndPos; ALDB.search.dwRegNonMatchEndPos = dwRegNonMatchEndPos; } else { /* non spell correction search */ /* first pos set opt (after previous word) */ if (ALDB.compare.bFirstPosSetOpt && *ALDB.search.pwLength == 1) { const ET9U8 bPos = 0; ET9U16 wFirstCode = __LdbGetCharCodeAtPos(bPos); __DeactivateCode(bPos, wFirstCode); WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, removing code from first set, code = %d\n", wFirstCode);) } /* loop over positions that must "match" */ bPosIndex = 0; while (bPosIndex < bRegCmpLength) { bPos = pbRegPosCurrOrder[bPosIndex]; WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, bPos = %d, dwItem = %d\n", bPos, dwCurrItem);) pCursor = &pCursors[bPos]; __ET9SearchCatchupStream(pLingInfo, pCursor, bPos, dwCurrItem); /* return inside macro */ /* if match continue with next pos */ if (__IsCodeActive(bPos, pCursor->wCode)) { ++bPosIndex; continue; } /* no match, skip this interval */ dwCurrItem = pCursor->dwEndPos; /* no match, restart the comparison run */ WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, no match, restart\n");) bPosIndex = 0; } /* while more positions */ } /* bSpcActive */ /* get non stem chars */ { ET9U8 bZeroPos = bPosCount; pCursor = &pCursors[bPosIndex]; for (; bPosIndex < bPosCount; ++bPosIndex, ++pCursor) { WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextStd, non stem part, bPos = %d\n", bPosIndex);) __ET9SearchCatchupStream(pLingInfo, pCursor, bPosIndex, dwCurrItem); /* return inside macro */ /* check for EOS */ if (pCursor->wCode == wCodeZero) { bZeroPos = bPosIndex; break; } } /* length */ *ALDB.search.pwLength = bZeroPos; } /* store locals */ ALDB.search.dwCurrItem = dwCurrItem; /* done */ WLOG3B({ ET9U16 wIndex; fprintf(pLogFile3, "__ET9SearchGetNextStd, found one, length = %d, dwItem = %d, word = ", *ALDB.search.pwLength, dwCurrItem); for (wIndex = 0; wIndex < *ALDB.search.pwLength; ++wIndex) { ET9SYMB sSymb = ALDB.search.psTarget[wIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F) { fprintf(pLogFile3, "%c", (char)sSymb); } else { fprintf(pLogFile3, "<%x>", (int)sSymb); } } fprintf(pLogFile3, "\n"); }) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * *** * * * * */ static void ET9LOCALCALL __ET9SearchGetNextFlex (ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; #ifdef ET9ASPC_DEV_SCREEN ET9ASPCFlexCompareData * const pCMP = &ASPC.u.sCmpDataFlex; #endif __ET9HeaderConstsGetNextInterval const ET9U8 bPosCount = ALDB.header.bPosCount; const ET9U16 wCodeZero = ALDB.header.wCodeZero; const ET9U8 bMatchPos = 0; const ET9U8 bNonZeroPos = ALDB.search.bSpcNonZeroPos; ET9ALdbCursorData * const pCursors = ALDB.pCursors; ET9ALdbCursorData * const pMatchCursor = &pCursors[bMatchPos]; ET9ALdbCursorData * const pNonZeroCursor = &pCursors[bNonZeroPos]; ET9U32 dwCurrItem = ALDB.search.dwCurrItem + 1; WLOG3B(fprintf(pLogFile3, "__ET9SearchGetNextFlex, dwCurrItem = %6d\n", dwCurrItem);) /* should not already be exhausted */ ET9Assert(!__IsExhausted()); /* find a "match" */ for (;;) { WLOG3B(fprintf(pLogFile3, " @ %u\n", dwCurrItem);) __ET9SearchCatchupStream(pLingInfo, pMatchCursor, bMatchPos, dwCurrItem); /* return inside macro */ if (!__IsCodeActiveSpc(ALDB.compare.pppbSpcFlexSection, bMatchPos, pMatchCursor->wCode)) { WLOG3B(fprintf(pLogFile3, " no match, skipping %u\n", (pMatchCursor->dwEndPos - dwCurrItem));) dwCurrItem = pMatchCursor->dwEndPos; continue; } __ET9SearchCatchupStreamCounters(pLingInfo, pNonZeroCursor, bNonZeroPos, dwCurrItem); /* return inside macro */ if (pNonZeroCursor->wCode == wCodeZero) { WLOG3B(fprintf(pLogFile3, " too short, skipping %u\n", (pNonZeroCursor->dwEndPos - dwCurrItem));) dwCurrItem = pNonZeroCursor->dwEndPos; continue; } /* exact matching? */ if (ALDB.compare.bSpcExactFilterTrace && !ALDB.compare.bSpcExactPrunedTrace && dwCurrItem > ALDB.header.dwTopCount) { ALDB.compare.bSpcExactPrunedTrace = 1; ALDB.compare.pppbSpcFlexSection = __GetActiveSectionFlexSpc(1); } /* get the rest */ { #ifdef ET9ASPC_DEV_SCREEN const ET9U16 wPrevWordLength = *ALDB.search.pwLength; #endif ET9U8 bPosIndex = 1; ET9ALdbCursorData * pCursor = &pCursors[bPosIndex]; for (; bPosIndex < bPosCount; ++bPosIndex, ++pCursor) { WLOG3B(fprintf(pLogFile3, " + bPos = %d\n", bPosIndex);) __ET9SearchCatchupStreamCounters(pLingInfo, pCursor, bPosIndex, dwCurrItem); /* return inside macro */ /* check for EOS */ if (pCursor->wCode == wCodeZero) { *ALDB.search.pwLength = bPosIndex; break; } } if (bPosIndex >= bPosCount) { *ALDB.search.pwLength = bPosIndex; } #ifdef ET9ASPC_DEV_SCREEN for (; bPosIndex < wPrevWordLength; ++bPosIndex, ++pCursor) { WLOG3B(fprintf(pLogFile3, " + bPos = %d\n", bPosIndex);) __ET9SearchCatchupStreamCounters(pLingInfo, pCursor, bPosIndex, dwCurrItem); /* return inside macro */ } #endif } __STAT_AWLdb_Flex_EDLL_Cnt; #ifdef ET9ASPC_DEV_SCREEN /* supported? */ if (pCMP->nSupportedQualityCount < (pCMP->nActiveQualityCount - pLingCmnInfo->Private.bCurrMaxEditDistance)) { ++dwCurrItem; __STAT_AWLdb_Flex_EDLLS_Cnt; WLOG3(fprintf(pLogFile3, " __ET9SearchGetNextFlex, screening, nSupportedQualityCount %2u, nActiveQualityCount %2u, bCurrMaxEditDistance %u\n", pCMP->nSupportedQualityCount, pCMP->nActiveQualityCount, pLingCmnInfo->Private.bCurrMaxEditDistance);) continue; } #endif /* found one */ break; } /* store locals */ ALDB.search.dwCurrItem = dwCurrItem; /* done */ WLOG3B({ ET9U16 wIndex; fprintf(pLogFile3, "__ET9SearchGetNextFlex, found one, length = %d, dwItem = %d, word = ", *ALDB.search.pwLength, dwCurrItem); for (wIndex = 0; wIndex < *ALDB.search.pwLength; ++wIndex) { ET9SYMB sSymb = ALDB.search.psTarget[wIndex]; if (sSymb >= 0x20 && sSymb <= 0x7F) { fprintf(pLogFile3, "%c", (char)sSymb); } else { fprintf(pLogFile3, "<%x>", (int)sSymb); } } fprintf(pLogFile3, "\n"); }) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * / * * * */ static void ET9LOCALCALL __ET9SearchStart (ET9AWLingInfo * const pLingInfo, ET9SYMB * const psTarget, ET9U16 * const pwLength, const ET9U16 wTargetLength, const ET9BOOL bUsingFlex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U8 bIndex; ET9ALdbCursorData *pCursor; WLOG3(fprintf(pLogFile3, "__ET9SearchStart, wTargetLength = %d\n", wTargetLength);) ALDB.search.psTarget = psTarget; ALDB.search.pwLength = pwLength; ALDB.search.wTargetLength = wTargetLength; ALDB.search.bExhausted = (ET9BOOL)(wTargetLength < ALDB.header.bPosCount || (ALDB.compare.wLength > ALDB.header.bPosCount && !bUsingFlex && !pLingCmnInfo->Private.ALdbMGD.bSupported) || (pLingCmnInfo->Private.wCurrMinSourceLength > ALDB.header.bPosCount)); /* set up non zero pos and the spc control pos */ if (ALDB.compare.bSpcActive || bUsingFlex) { /* the non zero pos is the same as the min length */ ALDB.search.bSpcNonZeroPos = (ET9U8)pLingCmnInfo->Private.wCurrMinSourceLength; if (ALDB.search.bSpcNonZeroPos) { --ALDB.search.bSpcNonZeroPos; } ALDB.search.bSpcControlPos = (ET9U8)pLingCmnInfo->Private.wCurrMaxSourceLength; if (ALDB.search.bSpcControlPos >= ALDB.header.bPosCount) { ALDB.search.bSpcControlPos = 0; } } else { ALDB.search.bSpcNonZeroPos = 0; ALDB.search.bSpcControlPos = 0; } WLOG3(fprintf(pLogFile3, "__ET9SearchStart, wSpcNonZeroPos = %d, wSpcControlPos = %d\n", ALDB.search.bSpcNonZeroPos, ALDB.search.bSpcControlPos);) ET9Assert(ALDB.search.bSpcNonZeroPos < ALDB.header.bPosCount || ALDB.search.bExhausted); /* set up word length info */ ALDB.search.bCurrWordLength = 0; ALDB.search.dwWordLengthEndPos = 0; /* set up pos order for regular compare (don't futz with array pointers) */ { ET9U8 bOrderIndex = 0; ALDB.search.bRegCmpLength = bUsingFlex? 1 : ALDB.compare.bActiveCmpLength; for (bIndex = 0; bOrderIndex < ALDB.search.bRegCmpLength; ++bIndex) { if (ALDB.header.pbPosOrder[bIndex] < ALDB.search.bRegCmpLength) { ET9Assert(bOrderIndex < ET9MAXLDBWORDSIZE); ALDB.search.pbRegPosCurrOrder[bOrderIndex++] = ALDB.header.pbPosOrder[bIndex]; } } for (bIndex = (ET9U8)ALDB.search.bRegCmpLength; bIndex < ALDB.header.bPosCount; ++bIndex) { ET9Assert(bOrderIndex < ET9MAXLDBWORDSIZE); ALDB.search.pbRegPosCurrOrder[bOrderIndex++] = bIndex; } WLOG3({ fprintf(pLogFile3, "__ET9SearchStart, reg pos order: "); for (bIndex = 0; bIndex < ALDB.header.bPosCount; ++bIndex) { fprintf(pLogFile3, "%2d ", ALDB.search.pbRegPosCurrOrder[bIndex]); } fprintf(pLogFile3, "\n"); }) } /* set up cursors */ pCursor = ALDB.pCursors; for (bIndex = 0; bIndex < ALDB.header.bPosCount; ++bIndex, ++pCursor) { pCursor->wCode = 0; pCursor->dwStartPos = 0; pCursor->dwEndPos = 0; pCursor->dwJumpPos = 0; pCursor->dwJumpAddress = 0; { ET9U8 ET9FARDATA_LDB *pbCurrData; __LdbSetIndex(pCursor, ALDB.header.pdwIntervalOffsets[bIndex]); pCursor->pbCurrData = pbCurrData; } if (!__IsExhausted()) { __ET9SearchGetNextIntervalSlow(pLingInfo, pCursor, bIndex, 0, bUsingFlex); } } /* set up initial search mode info */ if (ALDB.search.bSpcControlPos) { ALDB.search.bSpcCompare = (ET9BOOL)(ALDB.pCursors[ALDB.search.bSpcControlPos].wCode == ALDB.header.wCodeZero); ALDB.search.dwSpcControlEndPos = ALDB.pCursors[ALDB.search.bSpcControlPos].dwEndPos; } else { ALDB.search.bSpcCompare = 1; ALDB.search.dwSpcControlEndPos = ~((ET9U32)0); } ALDB.search.dwRegNonMatchEndPos = 0; /* move to the first word */ if (!__IsExhausted()) { /* __ET9SearchGetNext moves forward one step, compensate by moving one back first, to get to the first one... */ ALDB.search.dwCurrItem = 0; --ALDB.search.dwCurrItem; /* need a defined value for first pos opt (anything but '1') */ *ALDB.search.pwLength = 0; /* move */ if (bUsingFlex) { __ET9SearchGetNextFlex(pLingInfo); } else { __ET9SearchGetNextStd(pLingInfo); } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9SearchInit (ET9AWLingInfo * const pLingInfo, const ET9U32 dwChunkStartPos, const ET9BOOL bReadCharTable, const ET9BOOL bSkipCharTable) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9U32 dwIndex; ET9U32 dwStartPos = dwChunkStartPos; ET9SYMB *pSymb; ET9U8 *pbItem; ET9U16 *pwItem; ET9U32 *pDW; /* Number of character positions (max word length) - 1 byte */ ALDB.header.bPosCount = _ET9ReadLDBByte(pLingInfo, dwStartPos++); if (!ALDB.header.bPosCount || ALDB.header.bPosCount > ET9MAXLDBWORDSIZE) { return ET9STATUS_CORRUPT_DB; } /* Position order data - 1 byte per position */ pbItem = ALDB.header.pbPosOrder; for (dwIndex = 0; dwIndex < ALDB.header.bPosCount; ++dwIndex) { *pbItem++ = _ET9ReadLDBByte(pLingInfo, dwStartPos++); } /* Word length end (last) data - 3 bytes per position (obsolete) */ dwStartPos += 3 * ALDB.header.bPosCount; if (bReadCharTable) { /* Character decode data - 2 byte count plus 2 bytes per item */ ALDB.header.wCharacterDecodeCount = _ET9ReadLDBWord2(pLingInfo, dwStartPos); dwStartPos += 2; if (ALDB.header.wCharacterDecodeCount > ALDB_HEADER_MAX_CHAR_CODES) { return ET9STATUS_CORRUPT_DB; } pSymb = ALDB.header.psCharacterDecodeTable; for (dwIndex = 0; dwIndex < ALDB.header.wCharacterDecodeCount; ++dwIndex) { *pSymb++ = _ET9ReadLDBWord2(pLingInfo, dwStartPos); dwStartPos += 2; } } else if (bSkipCharTable) { dwStartPos += 2 + ALDB.header.wCharacterDecodeCount * 2; /* Skip the size of the table and the table. */ } /* One byte interval decode data (codes + lengths) - 2 * 255 bytes */ pbItem = ALDB.header.pbOneByteCodes; for (dwIndex = 0; dwIndex < ALDB_HEADER_ONE_BYTE_SIZE; ++dwIndex) { *pbItem++ = _ET9ReadLDBByte(pLingInfo, dwStartPos++); } *pbItem = 0xFF; /* really not in play more than for performance purposes (unused or rare code) */ pbItem = ALDB.header.pbOneByteLengths; for (dwIndex = 0; dwIndex < ALDB_HEADER_ONE_BYTE_SIZE; ++dwIndex) { *pbItem++ = _ET9ReadLDBByte(pLingInfo, dwStartPos++); } /* Start offset for each position's interval data stream - 3 bytes per (position + 1) */ pDW = ALDB.header.pdwIntervalOffsets; for (dwIndex = 0; dwIndex <= ALDB.header.bPosCount; ++dwIndex) { *pDW++ = _ET9ReadLDBWord3(pLingInfo, dwStartPos); dwStartPos += 3; } /* Locate the control codes */ ALDB.header.wCodeZero = 0xFFFF; ALDB.header.wCodeIntervalEnd = 0xFFFF; ALDB.header.wCodeIntervalJump = 0xFFFF; ALDB.header.wCodeIntervalExtend = 0xFFFF; pSymb = ALDB.header.psCharacterDecodeTable; for (dwIndex = 0; dwIndex < ALDB.header.wCharacterDecodeCount; ++dwIndex, ++pSymb) { if (*pSymb == 0) { ALDB.header.wCodeZero = (ET9U16)dwIndex; } else if (*pSymb == ALDB_INTERVAL_END_CHAR_VAL) { ALDB.header.wCodeIntervalEnd = (ET9U16)dwIndex; } else if (*pSymb == ALDB_INTERVAL_JUMP_CHAR_VAL) { ALDB.header.wCodeIntervalJump = (ET9U16)dwIndex; } else if (*pSymb == ALDB_INTERVAL_EXTEND_CHAR_VAL) { ALDB.header.wCodeIntervalExtend = (ET9U16)dwIndex; } } if (ALDB.header.wCodeZero == 0xFFFF || ALDB.header.wCodeIntervalEnd == 0xFFFF || ALDB.header.wCodeIntervalJump == 0xFFFF || ALDB.header.wCodeIntervalExtend == 0xFFFF) { return ET9STATUS_CORRUPT_DB; } /* Recreate character encode info - also check if there are upper/lower case characters */ ALDB.header.bLowerCount = 0; ALDB.header.bUpperCount = 0; pwItem = ALDB.header.pwCharacterEncodeTable; for (dwIndex = 0; dwIndex < ALDB_HEADER_MAX_DIRECT_ENCODE; ++dwIndex) { *pwItem++ = ALDB_CHAR_CODE_NONE; } ALDB.header.wCharacterEncodeExtendCount = 0; ALDB.header.sCharacterEncodeExtendFirstChar = 0xFFFF; ALDB.header.sCharacterEncodeExtendLastChar = 0; pSymb = ALDB.header.psCharacterDecodeTable; for (dwIndex = 0; dwIndex < ALDB.header.wCharacterDecodeCount; ++dwIndex, ++pSymb) { if (*pSymb == ALDB_INTERVAL_END_CHAR_VAL || *pSymb == ALDB_INTERVAL_JUMP_CHAR_VAL || *pSymb == ALDB_INTERVAL_EXTEND_CHAR_VAL) { continue; } /* Check to see if sym has table support; if not, flag LDB as incompatible */ if (*pSymb && _ET9_GetSymbolClass(*pSymb) == ET9_UnassSymbClass) { return ET9STATUS_DB_CORE_INCOMP; } if (_ET9SymIsLower(*pSymb, pWordSymbInfo->Private.dwLocale)) { ++ALDB.header.bLowerCount; } else if (_ET9SymIsUpper(*pSymb, pWordSymbInfo->Private.dwLocale)) { ++ALDB.header.bUpperCount; } if (*pSymb >= ALDB_HEADER_MAX_DIRECT_ENCODE) { ++ALDB.header.wCharacterEncodeExtendCount; if (ALDB.header.sCharacterEncodeExtendFirstChar > *pSymb) { ALDB.header.sCharacterEncodeExtendFirstChar = *pSymb; } if (ALDB.header.sCharacterEncodeExtendLastChar < *pSymb) { ALDB.header.sCharacterEncodeExtendLastChar = *pSymb; } continue; } ALDB.header.pwCharacterEncodeTable[*pSymb] = (ET9U16)dwIndex; } /* Const cursor info */ #ifdef ET9_DIRECT_LDB_ACCESS #else { ET9ALdbCursorData *pCursor; pCursor = ALDB.pCursors; for (dwIndex = 0; dwIndex < ALDB.header.bPosCount; ++dwIndex, ++pCursor) { pCursor->pbCacheEnd = &pCursor->pbCache[ALDB_CURSOR_DATA_CACHE_SIZE]; } } #endif /* ET9_DIRECT_LDB_ACCESS */ /* */ _ET9ClearMem((ET9U8*)ALDB.compare.pbCodeIsFree, sizeof(ALDB.compare.pbCodeIsFree)); { ET9U16 wCode; ET9SYMB *psSymb; psSymb = ALDB.header.psCharacterDecodeTable; for (wCode = 0; wCode < ALDB.header.wCharacterDecodeCount; ++wCode, ++psSymb) { if (_ET9_IsFree(*psSymb)) { __SetCodeIsFree(wCode); } } } #ifdef ET9_DEBUGLOG3 { ET9U16 wCode; ET9SYMB *psSymb; WLOG3(fprintf(pLogFile3, "\n=== Character Decode Table ===\n");) psSymb = ALDB.header.psCharacterDecodeTable; for (wCode = 0; wCode < ALDB.header.wCharacterDecodeCount; ++wCode, ++psSymb) { if (*psSymb >= 0x20 && *psSymb <= 0x7F) { WLOG3(fprintf(pLogFile3, "%c\n", (char)*psSymb);) } else { WLOG3(fprintf(pLogFile3, "<%x>\n", (int)*psSymb);) } } WLOG3(fprintf(pLogFile3, "\n");) } #endif /* Done */ return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __SetWordCount (ET9AWLingCmnInfo * const pLingCmnInfo, ET9U32 dwWordCount) { ALDB.header.dwWordCount = dwWordCount; ALDB.header.dwTopCount = (ET9U32)(dwWordCount / 20); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __InvestigateWordCount (ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U8 bLastPos = ALDB.header.bPosCount - 1; ET9ALdbCursorData *pCursor = &ALDB.pCursors[bLastPos]; ET9AWPrivWordInfo sWord; pCursor->wCode = 0; pCursor->dwStartPos = 0; pCursor->dwEndPos = 0; pCursor->dwJumpPos = 0; pCursor->dwJumpAddress = 0; { ET9U8 ET9FARDATA_LDB *pbCurrData; __LdbSetIndex(pCursor, ALDB.header.pdwIntervalOffsets[bLastPos]); pCursor->pbCurrData = pbCurrData; } ALDB.search.bExhausted = 0; ALDB.search.psTarget = sWord.Base.sWord; ALDB.search.pwLength = &sWord.Base.wWordLen; ALDB.search.wTargetLength = ET9MAXWORDSIZE; __ET9SearchGetNextIntervalSlow(pLingInfo, pCursor, bLastPos, 0, 0 /* bUsingFlex */); while (!__IsExhausted()) { __ET9SearchGetNextIntervalSlow(pLingInfo, pCursor, bLastPos, pCursor->dwEndPos, 0 /* bUsingFlex */); } __SetWordCount(pLingCmnInfo, pCursor->dwEndPos); } /* ************************************************************************* *********************** ACCESS ****************************************** ************************************************************************* */ #undef ET9_SPC_ED_GetMatchInfo #undef ET9_SPC_ED_GetMatchOnly #undef ET9_SPC_ED_GetWordSymb #undef ET9_SPC_ED_USE_COMPARE_CACHE #undef __ET9AWCalcEditDistance /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ #define ET9_SPC_ED_GetMatchInfo(index1, index2, bExactMatchOnly, pbMatch, pbFreq, pbLocked, pbLimited) \ { \ pbFreq = (ET9U8)__GetCodeFreq((index1), __LdbGetCharCodeAtPos(index2)); \ \ if (pbFreq) { \ if (__IsCodeExact((index1), __LdbGetCharCodeAtPos(index2))) { \ pbMatch = ET9_SPC_ED_MATCH_EXACT; \ } \ else { \ pbMatch = ET9_SPC_ED_MATCH_FULL; \ } \ } \ else { \ pbMatch = ET9_SPC_ED_MATCH_NONE; \ } \ \ pbLocked = __GetPosLock(index1); \ pbLimited = __IsCodeLimited((index1), __LdbGetCharCodeAtPos(index2)); \ \ __STAT_INC_CmpLdbInfo; \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ #define ET9_SPC_ED_GetMatchOnly(index1, index2, bExactMatchOnly, pbMatch, pbLimited) \ { \ if (__IsCodeActive((index1), __LdbGetCharCodeAtPos(index2))) { \ if (__IsCodeExact((index1), __LdbGetCharCodeAtPos(index2))) { \ pbMatch = ET9_SPC_ED_MATCH_EXACT; \ } \ else { \ pbMatch = ET9_SPC_ED_MATCH_FULL; \ } \ } \ else { \ pbMatch = ET9_SPC_ED_MATCH_NONE; \ } \ \ pbLimited = __IsCodeLimited((index1), __LdbGetCharCodeAtPos(index2)); \ \ __STAT_INC_CmpLdbOnly; \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ #define ET9_SPC_ED_GetWordSymb(index, psSymb) \ { \ (psSymb) = ALDB.search.psTarget[index]; \ } \ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ #define __ET9AWCalcEditDistance __ET9AWCalcEditDistanceALDB #include "et9aspci.h" /* ************************************************************************* *********************** HIGHER LEVEL ************************************ ************************************************************************* */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __SetTagPriorityValue(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U32 dwIndex, const ET9U8 bTagValue) { const ET9U32 dwCacheIndex = dwIndex % (ET9MAX_LDB_TAG_CACHE_BYTES * 4); ET9Assert(bTagValue <= 0x3); ALDB.tags.pbPriority[dwCacheIndex >> 2] |= (bTagValue & 0x3) << ((dwCacheIndex % 4) << 1); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static ET9U8 ET9LOCALCALL __GetTagPriorityValue(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U32 dwIndex) { const ET9U32 dwCacheIndex = dwIndex % (ET9MAX_LDB_TAG_CACHE_BYTES * 4); const ET9U8 bEntry = ALDB.tags.pbPriority[dwCacheIndex >> 2]; return (ET9U8)((bEntry >> ((dwCacheIndex % 4) << 1)) & 0x3); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9INLINE static void ET9LOCALCALL __UpdateTagPriorityValue(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U32 dwIndex, const ET9U8 bTagValue) { const ET9U8 bEntry = __GetTagPriorityValue(pLingCmnInfo, dwIndex); WLOG3(fprintf(pLogFile3, "__UpdateTagPriorityValue, dwIndex %6u, bTagValue %u, old %u\n", dwIndex, bTagValue, bEntry);) ++ALDB.tags.dwTotalCount; if (bEntry) { if (bEntry != bTagValue) { if (bEntry < bTagValue) { __SetTagPriorityValue(pLingCmnInfo, dwIndex, bTagValue); } ++ALDB.tags.dwCollisionCount; } } else { __SetTagPriorityValue(pLingCmnInfo, dwIndex, bTagValue); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U16 ET9LOCALCALL __ET9AWLdbGetWordClass(ET9AWLingInfo * const pLingInfo, const ET9U32 dwIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U16 wWordClass = 0; ET9Assert(pLingInfo); WLOG7(fprintf(pLogFile7, "__ET9AWLdbGetWordClass, dwALMStartAddress %u, dwIndex %u, dwNumEntries %u, wNumClasses %u\n", pLingCmnInfo->Private.ALdbLM.dwALMStartAddress, dwIndex, pLingCmnInfo->Private.ALdbLM.dwNumEntries, pLingCmnInfo->Private.ALdbLM.wNumClasses);) ET9Assert(dwIndex < pLingCmnInfo->Private.ALdbLM.dwNumEntries); if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 256) { /* Single byte */ wWordClass = _ET9ReadLDBByte(pLingInfo, pLingCmnInfo->Private.ALdbLM.dwALMStartAddress + dwIndex); } else if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 512) { /* Double byte */ wWordClass = _ET9ReadLDBWord2(pLingInfo, pLingCmnInfo->Private.ALdbLM.dwALMStartAddress + (dwIndex << 1)); } return wWordClass; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __TagContextWords(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U8 bOrder = (pLingCmnInfo->Private.ALdbNLM.bSupported && pLingCmnInfo->Private.ALdbNLM.bOrder > 3) ? pLingCmnInfo->Private.ALdbNLM.bOrder : 3; ET9INT snContextIndex; ET9U8 bExact; ET9U8 bLowercase; ET9U32 dwContextWordIndex; ET9Assert(pLingInfo); if (pLingInfo->Private.wLDBInitOK != ET9GOODSETUP) { return; } if (!pLingCmnInfo->Private.bUsingLM || !pLingCmnInfo->Private.ALdbLM.bSupported) { return; } WLOG7(fprintf(pLogFile7, "__TagContextWords\n");) /* init internal contexts */ for (snContextIndex = ET9NLM_CONTEXT_COUNT - 1; snContextIndex >= 0; --snContextIndex) { if (pLingCmnInfo->Private.pContextWords[snContextIndex].wLen) { pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextIndex] = pLingCmnInfo->Private.pContextWords[snContextIndex]; } else { pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextIndex].wLen = ET9DEFAULT_SYMB_LEN; pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextIndex].sString[0] = ET9DEFAULT_SYMB; } } /* */ for (snContextIndex = (ET9INT)bOrder - 1; snContextIndex >= 0; --snContextIndex) { ET9SYMB sWord[ET9MAXWORDSIZE]; const ET9U16 wWordSize = pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextIndex].wLen; _ET9SymCopy(sWord, pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextIndex].sString, wWordSize); WLOG7(fprintf(pLogFile7, "__TagContextWords, snContextIndex %d, wLdbNum %08x, wWordSize %u, word ", snContextIndex, pLingCmnInfo->dwLdbNum, wWordSize);) #ifdef ET9_DEBUGLOG7 { ET9U16 wIndex; for (wIndex = 0; wIndex < wWordSize; ++wIndex) { fprintf(pLogFile7, "%c", sWord[wIndex]); } } #endif WLOG7(fprintf(pLogFile7, "\n");) if (wWordSize == 1 && _ET9_IsPunctChar(sWord[0])) { continue; } if (_ET9AWLdbFindEntry(pLingInfo, pLingCmnInfo->dwLdbNum, ET9MGD_STAND_ALONE, sWord, wWordSize, &dwContextWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { WLOG7(fprintf(pLogFile7, "__TagContextWords, exact match or lower case variant available (1)\n");) } else { /* track for buildarounds */ ET9U16 wWordLen; ET9U16 wSubWordLen = 0; ET9U16 wPunctSubWordLen = 0; ET9BOOL bFoundBuildAround = 0; ET9SYMB sSubWord[ET9MAXWORDSIZE]; ET9SYMB sPunctSubWord[ET9MAXWORDSIZE]; for (wWordLen = wWordSize; wWordLen > 0; --wWordLen) { if (_ET9_IsPunctChar(sWord[wWordLen - 1])) { ET9U16 wCharIndex; for (wCharIndex = 0; wWordLen <= wWordSize; wCharIndex++, wWordLen++) { sPunctSubWord[wCharIndex] = sWord[wWordLen - 1]; if (wCharIndex) { sSubWord[wCharIndex - 1] = sWord[wWordLen - 1]; } } wPunctSubWordLen = wCharIndex; wSubWordLen = wCharIndex - 1; bFoundBuildAround = 1; break; } } WLOG7(fprintf(pLogFile7, "__TagContextWords, bFoundBuildAround %u\n", bFoundBuildAround);) if (bFoundBuildAround) { if (wPunctSubWordLen == 1 || _ET9AWLdbFindEntry(pLingInfo, pLingCmnInfo->dwLdbNum, ET9MGD_STAND_ALONE, sPunctSubWord, wPunctSubWordLen, &dwContextWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { ET9INT snContextCopyIndex; WLOG7(fprintf(pLogFile7, "__TagContextWords, exact match or lowercase variant available (2)\n");) for (snContextCopyIndex = (ET9INT)bOrder - 1; snContextCopyIndex > snContextIndex + 1; --snContextCopyIndex) { pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex] = pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex - 1]; } pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].wLen = wWordSize - wPunctSubWordLen; _ET9SymCopy(pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].sString, sWord, pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].wLen); pLingCmnInfo->Private.sCurrContext.pIntContextWords[--snContextCopyIndex].wLen = wPunctSubWordLen; _ET9SymCopy(pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].sString, sPunctSubWord, wPunctSubWordLen); } else if (wSubWordLen && _ET9AWLdbFindEntry(pLingInfo, pLingCmnInfo->dwLdbNum, ET9MGD_STAND_ALONE, sSubWord, wSubWordLen, &dwContextWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { ET9INT snContextCopyIndex; WLOG7(fprintf(pLogFile7, "__TagContextWords, exact match or lowercase variant available (3)\n");) for (snContextCopyIndex = (ET9INT)bOrder - 1; snContextCopyIndex > snContextIndex + 1; --snContextCopyIndex) { pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex] = pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex - 1]; } pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].wLen = wWordSize - wSubWordLen; _ET9SymCopy(pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].sString, sWord, pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].wLen); pLingCmnInfo->Private.sCurrContext.pIntContextWords[--snContextCopyIndex].wLen = wSubWordLen; _ET9SymCopy(pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextCopyIndex].sString, sSubWord, wSubWordLen); } } } } WLOG7(fprintf(pLogFile7, "__TagContextWords, done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U32 ET9LOCALCALL __GetWordIndexFromString(ET9AWLingInfo * const pLingInfo, ET9SimpleWord const * const pWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U8 bExact; ET9U8 bLowercase; ET9U32 dwWordIndex; ET9Assert(pLingInfo); ET9Assert(pWord); ET9Assert(pWord->wLen <= ET9MAXWORDSIZE); if (!pWord->wLen) { WLOG7(fprintf(pLogFile7, "__GetWordIndexFromString, zero len, NULL_INDEX\n");) return ET9NULL_INDEX; } if (_ET9AWLdbFindEntry(pLingInfo, pLingCmnInfo->dwLdbNum, ET9MGD_STAND_ALONE, pWord->sString, pWord->wLen, &dwWordIndex, &bExact, &bLowercase) == ET9STATUS_WORD_EXISTS) { /* exact match or lower case variant available */ WLOG7(fprintf(pLogFile7, "__GetWordIndexFromString, _ET9AWLdbFindEntry found word, dwWordIndex %u\n", dwWordIndex);) #ifdef ET9_DEBUG if (!pLingInfo->Private.pConvertSymb) { ET9AWPrivWordInfo sWord; _ET9AWLdbWordsByIndex(pLingInfo, pLingCmnInfo->dwLdbNum, &dwWordIndex, 1, 0, &sWord); ET9Assert(sWord.Base.wWordLen == pWord->wLen); { ET9UINT nCount; ET9SYMB const *psA = &sWord.Base.sWord[0]; ET9SYMB const *psB = &pWord->sString[0]; for (nCount = sWord.Base.wWordLen; nCount; --nCount, ++psA, ++psB) { ET9Assert(*psA == *psB || _ET9SymToLower(*psA, pLingCmnInfo->dwLdbNum) == _ET9SymToLower(*psB, pLingCmnInfo->dwLdbNum) || _ET9IsCaseExceptionChar(*psA, *psB)); } } } #endif return dwWordIndex; } else if (_ET9_IsNumericString(pWord->sString, pWord->wLen)) { WLOG7(fprintf(pLogFile7, "__GetWordIndexFromString, NUMERIC_INDEX\n", dwWordIndex);) return ET9NUMERIC_INDEX; } WLOG7(fprintf(pLogFile7, "__GetWordIndexFromString, UNKNOWN_INDEX\n", dwWordIndex);) return ET9UNKNOWN_INDEX; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __ClearNLMCache(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U32 dwDefaultIndexValue = 0xFFFFFFCE; ET9UINT nIndex1; _ET9ClearMem((ET9U8*)pLingCmnInfo->Private.ALdbNLM.ANLM_Cache, (ET9U32)sizeof(pLingCmnInfo->Private.ALdbNLM.ANLM_Cache)); for (nIndex1 = 0; nIndex1 < (ET9NLM_MAX_ORDER - 1); ++nIndex1) { ET9UINT nIndex2; for (nIndex2 = 0; nIndex2 < (ET9NLM_MAX_ORDER - 1); ++nIndex2) { pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[nIndex1].ANLM_Cache_Data[nIndex2].dwIndex = dwDefaultIndexValue; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __TagContextWordIndex(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo const * const pWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9INT snContextIndex; ET9Assert(pLingInfo); if (!pLingCmnInfo->Private.bUsingLM || !pLingCmnInfo->Private.ALdbLM.bSupported) { return; } WLOG3(fprintf(pLogFile3, "__TagContextWordIndex, start\n");) WLOG7(fprintf(pLogFile7, "__TagContextWordIndex, start\n");) __TagContextWords(pLingInfo); WLOG7(fprintf(pLogFile7, "__TagContextWordIndex, after tag context words\n");) if (pWord && pWord->Body.wPrefixLen) { /* Context word index for special tokens */ for (snContextIndex = ET9NLM_CONTEXT_COUNT - 1; snContextIndex > 0; --snContextIndex) { pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[snContextIndex] = __GetWordIndexFromString(pLingInfo, &pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextIndex - 1]); } { ET9SimpleWord sSimpleWord; ET9Assert(pWord->Body.wPrefixLen <= ET9MAXWORDSIZE); sSimpleWord.wLen = pWord->Body.wPrefixLen; sSimpleWord.wCompLen = 0; if (sSimpleWord.wLen) { _ET9SymCopy(sSimpleWord.sString, pWord->Base.sWord, sSimpleWord.wLen); } pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[snContextIndex] = __GetWordIndexFromString(pLingInfo, &sSimpleWord); } WLOG7(fprintf(pLogFile7, "__TagContextWordIndex (1), CW %u, PCW %u, PPCW %u\n", pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[0], pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[1], pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[2]);) } else { /* Context word index for regular tokens */ for (snContextIndex = ET9NLM_CONTEXT_COUNT - 1; snContextIndex >= 0; --snContextIndex) { pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[snContextIndex] = __GetWordIndexFromString(pLingInfo, &pLingCmnInfo->Private.sCurrContext.pIntContextWords[snContextIndex]); } WLOG7(fprintf(pLogFile7, "__TagContextWordIndex (2), CW %u, PCW %u, PPCW %u\n", pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[0], pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[1], pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[2]);) } #if defined(ET9_DEBUG) || defined(ET9_DEBUGLOG3) { ET9UINT nContextIndex; for (nContextIndex = 0; nContextIndex < ET9NLM_CONTEXT_COUNT; ++nContextIndex) { ET9AWPrivWordInfo sWord; ET9U32 dwWordIndex = pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[nContextIndex]; if (dwWordIndex < ALDB.header.dwWordCount) { _ET9AWLdbWordsByIndex(pLingInfo, pLingCmnInfo->dwLdbNum, &dwWordIndex, 1, 0, &sWord); ET9Assert(dwWordIndex == pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[nContextIndex]); } else { ET9UINT nIndex; char const *pcString; switch (dwWordIndex) { case ET9UNKNOWN_INDEX: pcString = ""; break; case ET9NUMERIC_INDEX: pcString = ""; break; case ET9NULL_INDEX: pcString = ""; break; default: pcString = "<\?\?\?>"; break; } _InitPrivWordInfo(&sWord); for (nIndex = 0; pcString[nIndex]; ++nIndex) { sWord.Base.sWord[sWord.Base.wWordLen++] = (unsigned char)pcString[nIndex]; } } #ifdef ET9_DEBUG _ET9PrivWordToSimpleWord(&sWord, &pLingCmnInfo->Private.sCurrContext.pReverseContextWords[nContextIndex]); #endif #ifdef ET9_DEBUGLOG3 { ET9U16 wCount; ET9SYMB *psSymb; WLOG3(fprintf(pLogFile3, "__TagContextWordIndex, nContextIndex %u, dwWordIndex %u, string-len %u, string ", nContextIndex, dwWordIndex, sWord.Base.wWordLen);) for (psSymb = sWord.Base.sWord, wCount = sWord.Base.wWordLen; wCount; ++psSymb, --wCount) { if (*psSymb >= 0x20 && *psSymb <= 0x7F) { fprintf(pLogFile3, "%c", (char)*psSymb); } else { fprintf(pLogFile3, "<%x>", (int)*psSymb); } } WLOG3(fprintf(pLogFile3, "\n", nContextIndex, dwWordIndex);) } #endif } } #endif WLOG3(fprintf(pLogFile3, "__TagContextWordIndex, done\n");) WLOG7(fprintf(pLogFile7, "__TagContextWordIndex, done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __TagContextClass(ET9AWLingInfo * const pLingInfo) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9Assert(pLingInfo); WLOG7(fprintf(pLogFile7, "__TagContextClass, pdwContextWordIndexes[0] %u\n", pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[0]);) if (!pLingCmnInfo->Private.bUsingLM || !pLingCmnInfo->Private.ALdbLM.bSupported) { pLingCmnInfo->Private.sCurrContext.wContextWordClass = 0; } else if (pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[0] >= ET9UNKNOWN_INDEX) { pLingCmnInfo->Private.sCurrContext.wContextWordClass = 0; } else { pLingCmnInfo->Private.sCurrContext.wContextWordClass = __ET9AWLdbGetWordClass(pLingInfo, pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[0]); } WLOG7(fprintf(pLogFile7, "__TagContextClass, done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ ET9STATUS ET9FARCALL _ET9AWLdbTagContext(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9AWPrivWordInfo const * const pWord) { ET9STATUS eStatus; WLOG3(fprintf(pLogFile3, "_ET9AWLdbTagContext\n");) WLOG7(fprintf(pLogFile7, "_ET9AWLdbTagContext\n");) eStatus = __AssureActiveLDB(pLingInfo, dwLdbNum); if (eStatus) { return eStatus; } __ClearNLMCache(pLingInfo); __TagContextWordIndex(pLingInfo, pWord); __TagContextClass(pLingInfo); WLOG3(fprintf(pLogFile3, "_ET9AWLdbTagContext, done\n");) WLOG7(fprintf(pLogFile7, "_ET9AWLdbTagContext, done\n");) return ET9STATUS_NONE; } #if 0 /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static ET9BOOL ET9LOCALCALL __ET9AWBinarySearchWordIndex(ET9U16 const * const pList, const ET9U16 wWordIndex, const ET9INT nEntryCount, ET9U16 * const pwSuffixOffset) { ET9INT nLow = 0; ET9INT nHigh = nEntryCount - 1; while (nLow <= nHigh) { const ET9INT nMid = (nLow + nHigh) / 2; const ET9U16 wMidWordIndex = pList[nMid]; if (wMidWordIndex > wWordIndex) { nHigh = nMid - 1; } else if (wMidWordIndex < wWordIndex) { nLow = nMid + 1; } else { *pwSuffixOffset = (ET9U16) nMid; return 1; } } return 0; } #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * / */ static ET9BOOL ET9LOCALCALL __ET9AWLMBinarySearchWordIndex(ET9AWLingInfo * const pLingInfo, const ET9U32 dwCurrentAddress, const ET9U32 dwWordIndex, const ET9U32 dwEntryCount, ET9U32 * const pdwTargetIndex, ET9U32 * const pdwTargetAddress) { const ET9U32 dwDefaultValue = 0xFFFFFFCD; ET9S32 sdwLow; ET9S32 sdwHigh; ET9Assert(pdwTargetIndex); ET9Assert(pdwTargetAddress); *pdwTargetIndex = dwDefaultValue; *pdwTargetAddress = dwDefaultValue; if (!dwEntryCount) { return 0; } sdwLow = 0; sdwHigh = dwEntryCount - 1; while (sdwLow <= sdwHigh) { const ET9S32 sdwMid = (ET9S32)((sdwLow + sdwHigh) / 2); const ET9U32 dwMidWordIndexAddress = (ET9U32)dwCurrentAddress + sdwMid * ET9WORD_INDEX_SIZE; const ET9U32 dwMidWordIndex = _ET9ReadLDBWord3(pLingInfo, dwMidWordIndexAddress); if (dwMidWordIndex > dwWordIndex) { sdwHigh = sdwMid - 1; } else if (dwMidWordIndex < dwWordIndex) { sdwLow = sdwMid + 1; } else { *pdwTargetIndex = (ET9U32)sdwMid; *pdwTargetAddress = dwMidWordIndexAddress; return 1; } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __ET9AWLMRootNodeSearchWord(ET9AWLingInfo * const pLingInfo, const ET9U32 dwWordIndex) { ET9ANLM_Info * const pANLM_Info = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[0]; ET9ANLM_Cache * const pANLM_Cache = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[0]; const ET9U32 dwNumEntries = pANLM_Info->dwInfoNumEntries; ET9U32 dwEntryIndex; ET9U32 dwCurrAddress; ET9U32 dwCacheCurrAddress; ET9Assert(pLingInfo); WLOG7(fprintf(pLogFile7, "__ET9AWLMRootNodeSearchWord, dwWordIndex %u\n", dwWordIndex);) if (pANLM_Cache->ANLM_Cache_Data[0].dwIndex == dwWordIndex) { WLOG7(fprintf(pLogFile7, " found in cache\n");) return; } dwCurrAddress = pANLM_Info->dwInfoStartAddress; if (__ET9AWLMBinarySearchWordIndex(pLingInfo, dwCurrAddress, dwWordIndex, dwNumEntries, &dwEntryIndex, &dwCacheCurrAddress)) { ET9U16 wNumChildren = 0; ET9U32 dwChildPtr = 0; WLOG7(fprintf(pLogFile7, " found @ index %u\n", dwEntryIndex);) dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] + dwEntryIndex * ET9CHILDREN_CT_SIZE; wNumChildren = _ET9ReadLDBWord2(pLingInfo, dwCurrAddress); dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILD_PTR_OFFSET_INDEX] + dwEntryIndex * ET9CHILD_PTR_SIZE; dwChildPtr = _ET9ReadLDBWord3(pLingInfo, dwCurrAddress); pANLM_Cache->ANLM_Cache_Data[0].dwIndex = dwWordIndex; pANLM_Cache->ANLM_Cache_Data[0].dwAddress = dwCacheCurrAddress; pANLM_Cache->ANLM_Cache_Data[0].dwNextAddress = pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[1].dwInfoStartAddress + dwChildPtr * ET9WORD_INDEX_SIZE; pANLM_Cache->ANLM_Cache_Data[0].wLinkCount = wNumChildren; return; } WLOG7(fprintf(pLogFile7, " not found\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __ET9AWLMIntermediateNodeSearchWord(ET9AWLingInfo * const pLingInfo, const ET9U8 bBackOffOrder, const ET9U8 bOrderIndex, const ET9U32 dwWordIndex) { const ET9U8 bNLMOrder = pLingInfo->pLingCmnInfo->Private.ALdbNLM.bOrder; const ET9U8 bLMInfoIndex = bNLMOrder - bOrderIndex; const ET9U8 bCacheIndex = bNLMOrder - bBackOffOrder; ET9ANLM_Cache * const pANLM_Cache = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bCacheIndex]; ET9ANLM_Info * const pANLM_Info = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex]; const ET9U32 dwIndexInTable = (ET9U32)(pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress - pANLM_Info->dwInfoStartAddress)/ET9WORD_INDEX_SIZE; ET9U32 dwNumCandidateEntries; ET9U32 dwCurrAddress; ET9U32 dwEntryIndex; ET9U32 dwCacheCurrAddress; ET9Assert(pLingInfo); ET9Assert(bNLMOrder > 3); ET9Assert(bNLMOrder >= bBackOffOrder); ET9Assert(bBackOffOrder >= bOrderIndex); WLOG7(fprintf(pLogFile7, "__ET9AWLMIntermediateNodeSearchWord, bBackOffOrder %u bOrderIndex % u dwWordIndex %u\n", bBackOffOrder, bOrderIndex, dwWordIndex);) if (pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwIndex == dwWordIndex) { WLOG7(fprintf(pLogFile7, " found in cache\n");) return; } if (bBackOffOrder != bOrderIndex) { dwNumCandidateEntries = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].wLinkCount; dwCurrAddress = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress; } else { dwNumCandidateEntries = pANLM_Info->dwCurrOrderNumEntries; dwCurrAddress = pANLM_Info->dwCurrOrderStartAddress; } if (__ET9AWLMBinarySearchWordIndex(pLingInfo, dwCurrAddress, dwWordIndex, dwNumCandidateEntries, &dwEntryIndex, &dwCacheCurrAddress)) { ET9U32 dwChildPtr = 0; ET9U32 dwNumEntries = pANLM_Info->dwInfoNumEntries; ET9U16 wNumChildren = 0; WLOG7(fprintf(pLogFile7, " found @ index %u\n", dwEntryIndex);) if (bBackOffOrder == bOrderIndex) { dwEntryIndex += (dwNumEntries - dwNumCandidateEntries); } else { dwEntryIndex += dwIndexInTable; } dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] + dwEntryIndex * ET9CHILDREN_CT_SIZE; wNumChildren = _ET9ReadLDBWord2(pLingInfo, dwCurrAddress); dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILD_PTR_OFFSET_INDEX] + dwEntryIndex * ET9CHILD_PTR_SIZE; dwChildPtr = _ET9ReadLDBWord3(pLingInfo, dwCurrAddress); pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwIndex = dwWordIndex; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwAddress = dwCacheCurrAddress; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwNextAddress = pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex + 1].dwInfoStartAddress + dwChildPtr * ET9WORD_INDEX_SIZE; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].wLinkCount = wNumChildren; return; } WLOG7(fprintf(pLogFile7, " not found\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __ET9AWLMBackoffNodeSearchWord(ET9AWLingInfo * const pLingInfo, const ET9U8 bBackOffOrder, const ET9U8 bOrderIndex, const ET9U32 dwWordIndex) { const ET9U8 bNLMOrder = pLingInfo->pLingCmnInfo->Private.ALdbNLM.bOrder; const ET9U8 bLMInfoIndex = bNLMOrder - bOrderIndex; const ET9U8 bCacheIndex = bNLMOrder - bBackOffOrder; ET9ANLM_Cache * const pANLM_Cache = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bCacheIndex]; ET9ANLM_Info * const pANLM_Info_Constant = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bCacheIndex]; ET9ANLM_Info * const pANLM_Info = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex]; const ET9U32 dwNumEntries = pANLM_Info->dwInfoNumEntries; const ET9U32 dwIndexInTable = (ET9U32)(pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress - pANLM_Info->dwInfoStartAddress) / ET9WORD_INDEX_SIZE; ET9U32 dwNumCandidateEntries; ET9U32 dwCurrAddress; ET9U32 dwEntryIndex; ET9U32 dwCacheCurrAddress; ET9Assert(bNLMOrder >= bBackOffOrder); ET9Assert(bBackOffOrder >= bOrderIndex); WLOG7(fprintf(pLogFile7, "__ET9AWLMBackoffNodeSearchWord, bBackOffOrder %u bOrderIndex % u dwWordIndex %u\n", bBackOffOrder, bOrderIndex, dwWordIndex);) if (pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwIndex == dwWordIndex) { WLOG7(fprintf(pLogFile7, " found in cache\n");) return; } if (bBackOffOrder != bOrderIndex) { dwNumCandidateEntries = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].wLinkCount; dwCurrAddress = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress; } else { dwNumCandidateEntries = pANLM_Info->dwCurrOrderNumEntries; dwCurrAddress = pANLM_Info->dwCurrOrderStartAddress; } if (__ET9AWLMBinarySearchWordIndex(pLingInfo, dwCurrAddress, dwWordIndex, dwNumCandidateEntries, &dwEntryIndex, &dwCacheCurrAddress)) { const ET9U8 bBackOffBits = pLingInfo->pLingCmnInfo->Private.ALdbNLM.bBWtBits; ET9BOOL bValueError = 0; ET9U16 wNumChildren; ET9U32 dwChildPtr; ET9U32 dwBackOffWeight; WLOG7(fprintf(pLogFile7, " found @ index %u\n", dwEntryIndex);) dwCacheCurrAddress = dwCurrAddress; if (bBackOffOrder == bOrderIndex) { dwEntryIndex += (dwNumEntries - dwNumCandidateEntries); } else { dwEntryIndex += dwIndexInTable; } dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] + dwEntryIndex * ET9CHILDREN_CT_SIZE; wNumChildren = _ET9ReadLDBWord2(pLingInfo, dwCurrAddress); dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILD_PTR_OFFSET_INDEX] + dwEntryIndex * ET9CHILD_PTR_SIZE; dwChildPtr = _ET9ReadLDBWord3(pLingInfo, dwCurrAddress); dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9BACKOFF_WT_OFFSET_INDEX]; if (bBackOffBits == 8) { dwBackOffWeight = _ET9ReadLDBByte(pLingInfo, dwCurrAddress + dwEntryIndex); } else if (bBackOffBits == 4) { dwBackOffWeight = _ET9ReadLDBByte(pLingInfo, dwCurrAddress + (dwEntryIndex >> 1)); /* odd is trailing and even is leading */ if (dwEntryIndex % 2) { dwBackOffWeight &= 0x0F; } else { dwBackOffWeight >>= 4; } } else if (bBackOffBits == 16) { dwBackOffWeight = _ET9ReadLDBWord2(pLingInfo, dwCurrAddress + (dwEntryIndex << 1)); } else { dwBackOffWeight = 0; bValueError = 1; } if (bValueError) { pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwIndex = 0; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwAddress = 0; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwNextAddress = 0; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].wLinkCount = 0; pANLM_Cache->pxBackOffWt[bLMInfoIndex] = 0; } else { pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwIndex = dwWordIndex; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwAddress = dwCacheCurrAddress; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].dwNextAddress = pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex + 1].dwInfoStartAddress + dwChildPtr * ET9WORD_INDEX_SIZE; pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex].wLinkCount = wNumChildren; #if 0 pANLM_Cache->pxBackOffWt[bLMInfoIndex] = (ET9FREQPART)(dwBackOffWeight - pANLM_Info_Constant->bBWtAddConstant) / pANLM_Info_Constant->wBWtScaleFactor; #else pANLM_Cache->pxBackOffWt[bLMInfoIndex] = (ET9FREQPART)(dwBackOffWeight - pANLM_Info_Constant->xBWtAddConstant) / pANLM_Info_Constant->xBWtScaleFactor; #endif } return; } WLOG7(fprintf(pLogFile7, " not found\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static void ET9LOCALCALL __ET9AWLMLeafNodeSearchWord(ET9AWLingInfo * const pLingInfo, const ET9U8 bBackOffOrder, const ET9U8 bOrderIndex, const ET9U32 dwWordIndex, ET9FREQPART * const pxProb, ET9U8 * const pbFound) { const ET9U8 bNLMOrder = pLingInfo->pLingCmnInfo->Private.ALdbNLM.bOrder; const ET9U8 bLMInfoIndex = bNLMOrder - bOrderIndex; const ET9U8 bCacheIndex = bNLMOrder - bBackOffOrder; ET9ANLM_Cache * const pANLM_Cache = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bCacheIndex]; ET9ANLM_Info * const pANLM_Info_Constant = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bCacheIndex]; ET9ANLM_Info * const pANLM_Info = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex]; const ET9U32 dwStartAddress = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress; const ET9U32 dwNumEntries = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].wLinkCount; ET9U32 dwCurrAddress; ET9U32 dwEntryIndex; ET9U32 dwIndexInTable; ET9U32 dwProb = 0; ET9FREQPART xProb = 0; ET9Assert(bNLMOrder >= bBackOffOrder); ET9Assert(bBackOffOrder >= bOrderIndex); WLOG7(fprintf(pLogFile7, "__ET9AWLMLeafNodeSearchWord, bBackOffOrder %u bOrderIndex % u dwWordIndex %u\n", bBackOffOrder, bOrderIndex, dwWordIndex);) dwIndexInTable = (ET9U32) (pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress - pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex].dwInfoStartAddress)/ET9WORD_INDEX_SIZE; dwCurrAddress = dwStartAddress; *pbFound = 0; if (__ET9AWLMBinarySearchWordIndex(pLingInfo, dwCurrAddress, dwWordIndex, dwNumEntries, &dwEntryIndex, &dwCurrAddress)) { const ET9U8 bProbBits = pLingInfo->pLingCmnInfo->Private.ALdbNLM.bPrBits; WLOG7(fprintf(pLogFile7, " found @ index %u\n", dwEntryIndex);) dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9PROBABILITY_OFFSET_INDEX]; if (bProbBits == 8) { dwProb = _ET9ReadLDBByte(pLingInfo, dwCurrAddress + dwIndexInTable + dwEntryIndex); } else if (bProbBits == 4) { dwProb = _ET9ReadLDBByte(pLingInfo, dwCurrAddress + ((dwIndexInTable + dwEntryIndex) >> 1)); /* odd is trailing and even is leading */ if (dwEntryIndex % 2) { dwProb &= 0x0F; } else { dwProb >>= 4; } } else if (bProbBits == 16) { dwProb = _ET9ReadLDBWord2(pLingInfo, dwCurrAddress + ((dwIndexInTable + dwEntryIndex) << 1)); } else { return; } *pbFound = 1; xProb = (ET9FREQPART)dwProb; #if 0 xProb += pANLM_Info_Constant->bPrAddConstant; xProb /= pANLM_Info_Constant->wPrScaleFactor; #else xProb -= pANLM_Info_Constant->xPrAddConstant; xProb /= pANLM_Info_Constant->xPrScaleFactor; #endif *pxProb = xProb; return; } WLOG7(fprintf(pLogFile7, " not found\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ static void ET9LOCALCALL __ET9AWLMSearchWord(ET9AWLingInfo * const pLingInfo, const ET9U8 bBackOffOrder, const ET9U8 bOrderIndex, const ET9U32 dwWordIndex, ET9FREQPART * const pxProb, ET9U8 * const pbFound) { const ET9U8 bNLMOrder = pLingInfo->pLingCmnInfo->Private.ALdbNLM.bOrder; ET9Assert(bOrderIndex <= bNLMOrder); WLOG7(fprintf(pLogFile7, "__ET9AWLMSearchWord, bBackOffOrder %u bOrderIndex % u dwWordIndex %u\n", bBackOffOrder, bOrderIndex, dwWordIndex);) if (bOrderIndex == bNLMOrder && bOrderIndex > 2) { __ET9AWLMRootNodeSearchWord(pLingInfo, dwWordIndex); } else if (bOrderIndex == 2) { __ET9AWLMBackoffNodeSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex); } else if (bOrderIndex == 1) { __ET9AWLMLeafNodeSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex, pxProb, pbFound); } else { __ET9AWLMIntermediateNodeSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex); } WLOG7(fprintf(pLogFile7, "__ET9AWLMSearchWord, done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * * * * * */ static ET9STATUS ET9LOCALCALL __LdbGetWordFreq(ET9AWLingInfo * const pLingInfo, const ET9U32 dwIndex, const ET9U16 wContextClass, ET9FREQPART * const pxWordFreq, ET9U16 * const pwEWordFreq, ET9U16 * const pwTWordFreq) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9Assert(pLingInfo); ET9Assert(pxWordFreq); ET9Assert(pwEWordFreq); ET9Assert(pwTWordFreq); ET9Assert(!pLingCmnInfo->Private.ALdbMGD.bSupported); /* currently not supported for MGD */ *pxWordFreq = 0; *pwEWordFreq = 0; *pwTWordFreq = 0; if (pLingInfo->Private.wLDBInitOK != ET9GOODSETUP) { return ET9STATUS_NO_DB_INIT; } WLOG7(fprintf(pLogFile7, "__LdbGetWordFreq, dwIndex %u\n", dwIndex);) if (pLingCmnInfo->Private.bUsingLM && pLingCmnInfo->Private.ALdbLM.bSupported) { ET9U16 wCurrentWordClass = 0; ET9U16 wEmissionScore = 0; ET9U16 wTransitionScore = 0; ET9U32 dwAddressTracker = 0; ET9U32 dwTransitionTableEntry = 0; const ET9U16 wMaxFreq = pLingCmnInfo->Private.ALdbLM.wEBits > pLingCmnInfo->Private.ALdbLM.wTBits ? (ET9U16) (1 << (pLingCmnInfo->Private.ALdbLM.wEBits + 1)) : (ET9U16) (1 << (pLingCmnInfo->Private.ALdbLM.wTBits + 1)); WLOG7(fprintf(pLogFile7, " ALdbLM.dwNumEntries %u\n", pLingInfo->pLingCmnInfo->Private.ALdbLM.dwNumEntries);) ET9Assert(dwIndex < pLingInfo->pLingCmnInfo->Private.ALdbLM.dwNumEntries); /* Get word class */ wCurrentWordClass = __ET9AWLdbGetWordClass(pLingInfo, dwIndex); if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 256) { dwAddressTracker = pLingCmnInfo->Private.ALdbLM.dwALMStartAddress + pLingCmnInfo->Private.ALdbLM.dwNumEntries; } else if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 512) { dwAddressTracker = pLingCmnInfo->Private.ALdbLM.dwALMStartAddress + (pLingCmnInfo->Private.ALdbLM.dwNumEntries << 1); } if (pLingCmnInfo->Private.ALdbLM.wEBits == 4) { wEmissionScore = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + (dwIndex >> 1)); /* odd is trailing and even is leading */ if (dwIndex % 2) { wEmissionScore &= 0x0F; } else { wEmissionScore >>= 4; } } else if (pLingCmnInfo->Private.ALdbLM.wEBits == 8) { wEmissionScore = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + dwIndex); } else if (pLingCmnInfo->Private.ALdbLM.wEBits == 16) { wEmissionScore = _ET9ReadLDBWord2(pLingInfo, dwAddressTracker + (dwIndex << 1)); } else { return ET9STATUS_ERROR; } dwAddressTracker += ((pLingCmnInfo->Private.ALdbLM.dwNumEntries * pLingCmnInfo->Private.ALdbLM.wEBits) >> 3); if (pLingCmnInfo->Private.ALdbLM.wEBits == 4 && pLingCmnInfo->Private.ALdbLM.dwNumEntries % 2) { ++dwAddressTracker; } /* Get transition score */ dwTransitionTableEntry = wContextClass * pLingCmnInfo->Private.ALdbLM.wNumClasses + wCurrentWordClass; if (pLingCmnInfo->Private.ALdbLM.wTBits == 4) { wTransitionScore = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + (dwTransitionTableEntry >> 1)); /* odd is trailing and even is leading */ if (dwTransitionTableEntry % 2) { wTransitionScore &= 0x0F; } else { wTransitionScore >>= 4; } } else if (pLingCmnInfo->Private.ALdbLM.wTBits == 8) { wTransitionScore = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + dwTransitionTableEntry); } else if (pLingCmnInfo->Private.ALdbLM.wTBits == 16) { wTransitionScore = _ET9ReadLDBWord2(pLingInfo, dwAddressTracker + (dwTransitionTableEntry << 1)); } else { return ET9STATUS_ERROR; } if (pLingCmnInfo->Private.ALdbLM.wTBits > pLingCmnInfo->Private.ALdbLM.wEBits) { wEmissionScore <<= (pLingCmnInfo->Private.ALdbLM.wTBits - pLingCmnInfo->Private.ALdbLM.wEBits); } else if (pLingCmnInfo->Private.ALdbLM.wEBits > pLingCmnInfo->Private.ALdbLM.wTBits) { wTransitionScore <<= (pLingCmnInfo->Private.ALdbLM.wEBits - pLingCmnInfo->Private.ALdbLM.wTBits); } #ifdef ET9_USE_FLOAT_FREQS { const ET9FREQPART xScalingFactor = pLingCmnInfo->Private.ALdbLM.xScalingFactor; const ET9FREQPART xAddConstant = pLingCmnInfo->Private.ALdbLM.xAddConstant; ET9FREQPART xEFreq; ET9FREQPART xTFreq; if (pLingCmnInfo->Private.ALdbNLM.bSupported) { xEFreq = _ET9pow_f(__fEulerNumberE, (ET9FLOAT)((wEmissionScore - xAddConstant) / xScalingFactor)); xTFreq = _ET9pow_f(__fEulerNumberE, (ET9FLOAT)((wTransitionScore - xAddConstant) / xScalingFactor)); } else { xEFreq = _ET9pow_f(__fEulerNumberE, (ET9FLOAT)((wEmissionScore - xAddConstant) / -xScalingFactor)); xTFreq = _ET9pow_f(__fEulerNumberE, (ET9FLOAT)((wTransitionScore - xAddConstant) / -xScalingFactor)); } *pxWordFreq = (ET9FREQPART)ET9_DB_MAX_FREQ * xEFreq * xTFreq; } #endif *pwEWordFreq = wMaxFreq - wEmissionScore; *pwTWordFreq = wMaxFreq - wTransitionScore; } else { /* fall back to basics */ *pxWordFreq = (ET9FREQPART)((ET9FREQPART)ET9_DB_MAX_FREQ / (dwIndex + 1)); } ET9Assert(*pxWordFreq >= 0); return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ void ET9FARCALL _ET9AWLMGetWordFreqByIndex(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9U32 dwIndex, ET9FREQPART * const pxFreq, ET9U8 * const pbNLMOrder) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U8 bFound = 0; ET9U32 dwProb = 0; ET9FREQPART xProb = 0; ET9U32 dwWordIndex = 16777215; ET9FREQPART xBackOffWt = 0; ET9Assert(pLingInfo); *pxFreq = 0; *pbNLMOrder = 0; if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { return; } if (ET9CONTEXTBASEDPREDICTION(pLingCmnInfo) && pLingCmnInfo->Private.ALdbNLM.bSupported && pLingInfo->Private.wLDBInitOK == ET9GOODSETUP) { const ET9U8 bMaxOrder = pLingCmnInfo->Private.ALdbNLM.bOrder; ET9U8 bBackOffOrder; ET9Assert(bMaxOrder <= ET9NLM_MAX_ORDER); dwProb = 0; xProb = 0; for (bBackOffOrder = bMaxOrder; bBackOffOrder > 1; --bBackOffOrder) { ET9U8 bOrderIndex; for (bOrderIndex = bBackOffOrder; bOrderIndex > 0; --bOrderIndex) { if (bOrderIndex <= 1) { dwWordIndex = dwIndex; } else { dwWordIndex = pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[bOrderIndex - 2]; } if (dwWordIndex == ET9NULL_INDEX) { break; } __ET9AWLMSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex, &xProb, &bFound); if (bFound) { *pbNLMOrder = bBackOffOrder; break; } } if (bFound) { xProb += xBackOffWt; break; } else { xBackOffWt += pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bMaxOrder - bBackOffOrder].pxBackOffWt[bMaxOrder - 2]; } } } if (!bFound) { ET9FREQPART xWordFreq = 0; ET9U16 wEWordFreq = 0; ET9U16 wTWordFreq = 0; __LdbGetWordFreq(pLingInfo, dwIndex, pLingCmnInfo->Private.sCurrContext.wContextWordClass, &xWordFreq, &wEWordFreq, &wTWordFreq); *pxFreq = (ET9FREQPART)(xWordFreq * _ET9pow_f(__fEulerNumberE, (ET9FLOAT)xBackOffWt)); } else { #ifdef ET9_USE_FLOAT_FREQS { *pxFreq = (ET9FREQPART)ET9_DB_MAX_FREQ * (ET9FREQPART)_ET9pow_f(__fEulerNumberE, (ET9FLOAT)xProb); } #else { /* to-do(erb): Fix frequency. */ xFreq = (ET9FREQPART)dwProb; } #endif } } /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * * */ void ET9FARCALL _ET9AWLMGetWordFreq(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9AWPrivWordInfo * const pWord) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U32 dwIndex = pWord->Body.dwWordIndex - 1; ET9U8 bFound = 0; ET9U32 dwProb = 0; ET9FREQPART xProb = 0; ET9U32 dwWordIndex = 16777215; ET9FREQPART xBackOffWt = 0; ET9Assert(pLingInfo); ET9Assert(pWord->Body.dwWordIndex); pWord->Body.wEWordFreq = 0; pWord->Body.wTWordFreq = 0; pWord->Body.bNLMOrder = 0; if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { return; } if (ET9CONTEXTBASEDPREDICTION(pLingCmnInfo) && pLingCmnInfo->Private.ALdbNLM.bSupported && pLingInfo->Private.wLDBInitOK == ET9GOODSETUP) { const ET9U8 bMaxOrder = pLingCmnInfo->Private.ALdbNLM.bOrder; ET9U8 bBackOffOrder; ET9Assert(bMaxOrder <= ET9NLM_MAX_ORDER); dwProb = 0; xProb = 0; for (bBackOffOrder = bMaxOrder; bBackOffOrder > 1; --bBackOffOrder) { ET9U8 bOrderIndex; for (bOrderIndex = bBackOffOrder; bOrderIndex > 0; --bOrderIndex) { if (bOrderIndex <= 1) { dwWordIndex = dwIndex; } else { dwWordIndex = pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[bOrderIndex - 2]; } if (dwWordIndex == ET9NULL_INDEX) { break; } __ET9AWLMSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex, &xProb, &bFound); if (bFound) { pWord->Body.bNLMOrder = bBackOffOrder; break; } } if (bFound) { xProb += xBackOffWt; break; } else { xBackOffWt += pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bMaxOrder - bBackOffOrder].pxBackOffWt[bMaxOrder - 2]; } } } if (bFound) { #ifdef ET9_USE_FLOAT_FREQS { pWord->Body.xWordFreq = (ET9FREQPART)ET9_DB_MAX_FREQ * (ET9FREQPART)_ET9pow_f(__fEulerNumberE, (ET9FLOAT)xProb); } #else { /* to-do(erb): Fix frequency. */ pWord->Body.xWordFreq = (ET9FREQPART)dwProb; } #endif } else { /* __LdbGetWordFreq does currently not support MGD */ if (pLingCmnInfo->Private.ALdbMGD.bSupported) { pWord->Body.xWordFreq = pWord->Body.xWordFreqOrg; } else { __LdbGetWordFreq(pLingInfo, dwIndex, pLingCmnInfo->Private.sCurrContext.wContextWordClass, &pWord->Body.xWordFreq, &pWord->Body.wEWordFreq, &pWord->Body.wTWordFreq); } pWord->Body.xWordFreq = (ET9FREQPART)(pWord->Body.xWordFreq * _ET9pow_f(__fEulerNumberE, (ET9FLOAT)xBackOffWt)); } #ifdef ET9_DEBUG pWord->Body.sScoreContext = pLingInfo->pLingCmnInfo->Private.sCurrContext; #endif #if 0 /* LM debug */ { static int nReportCount = 0; static int nCounts[4] = { 0 }; static float fMinVals[4] = { 0 }; static float fMaxVals[4] = { 0 }; static int nValueCount = 0; static valueItem pValues[10000000]; if (nValueCount < 10000000) { pValues[nValueCount].bOrder = pWord->Body.bNLMOrder; pValues[nValueCount].fValue = (float)pWord->Body.xWordFreq; pValues[nValueCount].nCount = 1; ++nValueCount; } if (!nCounts[pWord->Body.bNLMOrder]) { fMinVals[pWord->Body.bNLMOrder] = pWord->Body.xWordFreq; fMaxVals[pWord->Body.bNLMOrder] = pWord->Body.xWordFreq; } ++nCounts[pWord->Body.bNLMOrder]; if (fMinVals[pWord->Body.bNLMOrder] > pWord->Body.xWordFreq) { fMinVals[pWord->Body.bNLMOrder] = pWord->Body.xWordFreq; } if (fMaxVals[pWord->Body.bNLMOrder] < pWord->Body.xWordFreq) { fMaxVals[pWord->Body.bNLMOrder] = pWord->Body.xWordFreq; } if (++nReportCount % 10000 == 0) { printf("@ %5u: %5u %14.2f %14.2f, %5u %14.2f %14.2f, %5u %14.2f %14.2f, %5u %14.2f %14.2f\n", nReportCount, nCounts[0], fMinVals[0], fMaxVals[0], nCounts[1], fMinVals[1], fMaxVals[1], nCounts[2], fMinVals[2], fMaxVals[2], nCounts[3], fMinVals[3], fMaxVals[3]); } if (nReportCount == 2810000) { FILE *pf = fopen("zzzFreqs.bin", "wb"); fwrite(pValues, sizeof(valueItem), nValueCount, pf); fclose(pf); printf("\n\nwrote freqs file\n\n\n"); } } #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __LMLeafNodeAddWords(ET9AWLingInfo * const pLingInfo, const ET9U8 bBackOffOrder, const ET9U8 bOrderIndex, const ET9BOOL bLdbTagOnly) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U8 bNLMOrder = pLingCmnInfo->Private.ALdbNLM.bOrder; const ET9U8 bProbBits = pLingCmnInfo->Private.ALdbNLM.bPrBits; const ET9U8 bLMInfoIndex = bNLMOrder - bOrderIndex; const ET9U8 bCacheIndex = bNLMOrder - bBackOffOrder; ET9ANLM_Cache * const pANLM_Cache = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bCacheIndex]; ET9ANLM_Info * const pANLM_Info_Constant = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bCacheIndex]; ET9ANLM_Info * const pANLM_Info = &pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex]; const ET9U32 dwStartAddress = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress; const ET9U32 dwNumEntries = pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].wLinkCount; const ET9U32 dwIndexInTable = (ET9U32)(pANLM_Cache->ANLM_Cache_Data[bLMInfoIndex - 1].dwNextAddress - pLingInfo->pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bLMInfoIndex].dwInfoStartAddress) / ET9WORD_INDEX_SIZE; ET9U32 dwCurrAddress; ET9U32 dwEntryIndex; ET9U32 dwCurrWordIndexAddress; ET9U32 pdwCandidateIndexList[ET9MAXCOLLECTSIZE]; ET9FREQPART pxCandidateFreqList[ET9MAXCOLLECTSIZE] = {0}; ET9UINT nNumCandidatesInList = 0; ET9Assert(bNLMOrder >= bBackOffOrder); ET9Assert(bBackOffOrder >= bOrderIndex); ET9Assert(pLingCmnInfo->Private.bTotalNWP <= pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); WLOG7(fprintf(pLogFile7, "__ET9AWLMLeafNodeAddWords, bBackOffOrder %u bOrderIndex %u\n", bBackOffOrder, bOrderIndex);) if (pLingInfo->pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords >= pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize) { WLOG7(fprintf(pLogFile7, " list is full - skipping\n");) return; } dwCurrAddress = dwStartAddress; dwCurrWordIndexAddress = dwCurrAddress; for (dwEntryIndex = 0; dwEntryIndex < dwNumEntries; ++dwEntryIndex, dwCurrWordIndexAddress += ET9WORD_INDEX_SIZE) { const ET9U32 dwTestWordIndex = _ET9ReadLDBWord3(pLingInfo, dwCurrWordIndexAddress); ET9UINT i; ET9U32 dwProb; ET9FREQPART xProb; if (dwTestWordIndex >= ET9UNKNOWN_INDEX) { continue; } ET9Assert(dwTestWordIndex < ALDB.header.dwWordCount); #if 0 WLOG7(fprintf(pLogFile7, " adding %u\n", dwTestWordIndex);) #endif if (bLdbTagOnly) { __UpdateTagPriorityValue(pLingCmnInfo, dwTestWordIndex, bBackOffOrder); continue; } dwCurrAddress = pANLM_Info->ANLM_Offset.dwOffset[ET9PROBABILITY_OFFSET_INDEX]; if (bProbBits == 8) { dwProb = _ET9ReadLDBByte(pLingInfo, dwCurrAddress + dwIndexInTable + dwEntryIndex); } else if (bProbBits == 4) { dwProb = _ET9ReadLDBByte(pLingInfo, dwCurrAddress + ((dwIndexInTable + dwEntryIndex) >> 1)); /* odd is trailing and even is leading */ if (dwEntryIndex % 2) { dwProb &= 0x0F; } else { dwProb >>= 4; } } else if (bProbBits == 16) { dwProb = _ET9ReadLDBWord2(pLingInfo, dwCurrAddress + ((dwIndexInTable + dwEntryIndex) << 1)); } else { return; } xProb = (ET9FREQPART)dwProb; #if 0 xProb += pANLM_Info_Constant->bPrAddConstant; xProb /= pANLM_Info_Constant->wPrScaleFactor; #else xProb -= pANLM_Info_Constant->xPrAddConstant; xProb /= pANLM_Info_Constant->xPrScaleFactor; #endif xProb = (ET9FREQPART)_ET9pow_f(2, (ET9FLOAT)sizeof(ET9FREQPART) * 8) * (ET9FREQPART)_ET9pow_f(__fEulerNumberE, (ET9FLOAT)xProb); /* find appropriate position */ for (i = 0; i < pLingCmnInfo->Private.bTotalNWP && i < nNumCandidatesInList; ++i) { if (xProb > pxCandidateFreqList[i]) { break; } } if (i < pLingCmnInfo->Private.bTotalNWP) { if (nNumCandidatesInList) { /* shift entries down */ ET9UINT j; for (j = __ET9Min(nNumCandidatesInList, (pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize - 1)); j > i; --j) { ET9Assert(j < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); pdwCandidateIndexList[j] = pdwCandidateIndexList[j - 1]; pxCandidateFreqList[j] = pxCandidateFreqList[j - 1]; } } /* insert */ ET9Assert(i < pLingCmnInfo->Private.sWordC.pCurrC->nMaxCollectSize); pdwCandidateIndexList[i] = dwTestWordIndex; pxCandidateFreqList[i] = xProb; if (nNumCandidatesInList < pLingCmnInfo->Private.bTotalNWP) { ++nNumCandidatesInList; } } } /* Add words to list */ _ET9AWLdbWordsByIndex(pLingInfo, pLingInfo->pLingCmnInfo->dwLdbNum, pdwCandidateIndexList, nNumCandidatesInList, bBackOffOrder, NULL); WLOG7(fprintf(pLogFile7, "__ET9AWLMLeafNodeAddWords, done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ void ET9FARCALL _ET9AWLdbNwpWordsSearch(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9BOOL bLdbTagOnly) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9Assert(pLingInfo); WLOG3(fprintf(pLogFile3, "_ET9AWNLMWordsSearch\n");) WLOG7(fprintf(pLogFile7, "\n\n\n_ET9AWNLMWordsSearch\n\n");) if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { return; } if (ET9CONTEXTBASEDPREDICTION(pLingCmnInfo) && pLingCmnInfo->Private.ALdbNLM.bSupported && pLingInfo->Private.wLDBInitOK == ET9GOODSETUP) { const ET9U8 bMaxOrder = pLingCmnInfo->Private.ALdbNLM.bOrder; ET9U8 bBackOffOrder; ET9FREQPART xBackOffWt = 0; ET9Assert(bMaxOrder <= ET9NLM_MAX_ORDER); _ET9AWLdbTagContext(pLingInfo, dwLdbNum, NULL); for (bBackOffOrder = bMaxOrder; bBackOffOrder > 1; --bBackOffOrder) { ET9U8 bOrderIndex; for (bOrderIndex = bBackOffOrder; bOrderIndex > 1; --bOrderIndex) { ET9U32 dwWordIndex; WLOG7(fprintf(pLogFile7, "_ET9AWNLMWordsSearch, bBackOffOrder %u bOrderIndex %u (next)\n", bBackOffOrder, bOrderIndex);) { const ET9INT snIndex = (ET9INT)bOrderIndex - 2; if (snIndex >= 0 && snIndex <= ET9NLM_MAX_ORDER) { dwWordIndex = pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[snIndex]; } else { dwWordIndex = 16777215U; } } WLOG7(fprintf(pLogFile7, " dwWordIndex %u\n", dwWordIndex);) if (dwWordIndex == ET9NULL_INDEX) { break; } { ET9U8 bFound; ET9FREQPART xProb; __ET9AWLMSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex, &xProb, &bFound); } if (bOrderIndex == 2) { __LMLeafNodeAddWords(pLingInfo, bBackOffOrder, 1, bLdbTagOnly); } } if (bOrderIndex == 1) { xBackOffWt += pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bBackOffOrder].pxBackOffWt[bOrderIndex]; } } } WLOG3(fprintf(pLogFile3, "_ET9AWNLMWordsSearch, done\n");) WLOG7(fprintf(pLogFile7, "_ET9AWNLMWordsSearch, done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static void ET9LOCALCALL __CollectionPriorityTagCBLM(ET9AWLingInfo * const pLingInfo) { #ifdef ET9_ACTIVATE_CBLM_COLLECTION_SUPPORT ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9Assert(pLingInfo); if (pLingInfo->Private.wLDBInitOK != ET9GOODSETUP || !pLingCmnInfo->Private.bUsingLM || !pLingCmnInfo->Private.ALdbLM.bSupported || pLingCmnInfo->Private.ALdbNLM.bSupported || pLingCmnInfo->Private.ALdbLM.wNumClasses > ET9ALM_MAX_CLASS_COUNT) { return; } { ET9U16 wMinTransitionScores[3] = { 0xFFFF, 0xFFFF, 0xFFFF }; ET9U32 dwAddressTracker; if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 256) { dwAddressTracker = pLingCmnInfo->Private.ALdbLM.dwALMStartAddress + pLingCmnInfo->Private.ALdbLM.dwNumEntries; } else if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 512) { dwAddressTracker = pLingCmnInfo->Private.ALdbLM.dwALMStartAddress + (pLingCmnInfo->Private.ALdbLM.dwNumEntries << 1); } else { return; } /* emission scores */ dwAddressTracker += ((pLingCmnInfo->Private.ALdbLM.dwNumEntries * pLingCmnInfo->Private.ALdbLM.wEBits) >> 3); if (pLingCmnInfo->Private.ALdbLM.wEBits == 4 && pLingCmnInfo->Private.ALdbLM.dwNumEntries % 2) { ++dwAddressTracker; } /* Get transition scores */ { const ET9U32 dwTransitionTableStart = dwAddressTracker + (pLingCmnInfo->Private.sCurrContext.wContextWordClass * pLingCmnInfo->Private.ALdbLM.wNumClasses); ET9U16 wClassIndex; for (wClassIndex = 0; wClassIndex < pLingCmnInfo->Private.ALdbLM.wNumClasses; ++wClassIndex) { const ET9U32 dwTransitionTableEntry = dwTransitionTableStart + wClassIndex; ET9U16 wTransitionScore; if (pLingCmnInfo->Private.ALdbLM.wTBits == 4) { wTransitionScore = _ET9ReadLDBByte(pLingInfo, (dwTransitionTableEntry >> 1)); /* odd is trailing and even is leading */ if (dwTransitionTableEntry % 2) { wTransitionScore &= 0x0F; } else { wTransitionScore >>= 4; } } else if (pLingCmnInfo->Private.ALdbLM.wTBits == 8) { wTransitionScore = _ET9ReadLDBByte(pLingInfo, dwTransitionTableEntry); } else if (pLingCmnInfo->Private.ALdbLM.wTBits == 16) { wTransitionScore = _ET9ReadLDBWord2(pLingInfo, (dwTransitionTableEntry << 1)); } else { return; } pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wClassIndex] = wTransitionScore; if (wMinTransitionScores[0] > wTransitionScore) { wMinTransitionScores[2] = wMinTransitionScores[1]; wMinTransitionScores[1] = wMinTransitionScores[0]; wMinTransitionScores[0] = wTransitionScore; } else if (wMinTransitionScores[1] > wTransitionScore) { wMinTransitionScores[2] = wMinTransitionScores[1]; wMinTransitionScores[1] = wTransitionScore; } else if (wMinTransitionScores[2] > wTransitionScore) { wMinTransitionScores[2] = wTransitionScore; } } } /* replace transition scores with tag values */ { ET9U16 wClassIndex; for (wClassIndex = 0; wClassIndex < pLingCmnInfo->Private.ALdbLM.wNumClasses; ++wClassIndex) { const ET9U16 wScore = pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wClassIndex]; if (wScore == wMinTransitionScores[0]) { pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wClassIndex] = 3; } #if 0 else if (wScore == wMinTransitionScores[1]) { pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wClassIndex] = 2; } else if (wScore == wMinTransitionScores[2]) { pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wClassIndex] = 1; } #endif else { pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wClassIndex] = 0; } } } /* collect prio's */ { ET9U32 dwClassAddress = pLingCmnInfo->Private.ALdbLM.dwALMStartAddress; if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 256) { ET9U32 dwIndex; for (dwIndex = 0; dwIndex < pLingCmnInfo->Private.ALdbLM.dwNumEntries; ++dwIndex, ++dwClassAddress) { const ET9U16 wWordClass = _ET9ReadLDBByte(pLingInfo, dwClassAddress); const ET9U16 wPrio = pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wWordClass]; if (wPrio) { __UpdateTagPriorityValue(pLingCmnInfo, dwIndex, (ET9U8)wPrio); } } } else if (pLingCmnInfo->Private.ALdbLM.wNumClasses == 512) { ET9U32 dwIndex; for (dwIndex = 0; dwIndex < pLingCmnInfo->Private.ALdbLM.dwNumEntries; ++dwIndex, dwClassAddress += 2) { const ET9U16 wWordClass = _ET9ReadLDBWord2(pLingInfo, dwClassAddress); const ET9U16 wPrio = pLingCmnInfo->Private.ALdbLM.pwClassTransitions[wWordClass]; if (wPrio) { __UpdateTagPriorityValue(pLingCmnInfo, dwIndex, (ET9U8)wPrio); } } } } } #else ET9_UNUSED(pLingInfo); #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * / * * * * * */ static void ET9LOCALCALL __ET9AWLdbGetCandidates (ET9AWLingInfo * const pLingInfo, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator, const ET9U8 bSpcMode) { ET9AWPrivWordInfo sPublicWord; ET9AWPrivWordInfo sPrivateWord; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U8 bLangIndex = pLingCmnInfo->dwLdbNum == pLingInfo->pLingCmnInfo->dwFirstLdbNum ? ET9AWFIRST_LANGUAGE : ET9AWSECOND_LANGUAGE; const ET9U8 bWordSrc = ADDON_FROM_FREQ_IND(bFreqIndicator, ET9WORDSRC_LDB); /* if all LDB words being downshifted, or word being compounded, downshift it */ const ET9BOOL bDownShift = (ET9BOOL)((ET9DOWNSHIFTALLLDB(pLingCmnInfo) || wIndex) && ALDB.header.bUpperCount && _ET9_LanguageSpecific_ApplyShifting(pLingInfo, NULL)); ET9Assert(pLingInfo); if (!wLength || wLength > ET9MAXLDBWORDSIZE) { return; } if (_ET9AWCalcEditDistanceInit(pLingInfo, wIndex, wLength, bSpcMode, 0 /* bSpcFeatures */)) { return; } __STAT_AWLdb_Call; __ResetScreenCounters(); WLOG3(fprintf(pLogFile3, "__ET9AWLdbGetCandidates, start, wIndex = %2d, wLength = %2d, bSpcMode = %d\n", wIndex, wLength, bSpcMode);) _InitPrivWordInfo(&sPrivateWord); sPrivateWord.Body.bLangIndexScoring = bLangIndex; __ET9CompareStart(pLingCmnInfo, wIndex, wLength, 0 /* bUsingFlex */); __ET9SearchStart(pLingInfo, sPrivateWord.Base.sWord, &sPrivateWord.Base.wWordLen, ET9MAXWORDSIZE, 0 /* bUsingFlex */); for (; !__IsExhausted(); __ET9SearchGetNextStd(pLingInfo)) { __STAT_INC_TotLdbCount; WLOG3B(fprintf(pLogFile3, "__ET9AWLdbGetCandidates, word index = %6d, word len = %2d, wLength = %2d\n", __GetWordIndex(), sPrivateWord.Base.wWordLen, wLength);) ET9Assert(sPrivateWord.Base.wWordLen >= wLength || ET9_SPC_IS_ACTIVE(pLingCmnInfo->Private.bCurrSpcMode)); sPrivateWord.Body.bWordSrc = bWordSrc; if (__ET9AWCalcEditDistance(pLingInfo, &sPrivateWord, wIndex, wLength)) { continue; } ET9Assert(sPrivateWord.Body.xTapFreq >= 0); /* candidate found */ sPublicWord = sPrivateWord; /* downshift word if option selected */ if (bDownShift) { ET9U16 wCount = sPublicWord.Base.wWordLen; ET9SYMB *pSymb = sPublicWord.Base.sWord; for (; wCount; --wCount, ++pSymb) { *pSymb = _ET9SymToLower(*pSymb, pLingCmnInfo->dwLdbNum); } } /* prepare and add to list */ _ET9AWSelLstWordPreAdd(pLingInfo, &sPublicWord, wIndex, (ET9U8)wLength, 0, bFreqIndicator); { const ET9U32 dwCurrWordIndex = __GetWordIndex() + 1; sPublicWord.Body.xWordFreq = (ET9FREQPART)((ET9FREQPART)ET9_DB_MAX_FREQ / dwCurrWordIndex); /* zero freq becomes one in "add" */ sPublicWord.Body.wEWordFreq = 0; sPublicWord.Body.wTWordFreq = 0; sPublicWord.Body.dwWordIndex = dwCurrWordIndex; sPublicWord.Body.wSuffixIndex = 0; } ET9Assert(sPublicWord.Body.xTapFreq >= 0); ET9Assert(sPublicWord.Body.xTotFreq >= 0); sPublicWord.Base.wWordCompLen = sPrivateWord.Base.wWordCompLen; sPublicWord.Body.bIsTop5 = (__GetWordIndex() < ALDB.header.dwTopCount) ? 1 : 0; if (ALDB.tags.bActive) { sPublicWord.Body.bCollectionPrio = __GetTagPriorityValue(pLingCmnInfo, __GetWordIndex()); } _ET9AWSelLstAdd(pLingInfo, &sPublicWord, wLength, bFreqIndicator); } __LogScreenCounters(); _ET9AWCalcEditDistanceDone(pLingInfo); } /*---------------------------------------------------------------------------*/ #ifdef ET9_DEBUGLOG8 #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG8 ACTIVATED ***") #endif #include #include #define WLOG8(q) { if (pLogFile8 == NULL) { pLogFile8 = fopen("zzzET9LDBTRACE.txt", "w"); } { q fflush(pLogFile8); } } static FILE *pLogFile8 = NULL; #else #define WLOG8(q) #endif #ifdef ET9_DEBUGLOG8B #ifdef _WIN32 #pragma message ("*** ET9_DEBUGLOG8 ACTIVATED ***") #endif #define WLOG8B(q) WLOG8(q) #else #define WLOG8B(q) #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * / * * * */ static ET9STATUS ET9LOCALCALL __LDB_CalcEditDistanceFlex(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U16 wIndex, const ET9U16 wLength) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9ASPCFlexCompareData * const pCMP = &ASPC.u.sCmpDataFlex; ET9U8 const * const pbLockInfo = pCMP->pbLockInfo; ET9U8 const * const pbIsFreqPos = pCMP->pbIsFreqPos; ET9U8 const * const pbIsQualityKey = pCMP->pbIsQualityKey; const ET9U8 bMaxEditDist = pLingCmnInfo->Private.bCurrMaxEditDistance; const ET9U16 wCurrMinSourceLength = pLingCmnInfo->Private.wCurrMinSourceLength; const ET9BOOL bAllowSpcCmpl = pCMP->bAllowSpcCmpl; const ET9BOOL bAllowFreePunct = pCMP->bAllowFreePunct; const ET9BOOL bAllowFreeDouble = pCMP->bAllowFreeDouble; const ET9UINT nKeyCount = wLength; const ET9UINT nSymbCount = pWord->Base.wWordLen; const ET9UINT nQualityCount = pCMP->nQualityCount; ET9_UNUSED(wIndex); /* used in "init" */ ET9Assert(ASPC.bSpcState == ET9_SPC_STATE_FLEX_INIT_OK); _ET9AWValidateFlexArea(pLingCmnInfo); WLOG8B({ fprintf(pLogFile8, "__LDB_CalcEditDistanceFlex, spc cmpl %d, free punct %d, free double %d, word ", bAllowSpcCmpl, bAllowFreePunct, bAllowFreeDouble); { ET9U16 wIndex; for (wIndex = 0; wIndex < pWord->Base.wWordLen; ++wIndex) { fprintf(pLogFile8, "%c", pWord->Base.sWord[wIndex]); } } fprintf(pLogFile8, "\n"); }) /* screening */ __STAT_AWLdb_Flex_EDS_Call; if (nSymbCount < wCurrMinSourceLength) { WLOG8B(fprintf(pLogFile8, " no match (too short, %u < %u - %u)\n", nSymbCount, nQualityCount, bMaxEditDist);) ET9Assert(0); _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } if (nQualityCount == 1 && nSymbCount > nQualityCount) { const ET9BOOL bMatchExact = (ET9BOOL)__IsCodeExact(0, __LdbGetCharCodeAtPos(0)); if (!bMatchExact) { WLOG8B(fprintf(pLogFile8, " special completion suppression at length 1\n");) _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } /* screen calculation */ { ET9UINT nR; ET9UINT nC; ET9BOOL bInStem = 1; /* matrix */ for (nC = 1; nC <= nSymbCount; ++nC) { const ET9U16 wCharCode = __LdbGetCharCodeAtPos(nC - 1); if (bInStem) { if (pCMP->pwPrevWordSC[nC] == wCharCode) { continue; } else { bInStem = 0; } } pCMP->pwPrevWordSC[nC] = wCharCode; { const ET9BOOL bIsFreeC = (ET9BOOL)(bAllowFreePunct && !pCMP->psLockSymb[nC - 1] && __IsCodeFree(wCharCode)); ET9U8 bBestColEditDist; pCMP->ppbEditDist[0][nC - 0] = pCMP->ppbEditDist[0][nC - 1] + (bIsFreeC ? 0 : 1); bBestColEditDist = pCMP->ppbEditDist[0][nC]; for (nR = 1; nR <= nKeyCount; ++nR) { WLOG8B(fprintf(pLogFile8, "matrix %2d %2d\n", nR, nC);) /* initial */ pCMP->ppbEditDist[nR][nC] = _ET9_FLEX_TUBE_MAX_EDIT_DIST; /* pointless? */ if (pCMP->ppbEditDist[nR - 1][nC - 1] > bMaxEditDist && pCMP->ppbEditDist[nR - 1][nC - 0] > bMaxEditDist && pCMP->ppbEditDist[nR - 0][nC - 1] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " pointless (edit) [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = 0; continue; } { const ET9U8 bFreeR = !pbLockInfo[nR]; const ET9U8 bQualityR = pbIsQualityKey[nR]; const ET9U8 bMatchFreq = (ET9U8)__GetCodeFreq((nR - 1), wCharCode); /* */ pCMP->pbSubstFreqSC[nR][nC] = bMatchFreq; /* subst */ if (bMatchFreq) { WLOG8B(fprintf(pLogFile8, " subst-1, bMatchFreq %d (%u, max %u) [sc]\n", bMatchFreq, pCMP->ppbEditDist[nR - 1][nC - 1], bMaxEditDist);) pCMP->ppbEditDist[nR][nC] = pCMP->ppbEditDist[nR - 1][nC - 1]; } else if (pbLockInfo[nR] == ET9_LOCKED_SYMB_VALUE_AND_POS) { WLOG8B(fprintf(pLogFile8, " subst, non matching locked symb [sc]\n");) ET9Assert(pCMP->ppbEditDist[nR][nC] == _ET9_FLEX_TUBE_MAX_EDIT_DIST); continue; } else if (pCMP->ppbEditDist[nR - 1][nC - 1] < bMaxEditDist && bQualityR && bFreeR) { WLOG8B(fprintf(pLogFile8, " subst-2, ppbEditDist %d [sc]\n", pCMP->ppbEditDist[nR - 1][nC - 1]);) pCMP->ppbEditDist[nR][nC] = pCMP->ppbEditDist[nR - 1][nC - 1] + 1; } /* del */ if (!bQualityR) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " del-1 (%u, max %u) [sc]\n", bEditCost, bMaxEditDist);) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } else if (pCMP->ppbEditDist[nR - 1][nC - 0] < bMaxEditDist && bFreeR) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0] + 1; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " del-2 [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } /* ins */ if ((bAllowFreeDouble && bMatchFreq && bMatchFreq == pCMP->pbSubstFreqSC[nR][nC - 1]) || bIsFreeC) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-1 [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = pCMP->pbSubstFreqSC[nR][nC - 1]; pCMP->ppbEditDist[nR][nC] = bEditCost; } } else if (nR == nKeyCount && nC > nQualityCount && (!pCMP->ppbEditDist[nR - 0][nC - 1] || bAllowSpcCmpl)) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-2, nC %d, nQualityCount %d [sc]\n", nC, nQualityCount);) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } else if (pCMP->ppbEditDist[nR - 0][nC - 1] < bMaxEditDist) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1] + 1; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-3 (%u, max %u) [sc]\n", bEditCost, bMaxEditDist);) pCMP->pbSubstFreqSC[nR][nC] = bMatchFreq; pCMP->ppbEditDist[nR][nC] = bEditCost; } } /* best */ if (bBestColEditDist > pCMP->ppbEditDist[nR][nC]) { bBestColEditDist = pCMP->ppbEditDist[nR][nC]; } } } /* nR */ /* pre-empt? */ if (bBestColEditDist > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " no match (best edit) [sc], %d > %d\n", bBestColEditDist, bMaxEditDist);) pCMP->pwPrevWordSC[nC] = ALDB_CHAR_CODE_NONE; __STAT_AWLdb_Flex_EDS_Done; _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } } /* nC */ pCMP->pwPrevWordSC[nC] = ALDB_CHAR_CODE_NONE; WLOG8B({ fprintf(pLogFile8, "\n--- edit distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbEditDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n"); }) /* really a match? */ if (pCMP->ppbEditDist[nKeyCount][nSymbCount] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " no match (final edit), %d > %d (%d)\n", pCMP->ppbEditDist[nKeyCount][nSymbCount], bMaxEditDist, pCMP->ppbStemDist[nKeyCount][nSymbCount]);) __STAT_AWLdb_Flex_EDS_Done; _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } __STAT_AWLdb_Flex_EDS_Done; } /* full calculation */ __STAT_AWLdb_Flex_EDC_Call; { ET9UINT nR; ET9UINT nC; ET9BOOL bInStem = 1; /* matrix */ for (nC = 1; nC <= nSymbCount; ++nC) { const ET9U16 wCharCode = __LdbGetCharCodeAtPos(nC - 1); if (bInStem) { if (pCMP->pwPrevWordFC[nC] == wCharCode) { continue; } else { bInStem = 0; } } pCMP->pwPrevWordFC[nC] = wCharCode; { const ET9BOOL bIsFreeC = (ET9BOOL)(bAllowFreePunct && !pCMP->psLockSymb[nC - 1] && __IsCodeFree(wCharCode)); pCMP->pbComplLenFC[nC] = 0; pCMP->ppbStemDist[0][nC - 0] = pCMP->ppbEditDist[0][nC - 0]; pCMP->ppbFreeDist[0][nC - 0] = (ET9U8)(nC - pCMP->ppbEditDist[0][nC - 0]); pCMP->ppbEditDistIns[0][nC - 0] = pCMP->ppbEditDist[0][nC - 0]; for (nR = 1; nR <= nKeyCount; ++nR) { WLOG8B(fprintf(pLogFile8, "matrix %2d %2d\n", nR, nC);) /* pointless? */ if (pCMP->ppbEditDist[nR][nC] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " pointless (edit %u > %u) [full]\n", pCMP->ppbEditDist[nR][nC], bMaxEditDist);) pCMP->ppbFreeDist[nR][nC] = _ET9_FLEX_TUBE_MAX_FREE_DIST; pCMP->ppbStemDist[nR][nC] = _ET9_FLEX_TUBE_MAX_STEM_DIST; pCMP->ppxStemFreq[nR][nC] = 1; pCMP->pbSubstFreqFC[nR][nC] = 0; continue; } { const ET9U8 bFreeR = !pbLockInfo[nR]; const ET9U8 bQualityR = pbIsQualityKey[nR]; const ET9U8 bMatchFreq = (ET9U8)__GetCodeFreq((nR - 1), wCharCode); const ET9U8 bDefaultFreq = (bQualityR && nC < ET9_SPC_ED_MAX_FREQ_LEN) ? 10 : 1; ET9U8 bEditDist; ET9U8 bFreeDist; ET9U8 bStemDist; ET9FREQ xStemFreq; ET9FREQ xSymbDist; ET9U8 bEditDistSbt; ET9U8 bEditDistTrp; ET9U8 bEditDistIns; ET9U8 bEditDistDel; ET9U8 bLimitedCount; /* */ pCMP->pbSubstFreqFC[nR][nC] = bMatchFreq; /* subst */ if (bMatchFreq) { const ET9BOOL bMatchExact = (ET9BOOL)__IsCodeExact((nR - 1), wCharCode); const ET9BOOL bMatchLimited = (ET9BOOL)__IsCodeLimited((nR - 1), wCharCode); #ifdef ET9_USE_FLOAT_FREQS const ET9FREQ xFreq = (ET9FREQ)(!pbIsFreqPos[nR] ? bDefaultFreq : bMatchFreq); #else const ET9FREQ xFreq = (!pbIsFreqPos[nR] || bMatchFreq <= 17) ? 1 : ((bMatchFreq * 15) / 255); #endif const ET9FREQ xDist = (ET9FREQ)__GetSymbDistance((nR - 1), wCharCode); WLOG8B(fprintf(pLogFile8, " subst-1, bMatchFreq %d (%u, max %u) [full]\n", bMatchFreq, pCMP->ppbEditDist[nR - 1][nC - 1], bMaxEditDist);) bFreeDist = pCMP->ppbFreeDist[nR - 1][nC - 1] + (bQualityR ? 0 : 1); bEditDist = pCMP->ppbEditDist[nR - 1][nC - 1]; bStemDist = pCMP->ppbStemDist[nR - 1][nC - 1] + (bMatchExact ? 0 : 1); xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 1] * xFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 1] + xDist; bEditDistSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 1]; bEditDistTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 1]; bEditDistIns = pCMP->ppbEditDistIns[nR - 1][nC - 1]; bEditDistDel = pCMP->ppbEditDistDel[nR - 1][nC - 1]; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 1] + (bMatchLimited ? 1 : 0); } else if (pbLockInfo[nR] == ET9_LOCKED_SYMB_VALUE_AND_POS) { WLOG8B(fprintf(pLogFile8, " subst, non matching locked symb [full]\n");) ET9Assert(pCMP->ppbEditDist[nR][nC] == _ET9_FLEX_TUBE_MAX_EDIT_DIST); continue; } else if (pCMP->ppbEditDist[nR - 1][nC - 1] < bMaxEditDist && bQualityR && bFreeR) { WLOG8B(fprintf(pLogFile8, " subst-2, ppbEditDist %d [full]\n", pCMP->ppbEditDist[nR - 1][nC - 1]);) bFreeDist = pCMP->ppbFreeDist[nR - 1][nC - 1]; bEditDist = pCMP->ppbEditDist[nR - 1][nC - 1] + 1; bStemDist = pCMP->ppbStemDist[nR - 1][nC - 1] + 1; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 1] * bDefaultFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 1]; bEditDistSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 1] + 1; bEditDistTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 1]; bEditDistIns = pCMP->ppbEditDistIns[nR - 1][nC - 1]; bEditDistDel = pCMP->ppbEditDistDel[nR - 1][nC - 1]; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 1]; } else { bFreeDist = _ET9_FLEX_TUBE_MAX_FREE_DIST; bEditDist = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bStemDist = _ET9_FLEX_TUBE_MAX_STEM_DIST; xStemFreq = 1; xSymbDist = 0; bEditDistSbt = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistTrp = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistIns = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistDel = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bLimitedCount = 0; } /* del */ if (!bQualityR) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 1][nC - 0]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 1][nC - 0]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 0]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 0]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 1][nC - 0]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 1][nC - 0]; if (_ET9_Flex_IsBetterOp(-1, 0)) { WLOG8B(fprintf(pLogFile8, " del-1 [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = 0; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 0]; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 0]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 0]; } } else if (pCMP->ppbEditDist[nR - 1][nC - 0] < bMaxEditDist && bFreeR) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 1][nC - 0]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0] + 1; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 1][nC - 0] + 1; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 0]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 0]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 1][nC - 0]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 1][nC - 0] + 1; if (_ET9_Flex_IsBetterOp(-1, 0)) { WLOG8B(fprintf(pLogFile8, " del-2 [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = 0; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 0] * bDefaultFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 0]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 0]; } } /* ins */ if ((bAllowFreeDouble && bMatchFreq && bMatchFreq == pCMP->pbSubstFreqFC[nR][nC - 1]) || bIsFreeC) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1] + 1; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-1 (free) [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = pCMP->pbSubstFreqFC[nR][nC - 1]; if (nR == nKeyCount) { pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1]; } bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } else if (nR == nKeyCount && nC > nQualityCount && (!pCMP->ppbEditDist[nR - 0][nC - 1] || bAllowSpcCmpl)) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-2 (cmpl), nC %d, nQualityCount %d [full]\n", nC, nQualityCount);) pCMP->pbSubstFreqFC[nR][nC] = 0; pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1] + 1; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } else if (pCMP->ppbEditDist[nR - 0][nC - 1] < bMaxEditDist) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1] + 1; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1] + 1; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1] + 1; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-3 (spc) [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = bMatchFreq; if (nR == nKeyCount) { pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1]; } bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } /* assure same as screening */ if (bEditDist != pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " inconsistent (%u <-> %u) [full]\n", bEditDist, pCMP->ppbEditDist[nR][nC]);) } ET9Assert(bEditDist == pCMP->ppbEditDist[nR][nC]); /* persist */ pCMP->ppbFreeDist[nR][nC] = bFreeDist; pCMP->ppbStemDist[nR][nC] = bStemDist; pCMP->ppxStemFreq[nR][nC] = xStemFreq; pCMP->ppxSymbDist[nR][nC] = xSymbDist; pCMP->ppbEditDistSbt[nR][nC] = bEditDistSbt; pCMP->ppbEditDistTrp[nR][nC] = bEditDistTrp; pCMP->ppbEditDistIns[nR][nC] = bEditDistIns; pCMP->ppbEditDistDel[nR][nC] = bEditDistDel; pCMP->ppbLimitedCount[nR][nC] = bLimitedCount; } } /* nR */ } } /* nC */ pCMP->pwPrevWordFC[nC] = ALDB_CHAR_CODE_NONE; WLOG8B({ fprintf(pLogFile8, "\n--- free distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbFreeDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- edist distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbEditDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- stem distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbStemDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- edit kinds ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%x%x%x%x ", pCMP->ppbEditDistSbt[nR][nC], pCMP->ppbEditDistTrp[nR][nC], pCMP->ppbEditDistIns[nR][nC], pCMP->ppbEditDistDel[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- stem freq ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%8.0f ", pCMP->ppxStemFreq[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- completion length ---\n\n"); { fprintf(pLogFile8, " : "); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->pbComplLenFC[nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n"); }) /* assure same as screening */ ET9Assert(pCMP->ppbEditDist[nKeyCount][nSymbCount] <= bMaxEditDist); { const ET9UINT nMatchRow = nKeyCount; const ET9UINT nMatchCol = nSymbCount; const ET9BOOL bAvoidStems = (pLingCmnInfo->Private.bTraceBuild || wIndex) ? 1 : 0; ET9U8 bComplLenFC_nSymbCount = pCMP->pbComplLenFC[nMatchCol]; /* assign distance */ pWord->Body.bEditDistSpc = pCMP->ppbEditDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistStem = pCMP->ppbStemDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistFree = pCMP->ppbFreeDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcSbt = pCMP->ppbEditDistSbt[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcTrp = pCMP->ppbEditDistTrp[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcIns = pCMP->ppbEditDistIns[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcDel = pCMP->ppbEditDistDel[nMatchRow][nMatchCol]; pWord->Body.bLimitedCount = pCMP->ppbLimitedCount[nMatchRow][nMatchCol]; /* adjust to prefer spell correction over completion when applicable */ if (bComplLenFC_nSymbCount && (pWord->Body.bEditDistSpc + bComplLenFC_nSymbCount) <= bMaxEditDist && (pWord->Body.bEditDistSpc || pLingCmnInfo->Private.bTraceBuild)) { const ET9U8 bCost = bComplLenFC_nSymbCount; WLOG8B(fprintf(pLogFile8, " completion to spc term\n");) bComplLenFC_nSymbCount = 0; pWord->Body.bEditDistSpc = pWord->Body.bEditDistSpc + bCost; pWord->Body.bEditDistStem = pWord->Body.bEditDistStem + bCost; pWord->Body.bEditDistSpcIns = pWord->Body.bEditDistSpcIns + bCost; } /* suppress completions before point */ if (bComplLenFC_nSymbCount && nQualityCount < pLingCmnInfo->Private.wWordCompletionPoint) { if (bAvoidStems) { WLOG8B(fprintf(pLogFile8, " suppressed completion before point\n");) __STAT_AWLdb_Flex_EDC_Done(NULL); _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } /**/ if (bComplLenFC_nSymbCount && !pWord->Body.bEditDistSpc && (pWordSymbInfo->bNumSymbs < pLingCmnInfo->Private.wWordCompletionPoint)) { if (bAvoidStems) { WLOG8B(fprintf(pLogFile8, " assigning edit distance - before completion point\n");) pWord->Body.bEditDistSpc = 1; } } #if 1 if (pWord->Base.wWordLen > pLingCmnInfo->Private.wMaxWordLength + pWord->Body.bEditDistFree) { pWord->Body.bEditDistFree = 0; pWord->Base.wWordCompLen = 0; } else { pWord->Base.wWordCompLen = bComplLenFC_nSymbCount; } #else if (pbComplLen[nMatchCol] && (!pWord->Body.bEditDistSpc || bAllowSpcCmpl) && pWord->Base.wWordLen <= pLingCmnInfo->Private.wMaxWordLength) { pWord->Base.wWordCompLen = pbComplLen[nMatchCol]; } else { pWord->Base.wWordCompLen = 0; } #endif if (!pWord->Base.wWordCompLen && pWord->Base.wWordLen > pWordSymbInfo->bNumSymbs) { if (!pWord->Body.bEditDistSpc && pWord->Base.wWordLen <= pLingCmnInfo->Private.wMaxWordLength) { WLOG8B(fprintf(pLogFile8, " longer than input and shorter than max - special zero compl len\n");) pWord->Base.wWordCompLen = 0xFFFF; } else if (pWord->Base.wWordLen == pLingCmnInfo->Private.wMaxWordLength + pWord->Body.bEditDistFree) { WLOG8B(fprintf(pLogFile8, " longer than input and longer than max - special zero compl len\n");) pWord->Base.wWordCompLen = 0xFFFF; } } /* assign tap freq */ #ifdef ET9_USE_FLOAT_FREQS pWord->Body.xTapFreq = pCMP->ppxStemFreq[nMatchRow][nMatchCol] / pCMP->fTapFreqDivider; #else /* ET9_USE_FLOAT_FREQS */ if (nQualityCount <= 4) { pWord->Body.xTapFreq = (ET9FREQPART)pCMP->ppxStemFreq[nMatchRow][nMatchCol]; } else { pWord->Body.xTapFreq = (ET9FREQPART)(pCMP->ppxStemFreq[nMatchRow][nMatchCol] >> (4 * (nQualityCount - 4))); } #endif /* ET9_USE_FLOAT_FREQS */ if (!pWord->Body.xTapFreq) { pWord->Body.xTapFreq = 1; } ET9Assert(pWord->Body.xTapFreq >= 0); /* assign symb distance */ pWord->Body.xSymbDist = pCMP->ppxSymbDist[nMatchRow][nMatchCol]; } /* done */ WLOG8B(fprintf(pLogFile8, " match, bEditDistStem %d, bEditDistSpc %d, wWordCompLen %d, \n\n", pWord->Body.bEditDistStem, pWord->Body.bEditDistSpc, pWord->Base.wWordCompLen);) __STAT_AWLdb_Flex_EDC_Done(pWord); _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NONE; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * / * * * * * */ static void ET9LOCALCALL __ET9AWLdbGetCandidatesFlex (ET9AWLingInfo * const pLingInfo, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator, const ET9U8 bSpcMode) { ET9AWPrivWordInfo sPublicWord; ET9AWPrivWordInfo sPrivateWord; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U8 bLangIndex = pLingCmnInfo->dwLdbNum == pLingInfo->pLingCmnInfo->dwFirstLdbNum ? ET9AWFIRST_LANGUAGE : ET9AWSECOND_LANGUAGE; const ET9U8 bWordSrc = ADDON_FROM_FREQ_IND(bFreqIndicator, ET9WORDSRC_LDB); /* if all LDB words being downshifted, or word being compounded, downshift it */ const ET9BOOL bDownShift = (ET9BOOL)((ET9DOWNSHIFTALLLDB(pLingCmnInfo) || wIndex) && ALDB.header.bUpperCount && _ET9_LanguageSpecific_ApplyShifting(pLingInfo, NULL)); ET9Assert(pLingInfo); WLOG8(fprintf(pLogFile8, "\n__ET9AWLdbGetCandidatesFlex\n\n");) __STAT_AWLdb_Call; __STAT_AWLdb_Flex_Call; if (!wLength || wLength > ET9MAXWORDSIZE) { return; } if (_ET9AWCalcEditDistanceInit(pLingInfo, wIndex, wLength, bSpcMode, pLingCmnInfo->Private.ASpc.bSpcFeatures)) { return; } WLOG3(fprintf(pLogFile3, "__ET9AWLdbGetCandidatesFlex, start, wIndex = %2d, wLength = %2d, bSpcMode = %d\n", wIndex, wLength, bSpcMode);) _InitPrivWordInfo(&sPrivateWord); sPrivateWord.Body.bLangIndexScoring = bLangIndex; __ET9CompareStart(pLingCmnInfo, wIndex, wLength, 1 /* bUsingFlex */); __ET9SearchStart(pLingInfo, sPrivateWord.Base.sWord, &sPrivateWord.Base.wWordLen, ET9MAXWORDSIZE, 1 /* bUsingFlex */); for (; !__IsExhausted(); __ET9SearchGetNextFlex(pLingInfo)) { __STAT_INC_TotLdbCount; WLOG3B(fprintf(pLogFile3, "__ET9AWLdbGetCandidatesFlex, word index = %6d, word len = %2d, wLength = %2d\n", __GetWordIndex(), sPrivateWord.Base.wWordLen, wLength);) sPrivateWord.Body.bWordSrc = bWordSrc; if (__LDB_CalcEditDistanceFlex(pLingInfo, &sPrivateWord, wIndex, wLength)) { continue; } ET9Assert(sPrivateWord.Body.xTapFreq >= 0); /* candidate found */ sPublicWord = sPrivateWord; /* downshift word if option selected */ if (bDownShift) { ET9U16 wCount = sPublicWord.Base.wWordLen; ET9SYMB *pSymb = sPublicWord.Base.sWord; for (; wCount; --wCount, ++pSymb) { *pSymb = _ET9SymToLower(*pSymb, pLingCmnInfo->dwLdbNum); } } /* prepare and add to list */ __STAT_AWLdb_Flex_SLA_Call; _ET9AWSelLstWordPreAdd(pLingInfo, &sPublicWord, wIndex, (ET9U8)wLength, 0, bFreqIndicator); { const ET9U32 dwCurrWordIndex = __GetWordIndex() + 1; sPublicWord.Body.xWordFreq = (ET9FREQPART)((ET9FREQPART)ET9_DB_MAX_FREQ / dwCurrWordIndex); /* zero freq becomes one in "add" */ sPublicWord.Body.wEWordFreq = 0; sPublicWord.Body.wTWordFreq = 0; sPublicWord.Body.dwWordIndex = dwCurrWordIndex; } ET9Assert(sPublicWord.Body.xTapFreq >= 0); ET9Assert(sPublicWord.Body.xTotFreq >= 0); sPublicWord.Base.wWordCompLen = sPrivateWord.Base.wWordCompLen; sPublicWord.Body.bIsTop5 = (__GetWordIndex() < ALDB.header.dwTopCount) ? 1 : 0; if (pLingCmnInfo->Private.ALdb.tags.bActive) { sPublicWord.Body.bCollectionPrio = __GetTagPriorityValue(pLingCmnInfo, __GetWordIndex()); } _ET9AWSelLstAdd(pLingInfo, &sPublicWord, wLength, bFreqIndicator); __STAT_AWLdb_Flex_SLA_Done; } _ET9AWCalcEditDistanceDone(pLingInfo); __STAT_AWLdb_Flex_Done; } #ifdef ET9_MGD_MODULE /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ static ET9INT ET9LOCALCALL __MGD_CmpMatchLength(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9INT snIndex1, const ET9INT snIndex2) { ET9AWPrivWordInfo * const pWordList = pLingCmnInfo->Private.sWordC.pCurrC->pWordList; ET9Assert(snIndex1 >= 0 && snIndex1 < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); ET9Assert(snIndex2 >= 0 && snIndex2 < (ET9INT)pLingCmnInfo->Private.sWordC.pCurrC->nTotalWords); { ET9U8 bBestMatchMin1 = pWordList[snIndex1].Body.bBestMatchMin; ET9U8 bBestMatchMin2 = pWordList[snIndex2].Body.bBestMatchMin; if (bBestMatchMin2 > bBestMatchMin1) { return 1; } else if (bBestMatchMin2 < bBestMatchMin1) { return -1; } } { ET9U32 dwWordIndex1 = pWordList[snIndex1].Body.dwWordIndex; ET9U32 dwWordIndex2 = pWordList[snIndex2].Body.dwWordIndex; if (dwWordIndex2 > dwWordIndex1) { return 1; } else if (dwWordIndex2 < dwWordIndex1) { return -1; } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * / * / * * * * */ static ET9STATUS ET9LOCALCALL __LDB_CalcEditDistanceMGD(ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U8 * const pbSuffixCharCodes, const ET9U16 wIndex, const ET9U16 wLength, const ET9U16 nSymbCount, const ET9U8 bPartialOK) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9ASPCFlexCompareData * const pCMP = &ASPC.u.sCmpDataFlex; ET9U8 const * const pbLockInfo = pCMP->pbLockInfo; ET9U8 const * const pbIsFreqPos = pCMP->pbIsFreqPos; ET9U8 const * const pbIsQualityKey = pCMP->pbIsQualityKey; const ET9U8 bMaxEditDist = pLingCmnInfo->Private.bCurrMaxEditDistance; const ET9U16 wCurrMinSourceLength = pLingCmnInfo->Private.wCurrMinSourceLength; #ifdef __ET9_MGD_COMPLETION /* This conditional isn't expected to be turned on; it's just for preserving code to make the merge of this function back with __LDB_CalcEditDistanceMGD easier. */ const ET9BOOL bAllowSpcCmpl = pCMP->bAllowSpcCmpl; #endif const ET9BOOL bAllowFreePunct = pCMP->bAllowFreePunct; const ET9BOOL bAllowFreeDouble = pCMP->bAllowFreeDouble; const ET9UINT nKeyCount = wLength; ET9UINT nFirstChar = 1; /* 1-based index of first char to score. */ const ET9UINT nQualityCount = pCMP->nQualityCount; ET9UINT nMatchRow = nKeyCount; /* Length of input points we care about. */ ET9_UNUSED(wIndex); /* used in "init" */ ET9Assert(ASPC.bSpcState == ET9_SPC_STATE_FLEX_INIT_OK); _ET9AWValidateFlexArea(pLingCmnInfo); WLOG8B({ #ifdef __ET9_MGD_COMPLETION fprintf(pLogFile8, "__LDB_CalcEditDistanceMGD, spc cmpl %d, free punct %d, free double %d, word ", bAllowSpcCmpl, bAllowFreePunct, bAllowFreeDouble); #else fprintf(pLogFile8, "__LDB_CalcEditDistanceMGD, free punct %d, free double %d, word ", bAllowFreePunct, bAllowFreeDouble); #endif { ET9U16 wIndex; for (wIndex = 0; wIndex < pWord->Base.wWordLen; ++wIndex) { fprintf(pLogFile8, "%c", pWord->Base.sWord[wIndex]); } } fprintf(pLogFile8, "\n"); }) /* screening */ __STAT_AWLdb_Flex_EDS_Call; if (nSymbCount < wCurrMinSourceLength) { WLOG8B(fprintf(pLogFile8, " no match (too short, %u < %u - %u)\n", nSymbCount, nQualityCount, bMaxEditDist);) ET9Assert(0); _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } if (nQualityCount == 1 && nSymbCount > nQualityCount) { const ET9BOOL bMatchExact = (ET9BOOL)__IsCodeExact(0, __LdbGetCharCodeAtPos(0)); if (!bMatchExact) { WLOG8B(fprintf(pLogFile8, " special completion suppression at length 1\n");) _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } /* if we're scoring a suffix, the stem should already be scored. */ if (pbSuffixCharCodes) { nFirstChar = pWord->Base.wWordLen + 1; } /* screen calculation */ { ET9UINT nR; ET9UINT nC; ET9BOOL bInStem = 1; /* matrix */ for (nC = nFirstChar; nC <= nSymbCount; ++nC) { const ET9U16 wCharCode = (pbSuffixCharCodes) ? pbSuffixCharCodes[nC - nFirstChar] : __LdbGetCharCodeAtPos(nC - 1); if (bInStem) { if (pCMP->pwPrevWordSC[nC] == wCharCode) { continue; } else { bInStem = 0; } } pCMP->pwPrevWordSC[nC] = wCharCode; pCMP->pbBestMatchPos[nC] = (ET9U8)nKeyCount; { const ET9BOOL bIsFreeC = (ET9BOOL)(bAllowFreePunct && !pCMP->psLockSymb[nC - 1] && __IsCodeFree(wCharCode)); ET9U8 bBestColEditDist; pCMP->ppbEditDist[0][nC - 0] = pCMP->ppbEditDist[0][nC - 1] + (bIsFreeC ? 0 : 1); bBestColEditDist = pCMP->ppbEditDist[0][nC]; for (nR = 1; nR <= nKeyCount; ++nR) { WLOG8B(fprintf(pLogFile8, "matrix %2d %2d\n", nR, nC);) /* initial */ pCMP->ppbEditDist[nR][nC] = _ET9_FLEX_TUBE_MAX_EDIT_DIST; /* pointless? */ if (pCMP->ppbEditDist[nR - 1][nC - 1] > bMaxEditDist && pCMP->ppbEditDist[nR - 1][nC - 0] > bMaxEditDist && pCMP->ppbEditDist[nR - 0][nC - 1] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " pointless (edit) [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = 0; continue; } { const ET9U8 bFreeR = !pbLockInfo[nR]; const ET9U8 bQualityR = pbIsQualityKey[nR]; const ET9U8 bMatchFreq = (ET9U8)__GetCodeFreq((nR - 1), wCharCode); /* */ pCMP->pbSubstFreqSC[nR][nC] = bMatchFreq; /* subst */ if (bMatchFreq) { WLOG8B(fprintf(pLogFile8, " subst-1, bMatchFreq %d (%u, max %u) [sc]\n", bMatchFreq, pCMP->ppbEditDist[nR - 1][nC - 1], bMaxEditDist);) pCMP->ppbEditDist[nR][nC] = pCMP->ppbEditDist[nR - 1][nC - 1]; } else if (pbLockInfo[nR] == ET9_LOCKED_SYMB_VALUE_AND_POS) { WLOG8B(fprintf(pLogFile8, " subst, non matching locked symb [sc]\n");) ET9Assert(pCMP->ppbEditDist[nR][nC] == _ET9_FLEX_TUBE_MAX_EDIT_DIST); continue; } else if (pCMP->ppbEditDist[nR - 1][nC - 1] < bMaxEditDist && bQualityR && bFreeR) { WLOG8B(fprintf(pLogFile8, " subst-2, ppbEditDist %d [sc]\n", pCMP->ppbEditDist[nR - 1][nC - 1]);) pCMP->ppbEditDist[nR][nC] = pCMP->ppbEditDist[nR - 1][nC - 1] + 1; } /* del */ if (!bQualityR) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " del-1 (%u, max %u) [sc]\n", bEditCost, bMaxEditDist);) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } else if (pCMP->ppbEditDist[nR - 1][nC - 0] < bMaxEditDist && bFreeR) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0] + 1; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " del-2 [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } /* ins */ if ((bAllowFreeDouble && bMatchFreq && bMatchFreq == pCMP->pbSubstFreqSC[nR][nC - 1]) || bIsFreeC) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-1 [sc]\n");) pCMP->pbSubstFreqSC[nR][nC] = pCMP->pbSubstFreqSC[nR][nC - 1]; pCMP->ppbEditDist[nR][nC] = bEditCost; } } #ifdef __ET9_MGD_COMPLETION else if (nR == nKeyCount && nC > nQualityCount && (!pCMP->ppbEditDist[nR - 0][nC - 1] || bAllowSpcCmpl)) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-2, nC %d, nQualityCount %d [sc]\n", nC, nQualityCount);) pCMP->pbSubstFreqSC[nR][nC] = 0; pCMP->ppbEditDist[nR][nC] = bEditCost; } } #endif else if (pCMP->ppbEditDist[nR - 0][nC - 1] < bMaxEditDist) { const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1] + 1; if (bEditCost < pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " ins-3 (%u, max %u) [sc]\n", bEditCost, bMaxEditDist);) pCMP->pbSubstFreqSC[nR][nC] = bMatchFreq; pCMP->ppbEditDist[nR][nC] = bEditCost; } } /* best */ if (bBestColEditDist > pCMP->ppbEditDist[nR][nC]) { bBestColEditDist = pCMP->ppbEditDist[nR][nC]; pCMP->pbBestMatchPos[nC] = (ET9U8)nR; } } } /* nR */ /* pre-empt? */ if (bBestColEditDist > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " no match (best edit) [sc], %d > %d\n", bBestColEditDist, bMaxEditDist);) pCMP->pwPrevWordSC[nC] = ALDB_CHAR_CODE_NONE; __STAT_AWLdb_Flex_EDS_Done; _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } } /* nC */ pCMP->pwPrevWordSC[nC] = ALDB_CHAR_CODE_NONE; WLOG8B({ fprintf(pLogFile8, "\n--- edit distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbEditDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n"); }) /* Mark where we want to measure a match to.*/ if (bPartialOK) { nMatchRow = pCMP->pbBestMatchPos[nSymbCount]; } /* really a match? */ if (pCMP->ppbEditDist[nMatchRow][nSymbCount] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " no match (final edit), %d > %d (%d)\n", pCMP->ppbEditDist[nKeyCount][nSymbCount], bMaxEditDist, pCMP->ppbStemDist[nKeyCount][nSymbCount]);) __STAT_AWLdb_Flex_EDS_Done; _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } __STAT_AWLdb_Flex_EDS_Done; } /* full calculation */ __STAT_AWLdb_Flex_EDC_Call; { ET9UINT nR; ET9UINT nC; ET9BOOL bInStem = 1; /* matrix */ for (nC = nFirstChar; nC <= nSymbCount; ++nC) { const ET9U16 wCharCode = (pbSuffixCharCodes) ? pbSuffixCharCodes[nC - nFirstChar] : __LdbGetCharCodeAtPos(nC - 1); if (bInStem) { if (pCMP->pwPrevWordFC[nC] == wCharCode) { continue; } else { bInStem = 0; } } pCMP->pwPrevWordFC[nC] = wCharCode; { const ET9BOOL bIsFreeC = (ET9BOOL)(bAllowFreePunct && !pCMP->psLockSymb[nC - 1] && __IsCodeFree(wCharCode)); pCMP->pbComplLenFC[nC] = 0; pCMP->ppbStemDist[0][nC - 0] = pCMP->ppbEditDist[0][nC - 0]; pCMP->ppbFreeDist[0][nC - 0] = (ET9U8)(nC - pCMP->ppbEditDist[0][nC - 0]); pCMP->ppbEditDistIns[0][nC - 0] = pCMP->ppbEditDist[0][nC - 0]; for (nR = 1; nR <= nKeyCount; ++nR) { WLOG8B(fprintf(pLogFile8, "matrix %2d %2d\n", nR, nC);) /* pointless? */ if (pCMP->ppbEditDist[nR][nC] > bMaxEditDist) { WLOG8B(fprintf(pLogFile8, " pointless (edit %u > %u) [full]\n", pCMP->ppbEditDist[nR][nC], bMaxEditDist);) pCMP->ppbFreeDist[nR][nC] = _ET9_FLEX_TUBE_MAX_FREE_DIST; pCMP->ppbStemDist[nR][nC] = _ET9_FLEX_TUBE_MAX_STEM_DIST; pCMP->ppxStemFreq[nR][nC] = 1; pCMP->pbSubstFreqFC[nR][nC] = 0; continue; } { const ET9U8 bFreeR = !pbLockInfo[nR]; const ET9U8 bQualityR = pbIsQualityKey[nR]; const ET9U8 bMatchFreq = (ET9U8)__GetCodeFreq((nR - 1), wCharCode); const ET9U8 bDefaultFreq = (bQualityR && nC < ET9_SPC_ED_MAX_FREQ_LEN) ? 10 : 1; ET9U8 bEditDist; ET9U8 bFreeDist; ET9U8 bStemDist; ET9FREQ xStemFreq; ET9FREQ xSymbDist; ET9U8 bEditDistSbt; ET9U8 bEditDistTrp; ET9U8 bEditDistIns; ET9U8 bEditDistDel; ET9U8 bLimitedCount; /* */ pCMP->pbSubstFreqFC[nR][nC] = bMatchFreq; /* subst */ if (bMatchFreq) { const ET9BOOL bMatchExact = (ET9BOOL)__IsCodeExact((nR - 1), wCharCode); const ET9BOOL bMatchLimited = (ET9BOOL)__IsCodeLimited((nR - 1), wCharCode); #ifdef ET9_USE_FLOAT_FREQS const ET9FREQ xFreq = (ET9FREQ)(!pbIsFreqPos[nR] ? bDefaultFreq : bMatchFreq); #else const ET9FREQ xFreq = (!pbIsFreqPos[nR] || bMatchFreq <= 17) ? 1 : ((bMatchFreq * 15) / 255); #endif const ET9FREQ xDist = (ET9FREQ)__GetSymbDistance((nR - 1), wCharCode); WLOG8B(fprintf(pLogFile8, " subst-1, bMatchFreq %d (%u, max %u) [full]\n", bMatchFreq, pCMP->ppbEditDist[nR - 1][nC - 1], bMaxEditDist);) bFreeDist = pCMP->ppbFreeDist[nR - 1][nC - 1] + (bQualityR ? 0 : 1); bEditDist = pCMP->ppbEditDist[nR - 1][nC - 1]; bStemDist = pCMP->ppbStemDist[nR - 1][nC - 1] + (bMatchExact ? 0 : 1); xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 1] * xFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 1] + xDist; bEditDistSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 1]; bEditDistTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 1]; bEditDistIns = pCMP->ppbEditDistIns[nR - 1][nC - 1]; bEditDistDel = pCMP->ppbEditDistDel[nR - 1][nC - 1]; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 1] + (bMatchLimited ? 1 : 0); } else if (pbLockInfo[nR] == ET9_LOCKED_SYMB_VALUE_AND_POS) { WLOG8B(fprintf(pLogFile8, " subst, non matching locked symb [full]\n");) ET9Assert(pCMP->ppbEditDist[nR][nC] == _ET9_FLEX_TUBE_MAX_EDIT_DIST); continue; } else if (pCMP->ppbEditDist[nR - 1][nC - 1] < bMaxEditDist && bQualityR && bFreeR) { WLOG8B(fprintf(pLogFile8, " subst-2, ppbEditDist %d [full]\n", pCMP->ppbEditDist[nR - 1][nC - 1]);) bFreeDist = pCMP->ppbFreeDist[nR - 1][nC - 1]; bEditDist = pCMP->ppbEditDist[nR - 1][nC - 1] + 1; bStemDist = pCMP->ppbStemDist[nR - 1][nC - 1] + 1; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 1] * bDefaultFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 1]; bEditDistSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 1] + 1; bEditDistTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 1]; bEditDistIns = pCMP->ppbEditDistIns[nR - 1][nC - 1]; bEditDistDel = pCMP->ppbEditDistDel[nR - 1][nC - 1]; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 1]; } else { bFreeDist = _ET9_FLEX_TUBE_MAX_FREE_DIST; bEditDist = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bStemDist = _ET9_FLEX_TUBE_MAX_STEM_DIST; xStemFreq = 1; xSymbDist = 0; bEditDistSbt = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistTrp = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistIns = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bEditDistDel = _ET9_FLEX_TUBE_MAX_EDIT_DIST; bLimitedCount = 0; } /* del */ if (!bQualityR) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 1][nC - 0]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 1][nC - 0]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 0]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 0]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 1][nC - 0]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 1][nC - 0]; if (_ET9_Flex_IsBetterOp(-1, 0)) { WLOG8B(fprintf(pLogFile8, " del-1 [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = 0; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 0]; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 0]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 0]; } } else if (pCMP->ppbEditDist[nR - 1][nC - 0] < bMaxEditDist && bFreeR) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 1][nC - 0]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 1][nC - 0] + 1; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 1][nC - 0] + 1; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 1][nC - 0]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 1][nC - 0]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 1][nC - 0]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 1][nC - 0] + 1; if (_ET9_Flex_IsBetterOp(-1, 0)) { WLOG8B(fprintf(pLogFile8, " del-2 [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = 0; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 1][nC - 0] * bDefaultFreq; xSymbDist = pCMP->ppxSymbDist[nR - 1][nC - 0]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 1][nC - 0]; } } /* ins */ if ((bAllowFreeDouble && bMatchFreq && bMatchFreq == pCMP->pbSubstFreqFC[nR][nC - 1]) || bIsFreeC) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1] + 1; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-1 (free) [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = pCMP->pbSubstFreqFC[nR][nC - 1]; if (nR == nKeyCount) { pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1]; } bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } #ifdef __ET9_MGD_COMPLETION else if (nR == nKeyCount && nC > nQualityCount && (!pCMP->ppbEditDist[nR - 0][nC - 1] || bAllowSpcCmpl)) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1]; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1]; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1]; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-2 (cmpl), nC %d, nQualityCount %d [full]\n", nC, nQualityCount);) pCMP->pbSubstFreqFC[nR][nC] = 0; pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1] + 1; bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } #endif else if (pCMP->ppbEditDist[nR - 0][nC - 1] < bMaxEditDist) { const ET9U8 bFreeCost = pCMP->ppbFreeDist[nR - 0][nC - 1]; const ET9U8 bEditCost = pCMP->ppbEditDist[nR - 0][nC - 1] + 1; const ET9U8 bStemCost = pCMP->ppbStemDist[nR - 0][nC - 1] + 1; const ET9U8 bEditCostSbt = pCMP->ppbEditDistSbt[nR - 0][nC - 1]; const ET9U8 bEditCostTrp = pCMP->ppbEditDistTrp[nR - 0][nC - 1]; const ET9U8 bEditCostIns = pCMP->ppbEditDistIns[nR - 0][nC - 1] + 1; const ET9U8 bEditCostDel = pCMP->ppbEditDistDel[nR - 0][nC - 1]; if (_ET9_Flex_IsBetterOp(0, -1)) { WLOG8B(fprintf(pLogFile8, " ins-3 (spc) [full]\n");) pCMP->pbSubstFreqFC[nR][nC] = bMatchFreq; if (nR == nKeyCount) { pCMP->pbComplLenFC[nC] = pCMP->pbComplLenFC[nC - 1]; } bFreeDist = bFreeCost; bEditDist = bEditCost; bStemDist = bStemCost; xStemFreq = pCMP->ppxStemFreq[nR - 0][nC - 1]; xSymbDist = pCMP->ppxSymbDist[nR - 0][nC - 1]; bEditDistSbt = bEditCostSbt; bEditDistTrp = bEditCostTrp; bEditDistIns = bEditCostIns; bEditDistDel = bEditCostDel; bLimitedCount = pCMP->ppbLimitedCount[nR - 0][nC - 1]; } } /* assure same as screening */ if (bEditDist != pCMP->ppbEditDist[nR][nC]) { WLOG8B(fprintf(pLogFile8, " inconsistent (%u <-> %u) [full]\n", bEditDist, pCMP->ppbEditDist[nR][nC]);) } ET9Assert(bEditDist == pCMP->ppbEditDist[nR][nC]); /* persist */ pCMP->ppbFreeDist[nR][nC] = bFreeDist; pCMP->ppbStemDist[nR][nC] = bStemDist; pCMP->ppxStemFreq[nR][nC] = xStemFreq; pCMP->ppxSymbDist[nR][nC] = xSymbDist; pCMP->ppbEditDistSbt[nR][nC] = bEditDistSbt; pCMP->ppbEditDistTrp[nR][nC] = bEditDistTrp; pCMP->ppbEditDistIns[nR][nC] = bEditDistIns; pCMP->ppbEditDistDel[nR][nC] = bEditDistDel; pCMP->ppbLimitedCount[nR][nC] = bLimitedCount; } } /* nR */ } } /* nC */ pCMP->pwPrevWordFC[nC] = ALDB_CHAR_CODE_NONE; WLOG8B({ fprintf(pLogFile8, "\n--- free distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbFreeDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- edist distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbEditDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- stem distance ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->ppbStemDist[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- edit kinds ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%x%x%x%x ", pCMP->ppbEditDistSbt[nR][nC], pCMP->ppbEditDistTrp[nR][nC], pCMP->ppbEditDistIns[nR][nC], pCMP->ppbEditDistDel[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- stem freq ---\n\n"); for (nR = 0; nR <= nKeyCount; ++nR) { fprintf(pLogFile8, "%2d %2x : ", nR, pbIsQualityKey[nR]); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%8.0f ", pCMP->ppxStemFreq[nR][nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n--- completion length ---\n\n"); { fprintf(pLogFile8, " : "); for (nC = 0; nC <= nSymbCount; ++nC) { fprintf(pLogFile8, "%2x ", pCMP->pbComplLenFC[nC]); } fprintf(pLogFile8, "\n"); } fprintf(pLogFile8, "\n"); }) /* assure same as screening */ ET9Assert(pCMP->ppbEditDist[nMatchRow][nSymbCount] <= bMaxEditDist); { const ET9UINT nMatchCol = nSymbCount; #ifdef __ET9_MGD_COMPLETION const ET9BOOL bAvoidStems = (pLingCmnInfo->Private.bTraceBuild || wIndex) ? 1 : 0; ET9U8 bComplLenFC_nSymbCount = pCMP->pbComplLenFC[nMatchCol]; #endif /* assign distance */ pWord->Body.bEditDistSpc = pCMP->ppbEditDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistStem = pCMP->ppbStemDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistFree = pCMP->ppbFreeDist[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcSbt = pCMP->ppbEditDistSbt[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcTrp = pCMP->ppbEditDistTrp[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcIns = pCMP->ppbEditDistIns[nMatchRow][nMatchCol]; pWord->Body.bEditDistSpcDel = pCMP->ppbEditDistDel[nMatchRow][nMatchCol]; pWord->Body.bLimitedCount = pCMP->ppbLimitedCount[nMatchRow][nMatchCol]; /* adjust to prefer spell correction over completion when applicable */ #ifdef __ET9_MGD_COMPLETION if (bComplLenFC_nSymbCount && (pWord->Body.bEditDistSpc + bComplLenFC_nSymbCount) <= bMaxEditDist && (pWord->Body.bEditDistSpc || pLingCmnInfo->Private.bTraceBuild)) { const ET9U8 bCost = bComplLenFC_nSymbCount; WLOG8B(fprintf(pLogFile8, " completion to spc term\n");) bComplLenFC_nSymbCount = 0; pWord->Body.bEditDistSpc = pWord->Body.bEditDistSpc + bCost; pWord->Body.bEditDistStem = pWord->Body.bEditDistStem + bCost; pWord->Body.bEditDistSpcIns = pWord->Body.bEditDistSpcIns + bCost; } /* suppress completions before point */ if (bComplLenFC_nSymbCount && nQualityCount < pLingCmnInfo->Private.wWordCompletionPoint) { if (bAvoidStems) { WLOG8B(fprintf(pLogFile8, " suppressed completion before point\n");) __STAT_AWLdb_Flex_EDC_Done(NULL); _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NO_MATCH; } } /**/ if (bComplLenFC_nSymbCount && !pWord->Body.bEditDistSpc && (pWordSymbInfo->bNumSymbs < pLingCmnInfo->Private.wWordCompletionPoint)) { if (bAvoidStems) { WLOG8B(fprintf(pLogFile8, " assigning edit distance - before completion point\n");) pWord->Body.bEditDistSpc = 1; } } #endif #if 1 if (pWord->Base.wWordLen > pLingCmnInfo->Private.wMaxWordLength + pWord->Body.bEditDistFree) { pWord->Body.bEditDistFree = 0; pWord->Base.wWordCompLen = 0; } else { #ifdef __ET9_MGD_COMPLETION pWord->Base.wWordCompLen = bComplLenFC_nSymbCount; #else pWord->Base.wWordCompLen = 0; #endif } #else if (pbComplLen[nMatchCol] && (!pWord->Body.bEditDistSpc || bAllowSpcCmpl) && pWord->Base.wWordLen <= pLingCmnInfo->Private.wMaxWordLength) { pWord->Base.wWordCompLen = pbComplLen[nMatchCol]; } else { pWord->Base.wWordCompLen = 0; } #endif if (!pWord->Base.wWordCompLen && pWord->Base.wWordLen > pWordSymbInfo->bNumSymbs) { if (!pWord->Body.bEditDistSpc && pWord->Base.wWordLen <= pLingCmnInfo->Private.wMaxWordLength) { WLOG8B(fprintf(pLogFile8, " longer than input and shorter than max - special zero compl len\n");) pWord->Base.wWordCompLen = 0xFFFF; } else if (pWord->Base.wWordLen == pLingCmnInfo->Private.wMaxWordLength + pWord->Body.bEditDistFree) { WLOG8B(fprintf(pLogFile8, " longer than input and longer than max - special zero compl len\n");) pWord->Base.wWordCompLen = 0xFFFF; } } /* assign tap freq */ #ifdef ET9_USE_FLOAT_FREQS pWord->Body.xTapFreq = pCMP->ppxStemFreq[nMatchRow][nMatchCol] / pCMP->fTapFreqDivider; #else /* ET9_USE_FLOAT_FREQS */ if (nQualityCount <= 4) { pWord->Body.xTapFreq = (ET9FREQPART)pCMP->ppxStemFreq[nMatchRow][nMatchCol]; } else { pWord->Body.xTapFreq = (ET9FREQPART)(pCMP->ppxStemFreq[nMatchRow][nMatchCol] >> (4 * (nQualityCount - 4))); } #endif /* ET9_USE_FLOAT_FREQS */ if (!pWord->Body.xTapFreq) { pWord->Body.xTapFreq = 1; } ET9Assert(pWord->Body.xTapFreq >= 0); /* assign symb distance */ pWord->Body.xSymbDist = pCMP->ppxSymbDist[nMatchRow][nMatchCol]; /* best match len */ pWord->Body.bBestMatchLen = pCMP->pbBestMatchPos[nMatchCol]; } /* done */ WLOG8B(fprintf(pLogFile8, " match, bEditDistStem %d, bEditDistSpc %d, wWordCompLen %d, \n\n", pWord->Body.bEditDistStem, pWord->Body.bEditDistSpc, pWord->Base.wWordCompLen);) __STAT_AWLdb_Flex_EDC_Done(pWord); _ET9AWModifiedFlexArea(pLingCmnInfo); return ET9STATUS_NONE; } } /*---------------------------------------------------------------------------*/ #define _ET9AWMGD_STEM 0 /* < -IDR- */ #define _ET9AWMGD_SUFFIX 1 /* < -IDR- */ #define _ET9AWMGD_NEITHER 2 /* < -IDR- */ /* -IDR- */ typedef struct __MGD_SearchPoint_s { ET9U32 dwReadPoint; /* Where we are in the class-to-suffix table. */ ET9U32 dwEnd; /* Where to read to in the class-to-suffix table. */ /* "output" fields. */ ET9U16 wClassID; ET9U8 bFreq; } __MGD_SearchPoint; /* ** -IDR- / *****/ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ static void ET9LOCALCALL __MGD_InitClassesScan(ET9AWLingInfo *pLingInfo, __MGD_SearchPoint *pSearchPoint, ET9U32 dwIndex, ET9U8 bMGDSection) { const ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U32 dwClassReadPoint = _ET9ReadLDBWord3(pLingInfo, pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwToClassOffsetsAddress + 3 * (dwIndex - 1)); ET9U32 dwClassesEnd = _ET9ReadLDBWord3(pLingInfo, pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwToClassOffsetsAddress + 3 * dwIndex); ET9Assert(dwIndex > 0); ET9Assert(dwIndex <= pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwNum); ET9Assert(dwClassReadPoint < pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwToClassCount); ET9Assert(dwClassesEnd <= pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwToClassCount); pSearchPoint->dwReadPoint = dwClassReadPoint + pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwToClassAddress; pSearchPoint->dwEnd = dwClassesEnd + pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwToClassAddress; pSearchPoint->wClassID = 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * / */ static ET9BOOL ET9LOCALCALL __MGD_GetNextClassFreq(ET9AWLingInfo *pLingInfo, __MGD_SearchPoint *pSearchPoint) { const ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (pSearchPoint->dwReadPoint >= pSearchPoint->dwEnd) { return 0; } else { ET9U8 bClassFreqByte1 = _ET9ReadLDBByte(pLingInfo, pSearchPoint->dwReadPoint++); ET9U16 wClassDelta = bClassFreqByte1 >> pLingCmnInfo->Private.ALdbMGD.bNumFreqBits; pSearchPoint->bFreq = bClassFreqByte1 & ((1 << pLingCmnInfo->Private.ALdbMGD.bNumFreqBits) - 1); if (wClassDelta >= pLingCmnInfo->Private.ALdbMGD.bClassHuffBase) { wClassDelta += _ET9ReadLDBByte(pLingInfo, pSearchPoint->dwReadPoint++) * pLingCmnInfo->Private.ALdbMGD.bInvClassHuffBase; } pSearchPoint->wClassID = (ET9U16)(pSearchPoint->wClassID + wClassDelta); return 1; } } /* -IDR- */ typedef struct __MGD_SuffixIterator_s { __MGD_SearchPoint xStemSearchPoint; /* Where we are in loading the stem classes. */ ET9BOOL bAnotherStemClass; /* Do we have another stem class? */ ET9BOOL bStandAloneOK; /* Can an empty suffix be returned? */ ET9U8 bStandAloneFreq; /* Freq. of the stem standing alone. */ ET9INT snStartSuffixPos; /* Where to start looking in the suffix array. */ ET9INT snEndSuffixPos; /* Where to stop looking in the suffix array. */ ET9INT snSuffixPos; /* Where we are in the suffix array. */ ET9UINT nClasses; /* Number of classes for the stem. */ ET9U16 pwClass[__ET9AWMGD_CLASS_PER_STEM]; /* Classes enabled for stem. */ ET9U8 pbFreq[__ET9AWMGD_CLASS_PER_STEM]; /* Freq. for those classes. */ /* "Outputs" after an iteration. */ ET9U8 bFreq; /* Frequency of the stem/suffix combination. */ ET9U16 wSuffixIndex; /* Universal suffix ID. */ } __MGD_SuffixIterator; /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * * */ ET9INLINE static ET9U8 ET9LOCALCALL __MGD_GetBestSuffixFreq(ET9AWLingInfo *pLingInfo, __MGD_SuffixIterator *pIterator, ET9U32 dwSuffixIndex) { __MGD_SearchPoint xSuffixSearchPoint; ET9UINT nStemClassIndex = 0; ET9U8 bBestFreq = __ET9AWMGD_NOFREQ; if (!dwSuffixIndex) { return __ET9AWMGD_NOFREQ; /* Zero is stand-alone; can't match. */ } __MGD_InitClassesScan(pLingInfo, &xSuffixSearchPoint, dwSuffixIndex, _ET9AWMGD_SUFFIX); for (;;) { /* Step through the classes of each component */ if (xSuffixSearchPoint.wClassID <= pIterator->pwClass[nStemClassIndex]) { if (!__MGD_GetNextClassFreq(pLingInfo, &xSuffixSearchPoint)) { return bBestFreq; } } else { if (++nStemClassIndex >= pIterator->nClasses) { return bBestFreq; } } if (pIterator->pwClass[nStemClassIndex] == xSuffixSearchPoint.wClassID) { ET9U8 bNewFreq = pIterator->pbFreq[nStemClassIndex] + xSuffixSearchPoint.bFreq; if (bBestFreq == __ET9AWMGD_NOFREQ || bNewFreq > bBestFreq) { bBestFreq = bNewFreq; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static ET9UINT ET9LOCALCALL __IsInflection (ET9SymbInfo * const pSymbInfo) { return !(pSymbInfo->bTraceIndex && pSymbInfo->bTraceProbability != 0xFF); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * / * / * * * */ static ET9U8 ET9LOCALCALL __CountInflection (ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U16 wIndex, const ET9U16 wLength) { ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9UINT nIndex; ET9U8 bResult = 0; for (nIndex = 0; nIndex < wLength; ++nIndex) { if (__IsInflection(&pWordSymbInfo->SymbsInfo[wIndex + nIndex])) { ++bResult; } } return bResult; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * * * * */ ET9INLINE static void ET9LOCALCALL __MGD_InitSuffixIterator (ET9AWLingInfo *pLingInfo, __MGD_SuffixIterator * const pIterator, const ET9U8 bStemMatchLen, const ET9U8 bStemMatchMin, const ET9U16 wIndex, const ET9U16 wLength, const ET9U16 wMinWordLen, const ET9U8 bMGDSection) { const ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (bMGDSection == _ET9AWMGD_SUFFIX) { pIterator->bStandAloneOK = 1; pIterator->snStartSuffixPos = 0; pIterator->snEndSuffixPos = -1; pIterator->bStandAloneFreq = __ET9AWMGD_NOFREQ; } else { ET9INT snLengthStart = wLength - bStemMatchLen - __ET9AWMGD_PATH_SUBTRACT; ET9INT snLengthEnd = wMinWordLen - bStemMatchMin + __ET9AWMGD_PATH_ADD; { /* Relax our lower bound on length for positions which aren't an inflection point. */ ET9INT snIndex; for (snIndex = bStemMatchLen; snIndex <= wLength - snLengthStart; snIndex++) { if (!__IsInflection(&pLingCmnInfo->Base.pWordSymbInfo->SymbsInfo[wIndex + snIndex])) { --snLengthStart; } } } if (snLengthStart < 0) { snLengthStart = 0; } if (snLengthStart > ALDB_COMPARE_MAX_POS + 1) { snLengthStart = ALDB_COMPARE_MAX_POS + 1; } /* Since we store suffixes by inflection match, adjust down from path length. */ if (snLengthStart > 1) { snLengthStart -= pLingCmnInfo->Private.pwSuffixPathExtra[snLengthStart - 1]; } if (snLengthEnd > pLingCmnInfo->Private.bMaxSuffixLen) { snLengthEnd = pLingCmnInfo->Private.bMaxSuffixLen; } if (snLengthStart > snLengthEnd) { /* Catches a start > bMaxSuffixLen or end < 0 */ pIterator->snStartSuffixPos = 0; pIterator->snEndSuffixPos = -1; pIterator->bStandAloneOK = 0; } else { if (snLengthStart == 0) { pIterator->bStandAloneOK = pIterator->bStandAloneFreq != __ET9AWMGD_NOFREQ; snLengthStart = 1; } else { pIterator->bStandAloneOK = 0; } /* Index of pwSuffixPerLength is Start - 1 due to the array starting at length 1. */ /* We subtract one from snStartSuffixPos because we pre-increment as we iterate. */ pIterator->snStartSuffixPos = pLingCmnInfo->Private.pwSuffixPerLength[snLengthStart - 1] - 1; pIterator->snEndSuffixPos = pLingCmnInfo->Private.pwSuffixPerLength[snLengthEnd]; } } pIterator->snSuffixPos = pIterator->snStartSuffixPos; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static ET9UINT ET9LOCALCALL __MGD_NextStemSet (ET9AWLingInfo *pLingInfo, __MGD_SuffixIterator *pIterator) { const ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9UINT nClasses = 0; while (pIterator->bAnotherStemClass && nClasses < __ET9AWMGD_CLASS_PER_STEM) { ET9U16 wClassID = pIterator->xStemSearchPoint.wClassID; if (__ET9BitCheck(pLingCmnInfo->Private.pbEnabledClasses, wClassID)) { pIterator->pwClass[nClasses] = wClassID; pIterator->pbFreq[nClasses] = pIterator->xStemSearchPoint.bFreq; ++nClasses; } pIterator->bAnotherStemClass = __MGD_GetNextClassFreq(pLingInfo, &pIterator->xStemSearchPoint); } pIterator->nClasses = nClasses; return nClasses; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __MGD_StepSuffixIterator (ET9AWLingInfo *pLingInfo, __MGD_SuffixIterator *pIterator) { const ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; if (pIterator->bStandAloneOK) { pIterator->wSuffixIndex = 0; pIterator->bFreq = pIterator->bStandAloneFreq; pIterator->bStandAloneOK = 0; return 1; } if (!pIterator->nClasses) { /* No valid classes for this stem. */ return 0; } for (;;) { while (++pIterator->snSuffixPos < pIterator->snEndSuffixPos) { ET9U16 wSuffixIndex = pLingCmnInfo->Private.pwSuffixIndexes[pIterator->snSuffixPos]; pIterator->bFreq = __MGD_GetBestSuffixFreq(pLingInfo, pIterator, wSuffixIndex); if (pIterator->bFreq != __ET9AWMGD_NOFREQ) { pIterator->wSuffixIndex = wSuffixIndex; return 1; } } if (pIterator->snStartSuffixPos < pIterator->snEndSuffixPos && __MGD_NextStemSet(pLingInfo, pIterator)) { /* If we have any more stem classes, cycle through suffix classes again. */ pIterator->snSuffixPos = pIterator->snStartSuffixPos; } else { return 0; } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * / * * * * * */ static void ET9LOCALCALL __MGD_ConcatStemSuffix(const ET9AWLingCmnInfo * const pLingCmnInfo, ET9AWPrivWordInfo *pResultWord, const ET9U8 * const pbSuffixChars, const ET9U16 wSize) { const ET9U16 wStemLen = pResultWord->Base.wWordLen; const ET9U8 *pSource = pbSuffixChars; ET9SYMB *pDest = &(pResultWord->Base.sWord[wStemLen]); ET9U16 wCount; pResultWord->Base.wWordLen = wSize; for (wCount = wStemLen; wCount < wSize; ++wCount) { *pDest++ = ALDB.header.psCharacterDecodeTable[*pSource++]; } } /*---------------------------------------------------------------------------*/ /* -IDR- * / * / * / * * * */ static void ET9LOCALCALL __MGD_ReverseSymbsInfo(ET9AWLingCmnInfo * const pLingCmnInfo, const ET9U16 wIndex, const ET9U16 wLength) { ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9UINT nLimit = wLength / 2; /* # of steps to reverse list. */ ET9UINT nIndex; for (nIndex = 0; nIndex < nLimit; ++nIndex) { ET9SymbInfo temp = pWordSymbInfo->SymbsInfo[wIndex + nIndex]; pWordSymbInfo->SymbsInfo[wIndex + nIndex] = pWordSymbInfo->SymbsInfo[wIndex + wLength - nIndex - 1]; pWordSymbInfo->SymbsInfo[wIndex + wLength - nIndex - 1] = temp; } if (wIndex == pWordSymbInfo->Private.wReverseIndex && wLength == pWordSymbInfo->Private.wReverseLength) { pWordSymbInfo->Private.wReverseIndex = 0; pWordSymbInfo->Private.wReverseLength = 0; } else { pWordSymbInfo->Private.wReverseIndex = wIndex; pWordSymbInfo->Private.wReverseLength = wLength; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * / * * * * * * */ static void ET9LOCALCALL __ET9AWLdbGetCandidatesPartialMGD (ET9AWLingInfo * const pLingInfo, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator, const ET9U8 bSpcMode, const ET9U8 bMGDSection) { ET9AWPrivWordInfo sPrivateWord; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U8 bLangIndex = pLingCmnInfo->dwLdbNum == pLingInfo->pLingCmnInfo->dwFirstLdbNum ? ET9AWFIRST_LANGUAGE : ET9AWSECOND_LANGUAGE; const ET9U8 bWordSrc = ADDON_FROM_FREQ_IND(bFreqIndicator, ET9WORDSRC_LDB); /* if all LDB words being downshifted, or word being compounded, downshift it */ const ET9BOOL bDownShift = (ET9BOOL)((ET9DOWNSHIFTALLLDB(pLingCmnInfo) || wIndex) && ALDB.header.bUpperCount && _ET9_LanguageSpecific_ApplyShifting(pLingInfo, NULL)); /* Word length calculated from the path inflection points. */ ET9U16 wMinWordLen = __CountInflection(pLingCmnInfo, wIndex, wLength); ET9Assert(pLingInfo); WLOG8(fprintf(pLogFile8, "\__ET9AWLdbGetCandidatesPartialMGD\n\n");) __STAT_AWLdb_Call; __STAT_AWLdb_Flex_Call; if (!wLength || wLength > ET9MAXWORDSIZE) { return; } if (_ET9AWCalcEditDistanceInit(pLingInfo, wIndex, wLength, bSpcMode, (ET9U8)(pLingCmnInfo->Private.ASpc.bSpcFeatures ? pLingCmnInfo->Private.ASpc.bSpcFeatures : (ET9_FLEX_FEATURE_ACTIVATE_MASK | ET9_FLEX_FEATURE_FREE_PUNCT_MASK)))) { return; } WLOG3(fprintf(pLogFile3, "__ET9AWLdbGetCandidatesPartialMGD, start, wIndex = %2d, wLength = %2d, bSpcMode = %d\n", wIndex, wLength, bSpcMode);) _InitPrivWordInfo(&sPrivateWord); sPrivateWord.Body.bLangIndexScoring = bLangIndex; __ET9CompareStart(pLingCmnInfo, wIndex, wLength, 1 /* bUsingFlex */); ALDB.compare.bSpcExactFilter = (bMGDSection == _ET9AWMGD_STEM); /* Disallow ambiguous start for MGD stems. */ ALDB.compare.bSpcExactFilterTrace = (bMGDSection == _ET9AWMGD_STEM); /* Disallow ambiguous start for MGD stems. */ __ET9SearchStart(pLingInfo, sPrivateWord.Base.sWord, &sPrivateWord.Base.wWordLen, ET9MAXWORDSIZE, 1 /* bUsingFlex */); for (; !__IsExhausted(); __ET9SearchGetNextFlex(pLingInfo)) { const ET9U32 dwCurrWordIndex = __GetWordIndex() + 1; __MGD_SuffixIterator sIterator; ET9BOOL bPartialOK = bMGDSection <= _ET9AWMGD_SUFFIX; /* Do we allow partial string match? */ __STAT_INC_TotLdbCount; WLOG3B(fprintf(pLogFile3, "__ET9AWLdbGetCandidatesPartialMGD, word index = %6d, word len = %2d, wLength = %2d\n", __GetWordIndex(), sPrivateWord.Base.wWordLen, wLength);) /* Check the stem against allowed suffix classes to maybe save a call to */ /* __LDB_CalcEditDistanceMGD() */ if (bMGDSection == _ET9AWMGD_STEM) { /* Get the stem stand-alone freq. and start read of the stem's class list */ __MGD_InitClassesScan(pLingInfo, &sIterator.xStemSearchPoint, dwCurrWordIndex, _ET9AWMGD_STEM); sIterator.bAnotherStemClass = __MGD_GetNextClassFreq(pLingInfo, &sIterator.xStemSearchPoint); ET9Assert(sIterator.bAnotherStemClass); /* There must be at least 1 class. */ sIterator.bStandAloneFreq = __ET9AWMGD_NOFREQ; if (sIterator.xStemSearchPoint.wClassID == 0) { /* Stand-alone class is only at front. */ sIterator.bStandAloneFreq = sIterator.xStemSearchPoint.bFreq + 7; /* Stand-alone words have "suffix freq" of 7. */; sIterator.bAnotherStemClass = __MGD_GetNextClassFreq(pLingInfo, &sIterator.xStemSearchPoint); } if (!__MGD_NextStemSet(pLingInfo, &sIterator)) { if (sIterator.bStandAloneFreq == __ET9AWMGD_NOFREQ) { continue; } else { bPartialOK = 0; } } } if (!bPartialOK && sPrivateWord.Base.wWordLen < wMinWordLen) { continue; } { ET9STATUS eMatchStatus; __ProfileStart; eMatchStatus = __LDB_CalcEditDistanceMGD(pLingInfo, &sPrivateWord, NULL /* pbSuffixChars */, wIndex, wLength, sPrivateWord.Base.wWordLen, bPartialOK); if (bMGDSection == _ET9AWMGD_STEM) { __ProfileEnd(tAW_SelLst_MgdStemMatching); } else { __ProfileEnd(tAW_SelLst_MgdSuffixMatching); } if (eMatchStatus) { continue; } } sPrivateWord.Body.bBestMatchMin = __CountInflection(pLingCmnInfo, wIndex, sPrivateWord.Body.bBestMatchLen); ET9Assert(sPrivateWord.Body.xTapFreq >= 0); /* candidate found */ sPrivateWord.Body.dwWordIndex = dwCurrWordIndex; sPrivateWord.Body.bWordSrc = bWordSrc; sPrivateWord.Body.wEWordFreq = 0; sPrivateWord.Body.wTWordFreq = 0; sPrivateWord.Body.wSuffixIndex = 0; { /* Loop over possible MGD words and add them to the output list */ __MGD_InitSuffixIterator(pLingInfo, &sIterator, sPrivateWord.Body.bBestMatchLen, sPrivateWord.Body.bBestMatchMin, wIndex, wLength, wMinWordLen, bMGDSection); /* Loop over all suffixes for this stem's classes and path-match length. */ /* Implicit parameter of pLingCmnInfo->Private.nSuffixes etc. */ while (__MGD_StepSuffixIterator(pLingInfo, &sIterator)) { ET9AWPrivWordInfo sPublicWord = sPrivateWord; sPublicWord.Body.wSuffixIndex = sIterator.wSuffixIndex; if (sIterator.wSuffixIndex) { /* For an MGD constructed word. */ ET9U16 wSuffixCharOffset = sIterator.snSuffixPos ? pLingCmnInfo->Private.pwSuffixCharOffsets[sIterator.snSuffixPos - 1] : 0; ET9U16 wSuffixLen = pLingCmnInfo->Private.pwSuffixCharOffsets[sIterator.snSuffixPos] - wSuffixCharOffset; ET9U16 wStemLen = sPrivateWord.Base.wWordLen; ET9U16 wSize = wStemLen + wSuffixLen; ET9U8 *pbSuffixChars = &(pLingCmnInfo->Private.pbSuffixChars[wSuffixCharOffset]); /* Internal letter codes of suffix. */ if (wSize > ET9MAXLDBWORDSIZE) { continue; } { ET9STATUS eMatchStatus; __ProfileStart; eMatchStatus = __LDB_CalcEditDistanceMGD(pLingInfo, &sPublicWord, pbSuffixChars, wIndex, wLength, wSize, 0 /* bPartialOK */); if (bMGDSection == _ET9AWMGD_STEM) { __ProfileEnd(tAW_SelLst_MgdStemMatching); } else { __ProfileEnd(tAW_SelLst_MgdSuffixMatching); } if (eMatchStatus) { continue; } } __MGD_ConcatStemSuffix(pLingCmnInfo, &sPublicWord, pbSuffixChars, wSize); } /* Rescore stand-alone word disallowing partial match. */ else if (bPartialOK && bMGDSection == _ET9AWMGD_STEM) { ET9STATUS eMatchStatus; __ProfileStart; eMatchStatus = __LDB_CalcEditDistanceMGD(pLingInfo, &sPublicWord, NULL /* pbSuffixChars */, wIndex, wLength, sPrivateWord.Base.wWordLen, 0 /* bPartialOK */); if (bMGDSection == _ET9AWMGD_STEM) { __ProfileEnd(tAW_SelLst_MgdStemMatching); } else { __ProfileEnd(tAW_SelLst_MgdSuffixMatching); } if (eMatchStatus) { continue; } } /* downshift word if option selected */ if (bDownShift) { ET9U16 wCount = sPublicWord.Base.wWordLen; ET9SYMB *pSymb = sPublicWord.Base.sWord; for (; wCount; --wCount, ++pSymb) { *pSymb = _ET9SymToLower(*pSymb, pLingCmnInfo->dwLdbNum); } } /* prepare and add to list */ __STAT_AWLdb_Flex_SLA_Call; /* Take care of locking and shifting. */ if (bMGDSection == _ET9AWMGD_STEM) { _ET9AWSelLstWordPreAdd(pLingInfo, &sPublicWord, wIndex, (ET9U8)wLength, 0, bFreqIndicator); } if (sIterator.bFreq != __ET9AWMGD_NOFREQ) { /* See projecttracking\SDLC\XT9INPUT\XT9_Core\XT9_MGD_FSD.doc for details. */ #define ET9_DB_TYPICAL_SIZE (ET9FREQPART)100000 /* A typical # of words in a non-MGD database. */ #define N_14 ET9_DB_TYPICAL_SIZE / 255 /* Number of words to simulate being at rank 14. */ ET9FREQPART fNumAtFreq = N_14 * (1 << (14 - sIterator.bFreq)); /* Simulated number of words at this frequency level. */ ET9FREQPART fNumAtHigherFreq = fNumAtFreq - N_14; /* Simulated number of words above this freq. level, and thus the "rank" of the most common word at this frequency. */ ET9FREQPART fInterpolate = 1 - ((ET9FREQPART) 1 / (sPublicWord.Body.dwWordIndex + 1) + (ET9FREQPART) 1 / 2 / (sPublicWord.Body.wSuffixIndex + 1)); /* Incorporate rank info to pick where in the range to end up. */ ET9FREQPART fPseudoRank = 1 + fNumAtHigherFreq + fInterpolate * fNumAtFreq; sPublicWord.Body.xWordFreq = (ET9FREQPART)ET9_DB_MAX_FREQ / fPseudoRank; /* Since MGD frequencies are base-two logarithmic, the top four values contain (1 / (2^8) + 1 / (2^7) + 1 / (2^ 6) + 1 / (2^ 5)) ~= 6% of the word list. */ sPublicWord.Body.bIsTop5 = sIterator.bFreq >= 7 + 4; } ET9Assert(sPublicWord.Body.xTapFreq >= 0); ET9Assert(sPublicWord.Body.xTotFreq >= 0); sPublicWord.Base.wWordCompLen = sPrivateWord.Base.wWordCompLen; if (ALDB.tags.bActive) { sPublicWord.Body.bCollectionPrio = __GetTagPriorityValue(pLingCmnInfo, __GetWordIndex()); } _ET9AWSelLstAdd(pLingInfo, &sPublicWord, wLength, bFreqIndicator); __STAT_AWLdb_Flex_SLA_Done; } } } _ET9AWCalcEditDistanceDone(pLingInfo); __STAT_AWLdb_Flex_Done; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static void ET9LOCALCALL __MGD_SwitchLdbSection (ET9AWLingInfo * const pLingInfo, const ET9U8 bMGDSection) { /* start with suffix build */ ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9STATUS wStatus = __ET9SearchInit(pLingInfo, bMGDSection == (_ET9AWMGD_SUFFIX) ? pLingCmnInfo->Private.ALdbMGD.dwSuffixAddress : ET9LDBOFFSET_BODY, 0 /* bReadCharTable */, bMGDSection == _ET9AWMGD_STEM /* bSkipCharTable */); if (wStatus) { /* database should already have successfully read */ ET9Assert(0); } __SetWordCount(pLingCmnInfo, pLingCmnInfo->Private.ALdbMGD.pSections[bMGDSection].dwNum); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static ET9STATUS ET9LOCALCALL __MGD_CopySuffixToTable (ET9AWLingInfo * const pLingInfo, const ET9AWPrivWordInfo * const pSuffix) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U16 wSuffixLen = pSuffix->Base.wWordLen; const ET9SYMB *pSource = &(pSuffix->Base.sWord[wSuffixLen - 1]); ET9UINT nIndex = pLingCmnInfo->Private.nSuffixes; ET9UINT nSuffixPos = nIndex ? pLingCmnInfo->Private.pwSuffixCharOffsets[nIndex - 1] : 0; ET9UINT i; if (nSuffixPos + wSuffixLen > ET9TOTSUFFIXCHARS || nIndex >= ET9MAXCOLLECTSIZE) { ET9Assert(0); return ET9STATUS_BUFFER_TOO_SMALL; } for (i = 0; i < wSuffixLen; ++i) { pLingCmnInfo->Private.pbSuffixChars[nSuffixPos++] = (ET9U8)__SymbolToCode(pLingCmnInfo, *pSource--); } pLingCmnInfo->Private.pwSuffixIndexes[nIndex] = (ET9U16)pSuffix->Body.dwWordIndex; pLingCmnInfo->Private.pwSuffixCharOffsets[nIndex] = (ET9U16)nSuffixPos; ++pLingCmnInfo->Private.nSuffixes; return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static void ET9LOCALCALL __MGD_EnableSuffixClasses (ET9AWLingInfo * const pLingInfo, const ET9AWPrivWordInfo * const pSuffix) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; __MGD_SearchPoint xSuffixSearchPoint; __MGD_InitClassesScan(pLingInfo, &xSuffixSearchPoint, pSuffix->Body.dwWordIndex, _ET9AWMGD_SUFFIX); while (__MGD_GetNextClassFreq(pLingInfo, &xSuffixSearchPoint)) { ET9U16 wClassID = xSuffixSearchPoint.wClassID; ET9Assert(wClassID < ET9MAXMGDCLASSES); __ET9BitSet(pLingCmnInfo->Private.pbEnabledClasses, wClassID); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * / * * * * * */ static void ET9LOCALCALL __ET9AWLdbGetCandidatesMGD (ET9AWLingInfo * const pLingInfo, const ET9U16 wIndex, const ET9U16 wLength, const ET9_FREQ_DESIGNATION bFreqIndicator, const ET9U8 bSpcMode) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9UINT nMaxSpcTermCountSave = ASPC.nMaxSpcTermCount; __ProfileStart; /* there should be space available for the suffix collection */ if (!pLingCmnInfo->Private.sWordC.pAuxC || !pLingCmnInfo->Private.sWordC.pAuxC->nMaxCollectSize) { return; } /* Start with suffix build. */ __MGD_SwitchLdbSection(pLingInfo, _ET9AWMGD_SUFFIX); __MGD_ReverseSymbsInfo(pLingCmnInfo, wIndex, wLength); _ET9AW_PushWordCollection(pLingInfo); { _ET9AW_Collection * const pCollecton = pLingCmnInfo->Private.sWordC.pCurrC; pCollecton->eCurrSelectionListMode = ET9ASLMODE_SUFFIX; ASPC.nMaxSpcTermCount = pCollecton->nMaxCollectSize; /* Don't limit number of suffixes with 1 error. */ } _ET9AW_ResetWordList(pLingInfo); pLingCmnInfo->Private.sWordC.pCurrC->bPureCollection = 1; __ET9AWLdbGetCandidatesPartialMGD(pLingInfo, wIndex, wLength, bFreqIndicator, bSpcMode, _ET9AWMGD_SUFFIX); __MGD_ReverseSymbsInfo(pLingCmnInfo, wIndex, wLength); pLingCmnInfo->Private.nSuffixes = pLingCmnInfo->Private.sWordC.pAuxC->nTotalWords; _ET9AWSortSearchListQS(pLingCmnInfo, 0, pLingCmnInfo->Private.sWordC.pAuxC->nTotalWords - 1, __MGD_CmpMatchLength); /* Sort by match length - current collection */ _ET9_LogSelList(pLingCmnInfo); ASPC.nMaxSpcTermCount = nMaxSpcTermCountSave; _ET9AW_PopWordCollection(pLingInfo); /* Switch back to stem build. */ __MGD_SwitchLdbSection(pLingInfo, _ET9AWMGD_STEM); { /* Copy the suffixes out of the build list. */ ET9UINT nIndex; ET9U8 bPrevMatchMin = 0; /* Word length we've already written out. */ ET9U8 bLongestPathForLen = 0; /* For this word length, longest path seen. */ pLingCmnInfo->Private.nSuffixes = 0; _ET9ClearMem(pLingCmnInfo->Private.pbEnabledClasses, sizeof(pLingCmnInfo->Private.pbEnabledClasses)); _ET9ClearMem(pLingCmnInfo->Private.pwSuffixPathExtra, sizeof(pLingCmnInfo->Private.pwSuffixPathExtra)); for (nIndex = 0; nIndex < pLingCmnInfo->Private.sWordC.pAuxC->nTotalWords; ++nIndex) { const ET9AWPrivWordInfo * const pSuffix = &(pLingCmnInfo->Private.sWordC.pAuxC->pWordList[pLingCmnInfo->Private.sWordC.pAuxC->pnWordList[nIndex]]); while (pSuffix->Body.bBestMatchMin > bPrevMatchMin) { pLingCmnInfo->Private.pwSuffixPerLength[bPrevMatchMin++] = (ET9U16)pLingCmnInfo->Private.nSuffixes; } if (bLongestPathForLen < bPrevMatchMin) { /* Stick with zero offset for lengths with no suffixes seen. */ bLongestPathForLen = bPrevMatchMin; } while (pSuffix->Body.bBestMatchLen > bLongestPathForLen && bLongestPathForLen <= ALDB_COMPARE_MAX_POS) { /* Point all match lengths back to this word length. */ ++bLongestPathForLen; pLingCmnInfo->Private.pwSuffixPathExtra[bLongestPathForLen - 1] = bLongestPathForLen - bPrevMatchMin; } if (__MGD_CopySuffixToTable(pLingInfo, pSuffix)) { break; } __MGD_EnableSuffixClasses(pLingInfo, pSuffix); } pLingCmnInfo->Private.pwSuffixPerLength[bPrevMatchMin] = (ET9U16)pLingCmnInfo->Private.nSuffixes; pLingCmnInfo->Private.bMaxSuffixLen = bPrevMatchMin; } __ProfileEnd(tAW_SelLst_MgdSuffixBuild); /* Now get the words with suffixes attached. */ __ET9AWLdbGetCandidatesPartialMGD(pLingInfo, wIndex, wLength, bFreqIndicator, bSpcMode, _ET9AWMGD_STEM); __ProfileEnd(tAW_SelLst_MgdStemBuild); } #else /* ET9_MGD_MODULE */ #define __ET9AWLdbGetCandidatesMGD(pLingInfo,wIndex,wLength,bFreqIndicator,bSpcMode) ET9Assert(0) #endif /* ET9_MGD_MODULE */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9STATUS ET9LOCALCALL __ET9AWLdbInit(ET9AWLingInfo * const pLingInfo) { ET9STATUS wStatus; ET9U8 byData; ET9U8 byPrimeLanguageID1; ET9U8 byPrimeLanguageID2; ET9U8 bySecondaryLanguageID1; ET9U8 bySecondaryLanguageID2; ET9U16 wDatabaseType; ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U32 dwLdbNum = pLingCmnInfo->dwLdbNum; ET9Assert(pLingInfo); /* Skip checking for pLingInfo since this is not an API. */ /* note: not checking wLDBInitOK here, because this is handled by calling function.*/ /* Layout version check */ wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_LDBLAYOUTVER, 1, &byData); if (wStatus) { return wStatus; } if (byData != 3) { return ET9STATUS_LDB_VERSION_ERROR; } /* Check the database type */ wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_DATABASETYPE, 1, &byData); if (wStatus) { return wStatus; } /* Database type should not be greater than ET9MAXDBTYPE. */ if (byData > ET9MAXDBTYPE) { return ET9STATUS_INVALID_DB_TYPE; } /* Convert the number to bit mask. */ wDatabaseType = (ET9U16)(1 << byData); if (!(wDatabaseType & (ET9DB_LDB_LATIN_MASK))) { return ET9STATUS_INVALID_DB_TYPE; } /* Skip flags check */ /* Compatibility Check */ wStatus = __ET9AWLdbCheckCompat(pLingInfo); if (wStatus != ET9STATUS_NONE) { return wStatus; } /* OEM ID Check */ wStatus = __ET9AWLdbCheckOEMID(pLingInfo); if (wStatus != ET9STATUS_NONE) { return wStatus; } /* check primary language id */ byPrimeLanguageID1 = (ET9U8)(dwLdbNum & ET9PLIDMASK); wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_PRIMARYLANGID, 1, &byPrimeLanguageID2); if (wStatus) { return wStatus; } if (byPrimeLanguageID1 != byPrimeLanguageID2) { return ET9STATUS_LDB_ID_ERROR; /* this is NOT the requested LDB */ } /* check secondary language id */ bySecondaryLanguageID1 = (ET9U8)((dwLdbNum & ET9SLIDMASK) >> 8); wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_SECONDARYLANGID, 1, &bySecondaryLanguageID2); if (wStatus) { return wStatus; } if (bySecondaryLanguageID1 != bySecondaryLanguageID2) { return ET9STATUS_LDB_ID_ERROR; /* this is NOT the requested LDB */ } /* Specific LDB internals */ wStatus = __ET9SearchInit(pLingInfo, ET9LDBOFFSET_BODY, 1, 0); if (wStatus) { return wStatus; } pLingCmnInfo->Private.bStateLDBEnabled = 1; /* investigate word count */ __InvestigateWordCount(pLingInfo); /* done */ return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U32 ET9LOCALCALL __CalculateHashValueFindWord(ET9SYMB const * const psBuf, const ET9U16 wBufLen) { ET9U32 dwHashValue = 0; ET9UINT nCount; ET9SYMB const * psSymb; psSymb = psBuf; for (nCount = wBufLen; nCount; --nCount, ++psSymb) { dwHashValue = *psSymb + (65599 * dwHashValue); } return dwHashValue; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AWLdbFindEntry(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9U8 bInflectOK, ET9SYMB const * const psWord, const ET9U16 wWordLen, ET9U32 * const pdwIndex, ET9U8 * const pbExact, ET9U8 * const pbLowercase) { ET9AWPrivWordInfo sWord; ET9STATUS eStatus; #if 0 __ProfileStart; #endif ET9Assert(pLingInfo); ET9Assert(psWord); if (!wWordLen) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFindEntry, zero length word\n");) return ET9STATUS_NO_MATCHING_WORDS; } { _ALDB_FindWordCache * const pCache = &pLingInfo->pLingCmnInfo->Private.sAldbFindWordCache; const ET9U32 dwHashValue = __CalculateHashValueFindWord(psWord, wWordLen); /* check if word exists in cache */ { ET9UINT nCount; _ALDB_FindWordCacheItem *pItem; ET9Assert(pCache->nCurrInsertIndex < _ALDB_FindWordCacheSize); pItem = &pCache->pItems[pCache->nCurrInsertIndex]; for (nCount = _ALDB_FindWordCacheSize; nCount; --nCount) { if (pItem->dwHashValue == dwHashValue && pItem->dwLdbNum == dwLdbNum) { *pdwIndex = pItem->dwIndex; *pbExact = pItem->bExact; *pbLowercase = pItem->bLowercase; #if 0 __ProfileEnd(_TT_AW_LDB_FindEntry); __ProfileCount(_CT_AW_LDB_FindEntryHash_H); #endif return pItem->eStatus; } if (--pItem < pCache->pItems) { pItem = &pCache->pItems[_ALDB_FindWordCacheSize - 1]; } } } /* perform find */ { _InitPrivWordInfo(&sWord); sWord.Base.wWordLen = wWordLen; sWord.Base.wWordCompLen = 0; _ET9SymCopy(sWord.Base.sWord, psWord, wWordLen); eStatus = _ET9AWLdbFind(pLingInfo, dwLdbNum, bInflectOK, &sWord, pbExact, pbLowercase, 0 /* bRetrieve */); if (eStatus == ET9STATUS_WORD_EXISTS) { *pdwIndex = sWord.Body.dwWordIndex - 1; } WLOG3( if (eStatus == ET9STATUS_NO_MATCHING_WORDS) { fprintf(pLogFile3, "_ET9AWLdbFindEntry, ET9STATUS_NO_MATCHING_WORDS\n"); }) WLOG7(fprintf(pLogFile7, "_ET9AWLdbFindEntry, status %u (%c), dwWordIndex %u\n", eStatus, (eStatus == ET9STATUS_WORD_EXISTS ? 'Y' :'N'), *pdwIndex);) } /* update cache */ { ++pCache->nCurrInsertIndex; if (pCache->nCurrInsertIndex >= _ALDB_FindWordCacheSize) { pCache->nCurrInsertIndex = 0; } { _ALDB_FindWordCacheItem * const pItem = &pCache->pItems[pCache->nCurrInsertIndex]; pItem->dwLdbNum = dwLdbNum; pItem->dwHashValue = dwHashValue; pItem->dwIndex = *pdwIndex; pItem->bExact = *pbExact; pItem->bLowercase = *pbLowercase; pItem->eStatus = eStatus; } } } #if 0 __ProfileEnd(_TT_AW_LDB_FindEntry); __ProfileCount(_CT_AW_LDB_FindEntryHash_S); #endif return eStatus; } #ifdef ET9_MGD_MODULE /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U8 ET9LOCALCALL __MGD_GetStandAloneFreq(ET9AWLingInfo *pLingInfo, ET9U32 dwStemIndex) { __MGD_SearchPoint xSearchPoint; __MGD_InitClassesScan(pLingInfo, &xSearchPoint, dwStemIndex, _ET9AWMGD_STEM); if (__MGD_GetNextClassFreq(pLingInfo, &xSearchPoint) && xSearchPoint.wClassID == 0) { /* Stand-alone freq. is always first. */ return xSearchPoint.bFreq; } else { return __ET9AWMGD_NOFREQ; } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9U8 ET9LOCALCALL __MGD_GetStemSuffixFreq(ET9AWLingInfo *pLingInfo, ET9U32 dwStemIndex, ET9U32 dwSuffixIndex) { __MGD_SearchPoint xStemSearchPoint; __MGD_SearchPoint xSuffixSearchPoint; ET9U8 bBestFreq = __ET9AWMGD_NOFREQ; if (!dwStemIndex || !dwSuffixIndex) { return __ET9AWMGD_NOFREQ; /* Zero is stand-alone; can't match. */ } __MGD_InitClassesScan(pLingInfo, &xStemSearchPoint, dwStemIndex, _ET9AWMGD_STEM); __MGD_InitClassesScan(pLingInfo, &xSuffixSearchPoint, dwSuffixIndex, _ET9AWMGD_SUFFIX); for (;;) { /* Step through the classes of each component */ if (xSuffixSearchPoint.wClassID <= xStemSearchPoint.wClassID) { if (!__MGD_GetNextClassFreq(pLingInfo, &xSuffixSearchPoint)) { return bBestFreq; } } else { if (!__MGD_GetNextClassFreq(pLingInfo, &xStemSearchPoint)) { return bBestFreq; } } if (xStemSearchPoint.wClassID == xSuffixSearchPoint.wClassID) { ET9U8 bNewFreq = xStemSearchPoint.bFreq + xSuffixSearchPoint.bFreq; if (bBestFreq == __ET9AWMGD_NOFREQ || bNewFreq > bBestFreq) { bBestFreq = bNewFreq; } } } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9INLINE static void ET9LOCALCALL __MGD_ET9AWLdbFindSuffixes (ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord) { #if ET9MAXCOLLECTSIZE < ALDB_COMPARE_MAX_POS #error FindSuffixes expects at least ET9MAXCOLLECTSIZE is at least ALDB_COMPARE_MAX_POS. #endif ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9STATUS eStatus; ET9AWPrivWordInfo LocalWord; ET9Assert(pLingInfo); ET9Assert(pWord); WLOG3(fprintf(pLogFile3, "__MGD_ET9AWLdbFindSuffixes\n");) WLOG3Word(pLogFile3, "__MGD_ET9AWLdbFindSuffixes word", pWord); /* start word. Don't enable first letter of word, since a suffix can't be the whole word. */ eStatus = __ET9CompareStartWord(pLingInfo, pWord->Base.sWord + 1, pWord->Base.wWordLen - 1, 1 /* bPartialMatchOK */, 1 /* bReverse */); if (eStatus) { WLOG3(fprintf(pLogFile3, "__MGD_ET9AWLdbFindSuffixes, bad word\n");) return; } /* Switch to using the suffix portion of the LDB. */ __MGD_SwitchLdbSection(pLingInfo, _ET9AWMGD_SUFFIX); __ET9SearchStart(pLingInfo, LocalWord.Base.sWord, &LocalWord.Base.wWordLen, ET9MAXWORDSIZE, 0 /* bUsingFlex */); eStatus = ET9STATUS_NO_MATCHING_WORDS; pLingCmnInfo->Private.bMaxSuffixLen = 0; for (; !__IsExhausted(); __ET9SearchGetNextStd(pLingInfo)) { LocalWord.Body.dwWordIndex = __GetWordIndex() + 1; /* Word length includes "match" of all the nulls which are there for partial match. */ while (LocalWord.Base.wWordLen && !LocalWord.Base.sWord[LocalWord.Base.wWordLen - 1]) { --LocalWord.Base.wWordLen; } /* Require exact match (at end of input word, since this is a suffix). */ { ET9BOOL bExactMatch = 1; ET9U16 wCount = LocalWord.Base.wWordLen; ET9SYMB *pSymbWord = &pWord->Base.sWord[pWord->Base.wWordLen - 1]; ET9SYMB *pSymbLocal = LocalWord.Base.sWord; for (; wCount; --wCount, --pSymbWord, ++pSymbLocal) { if (*pSymbWord != *pSymbLocal) { bExactMatch = 0; break; } } if (bExactMatch) { while (pLingCmnInfo->Private.bMaxSuffixLen < LocalWord.Base.wWordLen) { pLingCmnInfo->Private.pwSuffixIndexes[pLingCmnInfo->Private.bMaxSuffixLen++] = 0; } pLingCmnInfo->Private.pwSuffixIndexes[LocalWord.Base.wWordLen - 1] = (ET9U16) LocalWord.Body.dwWordIndex; } } } /* Switch back to stem build. */ __MGD_SwitchLdbSection(pLingInfo, _ET9AWMGD_STEM); } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ ET9INLINE static ET9STATUS ET9LOCALCALL __MGD_ET9AWLdbFind (ET9AWLingInfo * const pLingInfo, ET9AWPrivWordInfo * const pWord, const ET9U8 bInflectOK, ET9U8 * const pbExact, ET9U8 * const pbLowercase, const ET9BOOL bRetrieve) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9STATUS eStatus; ET9AWPrivWordInfo LocalWord; ET9Assert(pLingInfo); ET9Assert(pWord); ET9Assert(pbExact); ET9Assert(pbLowercase); ET9Assert(pLingCmnInfo->Private.ALdbMGD.bSupported); WLOG3(fprintf(pLogFile3, "__MGD_ET9AWLdbFind\n");) WLOG3Word(pLogFile3, "__MGD_ET9AWLdbFind word", pWord); if (bInflectOK == ET9MGD_FULL_WORD && pWord->Base.wWordLen > 1) { /* Note that for bInflectOK == ET9MGD_STAND_ALONE, we only need to check that the stem can stand alone. */ __MGD_ET9AWLdbFindSuffixes(pLingInfo, pWord); } /* start word */ eStatus = __ET9CompareStartWord(pLingInfo, pWord->Base.sWord, pWord->Base.wWordLen, bInflectOK == ET9MGD_FULL_WORD /* bPartialMatchOK */, 0 /* bReverse */); if (eStatus) { WLOG3(fprintf(pLogFile3, "__MGD_ET9AWLdbFind, bad word\n");) return eStatus; } __ET9SearchStart(pLingInfo, LocalWord.Base.sWord, &LocalWord.Base.wWordLen, ET9MAXWORDSIZE, 0 /* bUsingFlex */); eStatus = ET9STATUS_NO_MATCHING_WORDS; for (; !__IsExhausted(); __ET9SearchGetNextStd(pLingInfo)) { ET9U8 bExactMatch = 1; ET9U8 bAllLowerCase = 1; ET9INT snSuffixLength; LocalWord.Body.dwWordIndex = __GetWordIndex() + 1; { ET9UINT nCount = pWord->Base.wWordLen; ET9SYMB *pSymbWord = pWord->Base.sWord; ET9SYMB *pSymbLocal = LocalWord.Base.sWord; LocalWord.Base.wWordLen = pWord->Base.wWordLen; /* Start with input length (due to null padding, a full match word will have input length plus one). */ for (; nCount; --nCount, ++pSymbWord, ++pSymbLocal) { if (!*pSymbLocal) { /* We found the null termination. */ LocalWord.Base.wWordLen = (ET9U16)(pWord->Base.wWordLen - nCount); break; } if (*pSymbWord != *pSymbLocal) { bExactMatch = 0; } if (_ET9SymIsUpper(*pSymbLocal, pWordSymbInfo->Private.dwLocale)) { bAllLowerCase = 0; } if (bRetrieve) { *pSymbWord = *pSymbLocal; } } snSuffixLength = pWord->Base.wWordLen - LocalWord.Base.wWordLen; ET9Assert(snSuffixLength >= 0); /* We only have exact match suffixes, so get LC and symbols from word. */ for (nCount = (ET9UINT)snSuffixLength; nCount; --nCount, ++pSymbWord) { if (_ET9SymIsUpper(*pSymbWord, pWordSymbInfo->Private.dwLocale)) { bAllLowerCase = 0; } if (bRetrieve) { *pSymbWord = *pSymbWord; } } } if (snSuffixLength > pLingCmnInfo->Private.bMaxSuffixLen || /* Didn't see a suffix this length. */ (snSuffixLength == 0 && __MGD_GetStandAloneFreq(pLingInfo, LocalWord.Body.dwWordIndex) == __ET9AWMGD_NOFREQ) || /* Can't stand alone. */ (snSuffixLength > 0 && __MGD_GetStemSuffixFreq(pLingInfo, LocalWord.Body.dwWordIndex, pLingCmnInfo->Private.pwSuffixIndexes[snSuffixLength - 1]) == __ET9AWMGD_NOFREQ)) { /* Suffix not found, or doesn't match stem's classes. */ continue; } if ((!_ET9_LanguageSpecific_ApplyCaseSensitiveRule(pLingInfo)) || bExactMatch) { eStatus = ET9STATUS_WORD_EXISTS; pWord->Body.dwWordIndex = LocalWord.Body.dwWordIndex; pWord->Body.wSuffixIndex = pLingCmnInfo->Private.pwSuffixIndexes[snSuffixLength]; if (bExactMatch) { WLOG3(fprintf(pLogFile3, "__MGD_ET9AWLdbFind, ET9STATUS_WORD_EXISTS exactly as entered\n");) *pbExact = 1; if (bAllLowerCase) { WLOG3(fprintf(pLogFile3, "__MGD_ET9AWLdbFind, ET9STATUS_WORD_EXISTS in lowercase form\n");) *pbLowercase = 1; } break; } } } WLOG3( if (eStatus == ET9STATUS_NO_MATCHING_WORDS) { fprintf(pLogFile3, "__MGD_ET9AWLdbFind, ET9STATUS_NO_MATCHING_WORDS\n"); } else if (!*pbLowercase && !*pbExact) { fprintf(pLogFile3, "__MGD_ET9AWLdbFind, ET9STATUS_WORD_EXISTS (but in new non-lowercase format)\n"); }) return eStatus; } #else #define __MGD_ET9AWLdbFind(pLingInfo, pWord, bInflectOK, pbExact, pbLowercase, bRetrieve) ET9STATUS_NO_MATCHING_WORDS #endif /*---------------------------------------------------------------------------*/ /* -IDR- * * * / * * * * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AWLdbFind (ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9U8 bInflectOK, ET9AWPrivWordInfo * const pWord, ET9U8 * const pbExact, ET9U8 * const pbLowercase, const ET9BOOL bRetrieve) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9WordSymbInfo * const pWordSymbInfo = pLingCmnInfo->Base.pWordSymbInfo; ET9STATUS eStatus; ET9AWPrivWordInfo LocalWord; ET9Assert(pLingInfo); ET9Assert(pWord); ET9Assert(pbExact); ET9Assert(pbLowercase); WLOG3(fprintf(pLogFile3, "_ET9AWLdbFind, dwLdbNum %08x, bInflectOK %u, bRetrieve %u\n", dwLdbNum, bInflectOK, bRetrieve);) WLOG3Word(pLogFile3, "_ET9AWLdbFind word", pWord); *pbExact = 0; *pbLowercase = 0; if (((dwLdbNum & ET9PLIDMASK) == ET9PLIDNone) || !ET9LDBENABLED(pLingCmnInfo)) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFind, no active LDB\n");) return ET9STATUS_NO_MATCHING_WORDS; } if (pWord->Base.wWordLen > ET9MAXLDBWORDSIZE) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFind, too long word\n");) return ET9STATUS_NO_MATCHING_WORDS; } /* assure cache */ eStatus = __AssureActiveLDB(pLingInfo, dwLdbNum); if (eStatus) { return eStatus; } if (pLingCmnInfo->Private.ALdbMGD.bSupported && bInflectOK != ET9MGD_STEM) { return __MGD_ET9AWLdbFind(pLingInfo, pWord, bInflectOK, pbExact, pbLowercase, bRetrieve); } /* start word */ { const ET9BOOL bReverse = 0; const ET9BOOL bPartialMatchOK = 0; eStatus = __ET9CompareStartWord(pLingInfo, pWord->Base.sWord, pWord->Base.wWordLen, bPartialMatchOK, bReverse); } if (eStatus) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFind, bad word\n");) return eStatus; } { const ET9BOOL bUsingFlex = 0; __ET9SearchStart(pLingInfo, LocalWord.Base.sWord, &LocalWord.Base.wWordLen, ET9MAXWORDSIZE, bUsingFlex); } eStatus = ET9STATUS_NO_MATCHING_WORDS; for (; !__IsExhausted(); __ET9SearchGetNextStd(pLingInfo)) { ET9U8 bExactMatch = 1; ET9U8 bAllLowerCase = 1; { ET9U16 wCount = pWord->Base.wWordLen; ET9SYMB *pSymbWord = pWord->Base.sWord; ET9SYMB *pSymbLocal = LocalWord.Base.sWord; ET9Assert(ALDB.compare.wLength < ALDB.header.bPosCount || LocalWord.Base.wWordLen == wCount); ET9Assert(ALDB.compare.wLength == ALDB.header.bPosCount || LocalWord.Base.wWordLen == wCount + 1); for (; wCount; --wCount, ++pSymbWord, ++pSymbLocal) { if (*pSymbWord != *pSymbLocal) { bExactMatch = 0; } if (bAllLowerCase && _ET9SymIsUpper(*pSymbLocal, pWordSymbInfo->Private.dwLocale)) { bAllLowerCase = 0; } if (bRetrieve) { *pSymbWord = *pSymbLocal; } } } if (bAllLowerCase) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFind, ET9STATUS_WORD_EXISTS in lowercase form\n");) *pbLowercase = 1; } if (bExactMatch || !_ET9_LanguageSpecific_ApplyCaseSensitiveRule(pLingInfo)) { eStatus = ET9STATUS_WORD_EXISTS; pWord->Body.dwWordIndex = __GetWordIndex() + 1; if (bExactMatch) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFind, ET9STATUS_WORD_EXISTS exactly as entered\n");) *pbExact = 1; break; } } } WLOG3( if (eStatus == ET9STATUS_NO_MATCHING_WORDS) { fprintf(pLogFile3, "_ET9AWLdbFind, ET9STATUS_NO_MATCHING_WORDS\n"); } else if (!*pbLowercase && !*pbExact) { fprintf(pLogFile3, "_ET9AWLdbFind, ET9STATUS_WORD_EXISTS (but in new non-lowercase format)\n"); }) WLOG3(fprintf(pLogFile3, "_ET9AWLdbFind, done (%u)\n", eStatus);) return eStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * * * * * */ static ET9STATUS ET9FARCALL __LdbFindAmbig (ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9SYMB const * const psWord, const ET9U16 wWordLen, ET9SimpleWord * const pWords, const ET9U16 wStartSymbIndex, ET9U8 * const pbExact, ET9U32 * const pdwIndex, ET9U8 * const pbNumberMatches, ET9U8 * const pbExactIndex) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9STATUS eStatus; ET9AWPrivWordInfo LocalWord; ET9U8 bNumMatches = 0; ET9U8 bExactMatch = 0; ET9Assert(pLingInfo); ET9Assert(pWords); ET9Assert(pbExact); ET9Assert(pbExactIndex); WLOG3(fprintf(pLogFile3, "__LdbFindAmbig\n");) *pbNumberMatches = 0; *pbExact = 0; *pbExactIndex = 0; if (((dwLdbNum & ET9PLIDMASK) == ET9PLIDNone) || !ET9LDBENABLED(pLingCmnInfo)) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFindAmbig, no active LDB\n");) return ET9STATUS_NO_MATCHING_WORDS; } if (wWordLen > ET9MAXLDBWORDSIZE) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFindAmbig, too long word\n");) return ET9STATUS_NO_MATCHING_WORDS; } /* assure cache */ eStatus = __AssureActiveLDB(pLingInfo, dwLdbNum); if (eStatus) { return eStatus; } /* start word */ eStatus = __ET9CompareStartAmbigWord(pLingInfo, psWord, wStartSymbIndex, wWordLen, 0 /* bPartialMatchOK */, 0 /* bReverse */); if (eStatus) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFindAmbig, bad word\n");) return eStatus; } __ET9SearchStart(pLingInfo, LocalWord.Base.sWord, &LocalWord.Base.wWordLen, ET9MAXWORDSIZE, 0 /* bUsingFlex */); eStatus = ET9STATUS_NO_MATCHING_WORDS; for (; !__IsExhausted(); __ET9SearchGetNextStd(pLingInfo)) { ET9U16 wNumMismatches = 0; { ET9U16 wIndex; ET9SYMB *pSymbLocal = LocalWord.Base.sWord; ET9BOOL bMismatch = 0; ET9Assert(ALDB.compare.wLength < ALDB.header.bPosCount || LocalWord.Base.wWordLen == wWordLen); ET9Assert(ALDB.compare.wLength == ALDB.header.bPosCount || LocalWord.Base.wWordLen == wWordLen + 1); for (wIndex = 0; wIndex < wWordLen; ++wIndex, ++pSymbLocal) { if (psWord[wIndex] != *pSymbLocal && psWord[wIndex] != _ET9SymToOther(*pSymbLocal, dwLdbNum)) { ++wNumMismatches; } if ((3 * wNumMismatches) > (wIndex + 3)) { bMismatch = 1; break; } } if (bMismatch) { continue; } } if (!_ET9_LanguageSpecific_ApplyCaseSensitiveRule(pLingInfo)) { ET9U16 wIndex; eStatus = ET9STATUS_WORD_EXISTS; pdwIndex[bNumMatches] = __GetWordIndex() + 1; for (wIndex = 0; wIndex < *ALDB.search.pwLength - 1; ++wIndex) { pWords[bNumMatches].sString[wIndex] = ALDB.search.psTarget[wIndex]; } pWords[bNumMatches].wLen = *ALDB.search.pwLength - 1; if (!bExactMatch && !wNumMismatches) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFindAmbig, ET9STATUS_WORD_EXISTS exactly as entered\n");) bExactMatch = 1; *pbExact = 1; *pbExactIndex = bNumMatches; } if (++bNumMatches == ET9AW_MAX_AMBIG_SEGMENT_MATCHES) { break; } } } WLOG3( if (eStatus == ET9STATUS_NO_MATCHING_WORDS) { fprintf(pLogFile3, "_ET9AWLdbFindAmbig, ET9STATUS_NO_MATCHING_WORDS\n"); }) *pbNumberMatches = bNumMatches; return eStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AWLdbFindAmbigEntry(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9SYMB const * const psWord, const ET9U16 wWordLen, const ET9U16 wStartSymbIndex, ET9SimpleWord * const psWords, ET9U8 * const pbExact, ET9U32 * const pdwIndex, ET9U8 * const pbNumberMatches, ET9U8 * const pbExactIndex) { ET9STATUS eStatus; ET9Assert(pLingInfo); ET9Assert(psWord); if (!wWordLen) { WLOG3(fprintf(pLogFile3, "_ET9AWLdbFindAmbigEntry, zero length word\n");) *pbExactIndex = 0; *pbNumberMatches = 0; return ET9STATUS_NO_MATCHING_WORDS; } { _ALDB_FindAmbigWordCache * const pCache = &pLingInfo->pLingCmnInfo->Private.sAldbFindAmbigWordCache; const ET9U32 dwHashValue = __CalculateHashValueFindWord(psWord, wWordLen); const ET9U16 wCacheIndex = (ET9U16)(dwHashValue % _ALDB_FindAmbigWordCacheSize); _ALDB_FindAmbigWordCacheItem * const pItem = &pCache->pCacheItems[wCacheIndex]; ET9Assert(wCacheIndex < _ALDB_FindAmbigWordCacheSize); /* check if word exists in cache */ { if (pItem->dwHashValue == dwHashValue && pItem->dwLdbNum == dwLdbNum && pItem->wWordLen == wWordLen && pItem->wStartSymbIndex == wStartSymbIndex) { *pbExact = pItem->bExact; *pbNumberMatches = pItem->bMatchCount; *pbExactIndex = pItem->bExactIndex; { ET9U16 wWordItemIndex; ET9U8 bIndex; wWordItemIndex = pItem->wFirstWordItemIndex; for (bIndex = 0; bIndex < *pbNumberMatches; ++bIndex) { _ALDB_FindWordCacheWordItem * const pWordItem = &pCache->pWordItems[wWordItemIndex]; ET9Assert(wWordItemIndex < _ALDB_FindAmbigWordCacheWordItemCount); ++wWordItemIndex; if (wWordItemIndex >= _ALDB_FindAmbigWordCacheWordItemCount) { wWordItemIndex = 0; } if (pWordItem->wOwner != wCacheIndex) { break; } pdwIndex[bIndex] = pWordItem->dwIndex; psWords[bIndex].wLen = pWordItem->wLen; psWords[bIndex].wCompLen = 0; _ET9SymCopy(psWords[bIndex].sString, pWordItem->sString, pWordItem->wLen); } if (bIndex >= *pbNumberMatches) { return pItem->eStatus; } } } } /* perform find */ eStatus = __LdbFindAmbig(pLingInfo, dwLdbNum, psWord, wWordLen, psWords, wStartSymbIndex, pbExact, pdwIndex, pbNumberMatches, pbExactIndex); /* update cache */ { pItem->dwLdbNum = dwLdbNum; pItem->dwHashValue = dwHashValue; pItem->wWordLen = wWordLen; pItem->wStartSymbIndex = wStartSymbIndex; pItem->bExact = *pbExact; pItem->bMatchCount = *pbNumberMatches; pItem->bExactIndex = *pbExactIndex; pItem->eStatus = eStatus; pItem->wFirstWordItemIndex = (ET9U16)pCache->nNextWordItemIndex; { ET9U8 bIndex; for (bIndex = 0; bIndex < *pbNumberMatches; ++bIndex) { _ALDB_FindWordCacheWordItem * const pWordItem = &pCache->pWordItems[pCache->nNextWordItemIndex]; ET9Assert(pCache->nNextWordItemIndex < _ALDB_FindAmbigWordCacheWordItemCount); ++pCache->nNextWordItemIndex; if (pCache->nNextWordItemIndex >= _ALDB_FindAmbigWordCacheWordItemCount) { pCache->nNextWordItemIndex = 0; } pWordItem->wOwner = wCacheIndex; pWordItem->dwIndex = pdwIndex[bIndex]; pWordItem->wLen = psWords[bIndex].wLen; _ET9SymCopy(pWordItem->sString, psWords[bIndex].sString, psWords[bIndex].wLen); } } } } return eStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * / * * * * * * * */ void ET9FARCALL _ET9AWLdbWordsSearch(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9U16 wIndex, const ET9U16 wLength, ET9U8 * const pbLdbEntries, const ET9_FREQ_DESIGNATION bFreqIndicator, const ET9U8 bSpcMode, const ET9BOOL bAllowContextTagging) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9UINT nStartValue = pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts; WLOG3(fprintf(pLogFile3, "_ET9AWLdbWordsSearch, dwLdbNum %08x, wIndex %u, wLength %u, bFreqIndicator %u, bSpcMode %u, bAllowContextTagging %u\n", dwLdbNum, wIndex, wLength, bFreqIndicator, bSpcMode, bAllowContextTagging);) *pbLdbEntries = 0; /* Major problems if any passed pointer params are NULL. */ ET9Assert(pLingInfo && pbLdbEntries); /* length within limits */ if ((ET9U16) (wIndex + wLength) > ET9MAXWORDSIZE) { ET9Assert(0); return; } /* if length is zero or ldb not initialized, no words from ldb. Return with no error, because only propagating low level read errors */ if (((dwLdbNum & ET9PLIDMASK) == ET9PLIDNone) || !wLength || !ET9LDBENABLED(pLingCmnInfo)) { return; } /* assure LDB cache */ if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { return; } /* pre tag words? */ if (bAllowContextTagging && wLength > 0 && wLength <= ET9AWLDB_MAX_COLLECTION_TAG_LEN) { _ET9ClearMem((ET9U8*)&ALDB.tags, (ET9U32)sizeof(ALDB.tags)); ALDB.tags.bActive = 1; if (pLingCmnInfo->Private.bUsingALM) { _ET9AWLdbNwpWordsSearch(pLingInfo, dwLdbNum, 1); } else if (pLingCmnInfo->Private.bUsingLM && pLingCmnInfo->Private.ALdbLM.bSupported) { __CollectionPriorityTagCBLM(pLingInfo); } } else { ALDB.tags.bActive = 0; } WLOG3(fprintf(pLogFile3, "_ET9AWLdbWordsSearch, tags.bActive = %c, dwCollisionCount %u, dwTotalCount %u, checksum %08X\n", pLingCmnInfo->Private.ALdb.tags.bActive ? 'Y' : 'N', pLingCmnInfo->Private.ALdb.tags.dwCollisionCount, pLingCmnInfo->Private.ALdb.tags.dwTotalCount, _ET9ByteCheckSum(pLingCmnInfo->Private.ALdb.tags.pbPriority, (ET9U32)sizeof(pLingCmnInfo->Private.ALdb.tags.pbPriority)));) /* get candidates */ if (pLingCmnInfo->Private.ALdbMGD.bSupported) { __ET9AWLdbGetCandidatesMGD(pLingInfo, wIndex, wLength, bFreqIndicator, bSpcMode); } else if (ASPC.bSpcFeatures && !ET9_FLEX_FEATURE_OPTIONAL_MODE(ASPC.bSpcFeatures)) { __ET9AWLdbGetCandidatesFlex(pLingInfo, wIndex, wLength, bFreqIndicator, bSpcMode); } else { __ET9AWLdbGetCandidates(pLingInfo, wIndex, wLength, bFreqIndicator, bSpcMode); } /* track shift significance */ if (ALDB.header.bLowerCount && ALDB.header.bUpperCount) { pLingCmnInfo->Private.bCurrBuildHasShiftSignificance = 1; } /* return the number of entries added to the selection list (proper value in most cases, != 0 is the most important part) */ if (pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts - nStartValue > 0xFF) { *pbLdbEntries = 0xFF; } else { *pbLdbEntries = (ET9U8)(pLingCmnInfo->Private.sWordC.pCurrC->nTotalWordInserts - nStartValue); } } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9INLINE static ET9BOOL ET9LOCALCALL __IsAcceptableNwpWord(ET9AWPrivWordInfo * const pWord) { if (pWord->Base.wWordLen == 1 && _ET9_GetSymbolClass(pWord->Base.sWord[0]) != ET9_AlphaSymbClass) { return 0; } if (pWord->Base.wWordLen == 3 && pWord->Base.sWord[0] == '.' && pWord->Base.sWord[1] == '.' && pWord->Base.sWord[2] == '.') { return 0; } return 1; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * * */ ET9BOOL ET9FARCALL _ET9AWLdbIsLMEntriesByIndex(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9U32 dwFirstWordIndex, const ET9U32 dwSecondWordIndex, ET9FREQPART * const pxWordFreq, ET9U8 * const pbNLMOrder) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9FREQPART xProb = 0; ET9U32 dwWordIndex = 16777215; ET9FREQPART xBackOffWt = 0; ET9Assert(pLingInfo); *pxWordFreq = 0; *pbNLMOrder = 0; if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { return 0; } if (dwFirstWordIndex < ET9UNKNOWN_INDEX && dwSecondWordIndex < ET9UNKNOWN_INDEX) { ET9FREQPART xFirstWordFreq; ET9U8 bFirstOrder; if (pLingCmnInfo->Private.bUsingALM) { const ET9U8 bMaxOrder = pLingCmnInfo->Private.ALdbNLM.bOrder; ET9U8 bBackOffOrder; ET9U8 bFound = 0; ET9Assert(bMaxOrder <= ET9NLM_MAX_ORDER); __ClearNLMCache(pLingInfo); _ET9AWLMGetWordFreqByIndex(pLingInfo, dwLdbNum, dwFirstWordIndex, &xFirstWordFreq, &bFirstOrder); __ClearNLMCache(pLingInfo); for (bBackOffOrder = bMaxOrder; bBackOffOrder > 1; --bBackOffOrder) { ET9U8 bOrderIndex; for (bOrderIndex = bBackOffOrder; bOrderIndex > 0; --bOrderIndex) { if (bOrderIndex <= 1) { dwWordIndex = dwSecondWordIndex; } else if (bOrderIndex == 2) { dwWordIndex = dwFirstWordIndex; } else { dwWordIndex = pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[bOrderIndex - 2 - 1]; } if (dwWordIndex == ET9NULL_INDEX) { break; } __ET9AWLMSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex, &xProb, &bFound); if (bFound) { *pbNLMOrder = bBackOffOrder; break; } } if (bFound) { xProb += xBackOffWt; #if 0 *pxWordFreq = (ET9FREQPART)ET9_DB_MAX_FREQ * (ET9FREQPART)_ET9pow_f(__fEulerNumberE, (ET9FLOAT)xProb); #endif *pxWordFreq = xFirstWordFreq * (ET9FREQPART)_ET9pow_f(__fEulerNumberE, (ET9FLOAT)xProb); return 1; } else { xBackOffWt += pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bMaxOrder - bBackOffOrder].pxBackOffWt[bMaxOrder - 2]; } } } else if (dwFirstWordIndex < ALDB.header.dwTopCount && dwSecondWordIndex < ALDB.header.dwTopCount) { if (pLingCmnInfo->Private.bUsingLM && pLingCmnInfo->Private.ALdbLM.bSupported) { ET9U16 wEWordFreq = 0; ET9U16 wTWordFreq = 0; const ET9U16 wContextWordClass = __ET9AWLdbGetWordClass(pLingInfo, dwFirstWordIndex); __LdbGetWordFreq(pLingInfo, dwSecondWordIndex, wContextWordClass, &xProb, &wEWordFreq, &wTWordFreq); *pxWordFreq = (ET9FREQPART)xProb; } else { *pxWordFreq = (ET9FREQPART)((ET9FREQPART)ET9_DB_MAX_FREQ / ((dwFirstWordIndex + 1) * (dwSecondWordIndex + 1))); } return 1; } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * * */ ET9BOOL ET9FARCALL _ET9AWLdbIsLMEntries(ET9AWLingInfo * const pLingInfo, ET9SimpleWord const * const pFirstWord, ET9SimpleWord const * const pSecondWord, ET9FREQPART * const pxWordFreq, ET9U8 * const pbNLMOrder) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U32 dwFirstWordIndex = __GetWordIndexFromString(pLingInfo, pFirstWord); const ET9U32 dwSecondWordIndex = __GetWordIndexFromString(pLingInfo, pSecondWord); ET9FREQPART xProb = 0; ET9U32 dwWordIndex = 16777215; ET9FREQPART xBackOffWt = 0; ET9Assert(pLingInfo); *pxWordFreq = 0; *pbNLMOrder = 0; if (dwFirstWordIndex < ET9UNKNOWN_INDEX && dwSecondWordIndex < ET9UNKNOWN_INDEX) { ET9AWPrivWordInfo sFirstWordPriv; __ClearNLMCache(pLingInfo); _ET9SimpleWordToPrivWord(pFirstWord, &sFirstWordPriv); sFirstWordPriv.Body.dwWordIndex = dwFirstWordIndex + 1; sFirstWordPriv.Body.xWordFreq = 0; /* Signal we have no unigram frequency. */ _ET9AWLMGetWordFreq(pLingInfo, pLingInfo->pLingCmnInfo->dwLdbNum, &sFirstWordPriv); if (_ET9AW_IsUsingALM(pLingInfo)) { const ET9U8 bMaxOrder = pLingCmnInfo->Private.ALdbNLM.bOrder; ET9U8 bBackOffOrder; ET9U8 bFound = 0; ET9Assert(bMaxOrder <= ET9NLM_MAX_ORDER); __ClearNLMCache(pLingInfo); for (bBackOffOrder = bMaxOrder; bBackOffOrder > 1; --bBackOffOrder) { ET9U8 bOrderIndex; for (bOrderIndex = bBackOffOrder; bOrderIndex > 0; --bOrderIndex) { if (bOrderIndex <= 1) { dwWordIndex = dwSecondWordIndex; } else if (bOrderIndex == 2) { dwWordIndex = dwFirstWordIndex; } else { dwWordIndex = pLingCmnInfo->Private.sCurrContext.pdwContextWordIndexes[bOrderIndex - 2 - 1]; } if (dwWordIndex == ET9NULL_INDEX) { break; } __ET9AWLMSearchWord(pLingInfo, bBackOffOrder, bOrderIndex, dwWordIndex, &xProb, &bFound); if (bFound) { *pbNLMOrder = bBackOffOrder; break; } } if (bFound) { xProb += xBackOffWt; #if 0 *pxWordFreq = (ET9FREQPART)ET9_DB_MAX_FREQ * (ET9FREQPART)_ET9pow_f(__fEulerNumberE, (ET9FLOAT)xProb); #endif *pxWordFreq = (ET9FREQPART)sFirstWordPriv.Body.xWordFreq * (ET9FREQPART)_ET9pow_f(__fEulerNumberE, (ET9FLOAT)xProb); return 1; } else { xBackOffWt += pLingCmnInfo->Private.ALdbNLM.ANLM_Cache[bMaxOrder - bBackOffOrder].pxBackOffWt[bMaxOrder - 2]; } } } else if (dwFirstWordIndex < ALDB.header.dwTopCount && dwSecondWordIndex < ALDB.header.dwTopCount) { if (pLingCmnInfo->Private.bUsingLM && pLingCmnInfo->Private.ALdbLM.bSupported) { ET9U16 wEWordFreq = 0; ET9U16 wTWordFreq = 0; const ET9U16 wContextWordClass = __ET9AWLdbGetWordClass(pLingInfo, dwFirstWordIndex); __LdbGetWordFreq(pLingInfo, dwSecondWordIndex, wContextWordClass, &xProb, &wEWordFreq, &wTWordFreq); *pxWordFreq = (ET9FREQPART)xProb; } else { *pxWordFreq = (ET9FREQPART)((ET9FREQPART)ET9_DB_MAX_FREQ / ((dwFirstWordIndex + 1) * (dwSecondWordIndex + 1))); } return 1; } } return 0; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * / * * * * * */ void ET9FARCALL _ET9AWLdbWordsByIndex(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9U32 * const pdwIndexes, const ET9UINT nIndexCount, const ET9U8 bNLMOrder, ET9AWPrivWordInfo * const pWords) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; const ET9U16 wLength = pLingCmnInfo->Base.pWordSymbInfo->bNumSymbs; /* Major problems if any passed pointer params are NULL. */ ET9Assert(pLingInfo && pdwIndexes); WLOG3(fprintf(pLogFile3, "_ET9AWLdbWordsByIndex, nIndexCount %u\n", nIndexCount);) WLOG7(fprintf(pLogFile7, "_ET9AWLdbWordsByIndex, nIndexCount %u\n", nIndexCount);) /* indexes to get? */ if (!nIndexCount) { return; } /* init result */ if (pWords) { ET9UINT nIndex; for (nIndex = 0; nIndex < nIndexCount; ++nIndex) { _InitPrivWordInfo(&pWords[nIndex]); } } /* if length is zero or ldb not initialized, no words from ldb. Return with no error, because only propagating low level read errors */ if (!dwLdbNum || !ET9LDBENABLED(pLingCmnInfo)) { return; } /* assure cache */ if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { return; } /* sort indexes */ for (;;) { ET9BOOL bDirty = 0; ET9UINT nIndex; for (nIndex = 0; nIndex + 1 < nIndexCount; ++nIndex) { if (pdwIndexes[nIndex] > pdwIndexes[nIndex + 1]) { const ET9U32 dwTmp = pdwIndexes[nIndex]; pdwIndexes[nIndex] = pdwIndexes[nIndex + 1]; pdwIndexes[nIndex + 1] = dwTmp; bDirty = 1; } } if (!bDirty) { break; } } /* get words */ { const ET9U16 wCodeZero = ALDB.header.wCodeZero; const ET9BOOL bDownShift = (ET9BOOL)(ET9DOWNSHIFTALLLDB(pLingCmnInfo) && ALDB.header.bUpperCount); ET9UINT nIndex; ET9U8 bCursorIndex; ET9ALdbCursorData *pCursor; ET9AWPrivWordInfo sPrivateWord; /* set up 'search' */ ALDB.search.psTarget = sPrivateWord.Base.sWord; ALDB.search.bExhausted = 0; /* set up private word*/ _InitPrivWordInfo(&sPrivateWord); sPrivateWord.Body.bWordSrc = ET9WORDSRC_LDB; sPrivateWord.Body.bLangIndexScoring = (pLingCmnInfo->dwLdbNum == pLingInfo->pLingCmnInfo->dwFirstLdbNum) ? ET9AWFIRST_LANGUAGE : ET9AWSECOND_LANGUAGE; /* set up cursors */ pCursor = ALDB.pCursors; for (bCursorIndex = 0; bCursorIndex < ALDB.header.bPosCount; ++bCursorIndex, ++pCursor) { pCursor->wCode = 0; pCursor->dwStartPos = 0; pCursor->dwEndPos = 0; pCursor->dwJumpPos = 0; pCursor->dwJumpAddress = 0; { ET9U8 ET9FARDATA_LDB *pbCurrData; __LdbSetIndex(pCursor, ALDB.header.pdwIntervalOffsets[bCursorIndex]); pCursor->pbCurrData = pbCurrData; if (!__IsExhausted()) { __ET9SearchGetNextIntervalSlow(pLingInfo, pCursor, bCursorIndex, 0, 0 /* bUsingFlex */); } } } /* get index words */ for (nIndex = 0; nIndex < nIndexCount && !__IsExhausted(); ++nIndex) { __ET9HeaderConstsGetNextInterval; pCursor = ALDB.pCursors; for (bCursorIndex = 0; bCursorIndex < ALDB.header.bPosCount; ++bCursorIndex, ++pCursor) { __ET9SearchCatchupStream(pLingInfo, pCursor, bCursorIndex, pdwIndexes[nIndex]); /* return inside macro */ if (__IsExhausted()) { return; } if (pCursor->wCode == wCodeZero) { break; } } sPrivateWord.Base.wWordLen = bCursorIndex; { ET9AWPrivWordInfo sPublicWord; sPublicWord = sPrivateWord; /* downshift word if option selected */ if (bDownShift) { ET9U16 wCount = sPublicWord.Base.wWordLen; ET9SYMB *pSymb = sPublicWord.Base.sWord; for (; wCount; --wCount, ++pSymb) { *pSymb = _ET9SymToLower(*pSymb, pLingCmnInfo->dwLdbNum); } } /* prepare and add to list */ { const ET9U32 dwCurrWordIndex = pdwIndexes[nIndex] + 1; sPublicWord.Body.xWordFreq = (ET9FREQPART)((ET9FREQPART)ET9_DB_MAX_FREQ / dwCurrWordIndex); /* zero freq becomes one in "add" */ sPublicWord.Body.wEWordFreq = 0; sPublicWord.Body.wTWordFreq = 0; sPublicWord.Body.dwWordIndex = dwCurrWordIndex; sPublicWord.Body.wSuffixIndex = 0; sPublicWord.Body.bNLMOrder = bNLMOrder; #ifdef ET9_DEBUG sPublicWord.Body.sScoreContext = pLingInfo->pLingCmnInfo->Private.sCurrContext; #endif } if (pWords) { pWords[nIndex] = sPublicWord; } else if (__IsAcceptableNwpWord(&sPublicWord)) { _ET9AWSelLstAdd(pLingInfo, &sPublicWord, wLength, FREQ_NORMAL); } } } ET9Assert(nIndex == nIndexCount); } WLOG3(fprintf(pLogFile3, "_ET9AWLdbWordsByIndex, done\n");) WLOG7(fprintf(pLogFile7, "_ET9AWLdbWordsByIndex, done\n");) } /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * */ static ET9STATUS ET9LOCALCALL __LdbLanguageModelInit(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9U32 dwAddressTracker; ET9Assert(pLingInfo); ET9Assert(pLingCmnInfo); WLOG3(fprintf(pLogFile3, "__LdbLanguageModelInit, dwLdbNum %08x\n", dwLdbNum);) pLingCmnInfo->Private.ALdbLM.bSupported = 0; pLingCmnInfo->Private.ALdbNLM.bSupported = 0; if (!ET9CONTEXTBASEDPREDICTION(pLingCmnInfo)) { return ET9STATUS_NONE; } /* if LDB has been initialized/validated */ if ((dwLdbNum & ET9PLIDMASK) != ET9PLIDNone && pLingInfo->Private.wLDBInitOK == ET9GOODSETUP) { ET9U8 bACMChunkFound = 0; ET9U8 bANLMChunkFound = 0; /* check to see if LDB has context chunk */ if (!pLingCmnInfo->Private.ALdbNLM.bSupported) { dwAddressTracker = _ET9AW_FindChunk(pLingInfo, ET9CONTEXT_CHUNK_ID); if (dwAddressTracker) { WLOG3(fprintf(pLogFile3, "__LdbLanguageModelInit, found ET9CONTEXT_CHUNK_ID\n");) /* Save the context model specific info in the lingustics struct */ pLingCmnInfo->Private.ALdbLM.bALMVersion = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9ALM_VERSION_OFFSET); if (pLingCmnInfo->Private.ALdbLM.bALMVersion < 1 || pLingCmnInfo->Private.ALdbLM.bALMVersion > 2) { return ET9STATUS_LDB_VERSION_ERROR; } bACMChunkFound = 1; pLingCmnInfo->Private.ALdbLM.bSupported = 1; pLingCmnInfo->Private.ALdbLM.bALMLangID = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9ALM_LANGUAGE_OFFSET); pLingCmnInfo->Private.ALdbLM.dwALMStartAddress = _ET9ReadLDBWord3(pLingInfo, dwAddressTracker + ET9ALM_START_ADDRESS_OFFSET); pLingCmnInfo->Private.ALdbLM.dwALMEndAddress = _ET9ReadLDBWord3(pLingInfo, dwAddressTracker + ET9ALM_END_ADDRESS_OFFSET); pLingCmnInfo->Private.ALdbLM.dwNumEntries = _ET9ReadLDBWord3(pLingInfo, dwAddressTracker + ET9ALM_NUM_ENTRIES_OFFSET); pLingCmnInfo->Private.ALdbLM.wNumClasses = _ET9ReadLDBWord2(pLingInfo, dwAddressTracker + ET9ALM_NUM_CLASSES_OFFSET); pLingCmnInfo->Private.ALdbLM.wEBits = _ET9ReadLDBWord2(pLingInfo, dwAddressTracker + ET9ALM_EMISSION_ENCODING_OFFSET); pLingCmnInfo->Private.ALdbLM.wTBits = _ET9ReadLDBWord2(pLingInfo, dwAddressTracker + ET9ALM_TRANSITION_ENCODING_OFFSET); #if 0 pLingCmnInfo->Private.ALdbLM.bScalingFactor = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9ALM_SCALE_FACTOR_OFFSET); pLingCmnInfo->Private.ALdbLM.bAddConstant = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9ALM_ADD_CONSTANT_OFFSET); #else pLingCmnInfo->Private.ALdbLM.xScalingFactor = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9ALM_SCALE_FACTOR_OFFSET); pLingCmnInfo->Private.ALdbLM.xAddConstant = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9ALM_ADD_CONSTANT_OFFSET); #endif } } dwAddressTracker = _ET9AW_FindChunk(pLingInfo, ET9NGRAM_CHUNK_ID); if (dwAddressTracker) { ET9U8 bCurrOrder; ET9U8 bMaxOrder; ET9U8 bLexValue; ET9U8 bQuantizationType; ET9U8 bConstStorage; ET9U8 bConstPrecision; ET9U16 wPrScaling; ET9U16 wBWtScaling; ET9U16 wNLMFormatVersion; ET9U32 dwInfoStartPtr; WLOG3(fprintf(pLogFile3, "__LdbLanguageModelInit, found ET9NGRAM_CHUNK_ID\n");) pLingCmnInfo->Private.ALdbNLM.wVersion = (ET9U16)((_ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_MINOR_VERSION_OFFSET) << 8) | _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_MAJOR_VERSION_OFFSET)); if (pLingCmnInfo->Private.ALdbNLM.wVersion < 1 || pLingCmnInfo->Private.ALdbNLM.wVersion > 1) { return ET9STATUS_LDB_VERSION_ERROR; } bANLMChunkFound = 1; pLingCmnInfo->Private.ALdbNLM.bSupported = 1; pLingCmnInfo->Private.ALdbNLM.wLangID = (ET9U16)((_ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_SECONDARY_LANGUAGE_OFFSET) << 8) | _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_PRIMARY_LANGUAGE_OFFSET)); pLingCmnInfo->Private.ALdbNLM.dwStartAddress = _ET9ReadLDBWord3(pLingInfo, dwAddressTracker + ET9AWLM_START_ADDRESS_OFFSET); pLingCmnInfo->Private.ALdbNLM.dwEndAddress = _ET9ReadLDBWord3(pLingInfo, dwAddressTracker + ET9AWLM_END_ADDRESS_OFFSET); pLingCmnInfo->Private.ALdbNLM.bPrBits = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_P_BITS); wPrScaling = (ET9U16)_ET9pow_f((256 / _ET9pow_f(2, pLingCmnInfo->Private.ALdbNLM.bPrBits)), 2); pLingCmnInfo->Private.ALdbNLM.bBWtBits = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_W_BITS); wBWtScaling = (ET9U16)_ET9pow_f((256 / _ET9pow_f(2, pLingCmnInfo->Private.ALdbNLM.bBWtBits)), 2); pLingCmnInfo->Private.ALdbNLM.bOrder = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_ORDER); ET9Assert(pLingCmnInfo->Private.ALdbNLM.bOrder <= ET9NLM_MAX_ORDER); wNLMFormatVersion = _ET9ReadLDBWord2(pLingInfo, dwAddressTracker + ET9AWLM_FORMAT_VERSION); if (wNLMFormatVersion < 2 || wNLMFormatVersion > 2) { return ET9STATUS_LDB_VERSION_ERROR; } bLexValue = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_VALUE_TYPE); if (bLexValue != ET9VALUE_NEGATIVE_LOG_PROBABILITY) { return ET9STATUS_LDB_VERSION_ERROR; } bQuantizationType = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_QUANTIZATION); if (bQuantizationType != ET9QUANTIZATION_UNIFORM) { return ET9STATUS_LDB_VERSION_ERROR; } bConstStorage = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_CONST_STORAGE); if (bConstStorage != ET9CONSTANT_STORAGE_SIMPLE_FLOAT) { return ET9STATUS_LDB_VERSION_ERROR; } bConstPrecision = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9AWLM_CONST_PRECISION); dwInfoStartPtr = dwAddressTracker + ET9AWLM_HEADER_SIZE - 1; bMaxOrder = pLingCmnInfo->Private.ALdbNLM.bOrder; for (bCurrOrder = 0; bCurrOrder < bMaxOrder; ++bCurrOrder) { const ET9U8 bOrderIndex = bMaxOrder - bCurrOrder; ET9ANLM_Info * const pANLM_Info = &pLingCmnInfo->Private.ALdbNLM.ANLM_Info[bCurrOrder]; pANLM_Info->dwInfoStartAddress = _ET9ReadLDBWord3(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_START_ADDRESS_OFFSET); pANLM_Info->dwInfoNumEntries = _ET9ReadLDBWord3(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_NUM_ENTRIES); pANLM_Info->dwCurrOrderStartAddress = _ET9ReadLDBWord3(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_CURR_ORDER_START_ADDRESS_OFFSET); pANLM_Info->dwCurrOrderNumEntries = _ET9ReadLDBWord3(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_CURR_ORDER_NUM_ENTRIES); #if 0 pANLM_Info->wPrScaleFactor = _ET9ReadLDBWord2(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_P_SCALE_FACTOR) + 1; pANLM_Info->wPrScaleFactor = pANLM_Info->wPrScaleFactor * wPrScaling; pANLM_Info->bPrAddConstant = _ET9ReadLDBByte(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_P_ADD_CONST); pANLM_Info->wBWtScaleFactor = _ET9ReadLDBWord2(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_W_SCALE_FACTOR) + 1; pANLM_Info->wBWtScaleFactor = pANLM_Info->wBWtScaleFactor * wBWtScaling; pANLM_Info->bBWtAddConstant = _ET9ReadLDBByte(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_W_ADD_CONST) + 1; #else pANLM_Info->xPrScaleFactor = (ET9FREQPART)((ET9FLOAT)wPrScaling * __ReadLDBFloat(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_P_SCALE_FACTOR, bConstPrecision)); pANLM_Info->xPrAddConstant = (ET9FREQPART)((ET9FLOAT)wPrScaling * __ReadLDBFloat(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_P_ADD_CONST, bConstPrecision)); pANLM_Info->xBWtScaleFactor = (ET9FREQPART)((ET9FLOAT)wBWtScaling * __ReadLDBFloat(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_W_SCALE_FACTOR, bConstPrecision)); pANLM_Info->xBWtAddConstant = (ET9FREQPART)((ET9FLOAT)wBWtScaling * __ReadLDBFloat(pLingInfo, dwInfoStartPtr + ET9AWLM_INFO_W_ADD_CONST, bConstPrecision)); #endif if (bOrderIndex == bMaxOrder && bOrderIndex > 2) { pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] = pANLM_Info->dwInfoStartAddress + pANLM_Info->dwInfoNumEntries * ET9WORD_INDEX_SIZE; pANLM_Info->ANLM_Offset.dwOffset[ET9CHILD_PTR_OFFSET_INDEX] = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] + pANLM_Info->dwInfoNumEntries * ET9CHILDREN_CT_SIZE; } else if (bOrderIndex == 2) { pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] = pANLM_Info->dwInfoStartAddress + pANLM_Info->dwInfoNumEntries * ET9WORD_INDEX_SIZE; pANLM_Info->ANLM_Offset.dwOffset[ET9CHILD_PTR_OFFSET_INDEX] = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] + pANLM_Info->dwInfoNumEntries * ET9CHILDREN_CT_SIZE; pANLM_Info->ANLM_Offset.dwOffset[ET9BACKOFF_WT_OFFSET_INDEX] = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILD_PTR_OFFSET_INDEX] + pANLM_Info->dwInfoNumEntries * ET9CHILD_PTR_SIZE; } else if (bOrderIndex == 1) { pANLM_Info->ANLM_Offset.dwOffset[ET9PROBABILITY_OFFSET_INDEX] = pANLM_Info->dwInfoStartAddress + pANLM_Info->dwInfoNumEntries * ET9WORD_INDEX_SIZE; } else { pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] = pANLM_Info->dwInfoStartAddress + pANLM_Info->dwInfoNumEntries * ET9WORD_INDEX_SIZE; pANLM_Info->ANLM_Offset.dwOffset[ET9CHILD_PTR_OFFSET_INDEX] = pANLM_Info->ANLM_Offset.dwOffset[ET9CHILDREN_CT_OFFSET_INDEX] + pANLM_Info->dwInfoNumEntries * ET9CHILDREN_CT_SIZE; } dwInfoStartPtr += ET9AWLM_INFO_SIZE; } pLingCmnInfo->Private.ALdbNLM.ANLM_Cache->ANLM_Cache_Data[0].dwAddress = 0; ++dwInfoStartPtr; pLingCmnInfo->Private.ALdbLM.bALMVersion = _ET9ReadLDBByte(pLingInfo, dwInfoStartPtr + ET9ALM_VERSION_OFFSET); if (pLingCmnInfo->Private.ALdbLM.bALMVersion < 2 || pLingCmnInfo->Private.ALdbLM.bALMVersion > 2) { return ET9STATUS_LDB_VERSION_ERROR; } bACMChunkFound = 1; pLingCmnInfo->Private.ALdbLM.bSupported = 1; pLingCmnInfo->Private.ALdbLM.bALMLangID = _ET9ReadLDBByte(pLingInfo, dwInfoStartPtr + ET9ALM_LANGUAGE_OFFSET); pLingCmnInfo->Private.ALdbLM.dwALMStartAddress = _ET9ReadLDBWord3(pLingInfo, dwInfoStartPtr + ET9ALM_START_ADDRESS_OFFSET); pLingCmnInfo->Private.ALdbLM.dwALMEndAddress = _ET9ReadLDBWord3(pLingInfo, dwInfoStartPtr + ET9ALM_END_ADDRESS_OFFSET); pLingCmnInfo->Private.ALdbLM.dwNumEntries = _ET9ReadLDBWord3(pLingInfo, dwInfoStartPtr + ET9ALM_NUM_ENTRIES_OFFSET); pLingCmnInfo->Private.ALdbLM.wNumClasses = _ET9ReadLDBWord2(pLingInfo, dwInfoStartPtr + ET9ALM_NUM_CLASSES_OFFSET); pLingCmnInfo->Private.ALdbLM.wEBits = _ET9ReadLDBWord2(pLingInfo, dwInfoStartPtr + ET9ALM_EMISSION_ENCODING_OFFSET); pLingCmnInfo->Private.ALdbLM.wTBits = _ET9ReadLDBWord2(pLingInfo, dwInfoStartPtr + ET9ALM_TRANSITION_ENCODING_OFFSET); #if 0 pLingCmnInfo->Private.ALdbLM.bScalingFactor = _ET9ReadLDBByte(pLingInfo, dwInfoStartPtr + ET9ALM_SCALE_FACTOR_OFFSET); pLingCmnInfo->Private.ALdbLM.bAddConstant = _ET9ReadLDBByte(pLingInfo, dwInfoStartPtr + ET9ALM_ADD_CONSTANT_OFFSET); #else pLingCmnInfo->Private.ALdbLM.xScalingFactor = (ET9FREQPART)__ReadLDBFloat(pLingInfo, dwInfoStartPtr + ET9ALM_BACKOFF_SCALE_FACTOR_OFFSET, bConstPrecision); pLingCmnInfo->Private.ALdbLM.xAddConstant = (ET9FREQPART)__ReadLDBFloat(pLingInfo, dwInfoStartPtr + ET9ALM_BACKOFF_ADD_CONSTANT_OFFSET, bConstPrecision); #endif } if (bACMChunkFound) { if (!bANLMChunkFound) { if (pLingCmnInfo->Private.ALdbLM.bALMVersion < 1 || pLingCmnInfo->Private.ALdbLM.bALMVersion > 1) { pLingCmnInfo->Private.ALdbLM.bSupported = 0; return ET9STATUS_LDB_VERSION_ERROR; } } } else { pLingCmnInfo->Private.ALdbLM.bSupported = 0; pLingCmnInfo->Private.ALdbLM.dwNumEntries = 0; pLingCmnInfo->Private.ALdbLM.wNumClasses = 0; pLingCmnInfo->Private.ALdbLM.wEBits = 0; pLingCmnInfo->Private.ALdbLM.wTBits = 0; #if 0 pLingCmnInfo->Private.ALdbLM.bScalingFactor = 0; pLingCmnInfo->Private.ALdbLM.bAddConstant = 0; #else pLingCmnInfo->Private.ALdbLM.xScalingFactor = 0; pLingCmnInfo->Private.ALdbLM.xAddConstant = 0; #endif } if (bANLMChunkFound) { if (pLingCmnInfo->Private.ALdbLM.bALMVersion < 2 || pLingCmnInfo->Private.ALdbLM.bALMVersion > 2) { pLingCmnInfo->Private.ALdbNLM.bSupported = 0; return ET9STATUS_LDB_VERSION_ERROR; } } else { pLingCmnInfo->Private.ALdbNLM.bSupported = 0; pLingCmnInfo->Private.ALdbNLM.wVersion = 0; pLingCmnInfo->Private.ALdbNLM.wLangID = 0; pLingCmnInfo->Private.ALdbNLM.dwStartAddress = 0; pLingCmnInfo->Private.ALdbNLM.dwEndAddress = 0; pLingCmnInfo->Private.ALdbNLM.bOrder = 0; } } WLOG3(fprintf(pLogFile3, "__LdbLanguageModelInit, dwALMStartAddress %u, dwALMEndAddress %u, ALM size %u, dwNumEntries %u, wNumClasses %u\n", pLingCmnInfo->Private.ALdbLM.dwALMStartAddress, pLingCmnInfo->Private.ALdbLM.dwALMEndAddress, (pLingCmnInfo->Private.ALdbLM.dwALMEndAddress - pLingCmnInfo->Private.ALdbLM.dwALMStartAddress + 1), pLingCmnInfo->Private.ALdbLM.dwNumEntries, pLingCmnInfo->Private.ALdbLM.wNumClasses);) WLOG3(fprintf(pLogFile3, "__LdbLanguageModelInit, done\n");) return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9BOOL ET9FARCALL _ET9AW_IsUsingALM(ET9AWLingInfo * const pLingInfo) { ET9STATUS eStatus; ET9U32 dwActiveLdbNum; if ((eStatus = __ET9AWLdbBasicValidityCheck(pLingInfo, 0)) != ET9STATUS_NONE) { return 0; } if (!ET9CONTEXTBASEDPREDICTION(pLingInfo->pLingCmnInfo)) { return 0; } if (ET9AWLdbGetActiveLanguage(pLingInfo, &dwActiveLdbNum)) { return 0; } if (__AssureActiveLDB(pLingInfo, dwActiveLdbNum)) { return 0; } return pLingInfo->pLingCmnInfo->Private.ALdbNLM.bSupported; } #ifdef ET9_MGD_MODULE /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static void ET9LOCALCALL __LdbMgdSectionInit(ET9AWLingInfo * const pLingInfo, ET9U8 section, ET9U32 *pdwAddressTracker) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9Assert(pLingInfo); ET9Assert(pLingCmnInfo); pLingCmnInfo->Private.ALdbMGD.pSections[section].dwNum = _ET9ReadLDBWord3(pLingInfo, *pdwAddressTracker); *pdwAddressTracker += 3; pLingCmnInfo->Private.ALdbMGD.pSections[section].dwToClassOffsetsAddress = *pdwAddressTracker; *pdwAddressTracker += 3 * pLingCmnInfo->Private.ALdbMGD.pSections[section].dwNum; pLingCmnInfo->Private.ALdbMGD.pSections[section].dwToClassCount = _ET9ReadLDBWord3(pLingInfo, *pdwAddressTracker); *pdwAddressTracker += 3; pLingCmnInfo->Private.ALdbMGD.pSections[section].dwToClassAddress = *pdwAddressTracker; *pdwAddressTracker += pLingCmnInfo->Private.ALdbMGD.pSections[section].dwToClassCount; } /*---------------------------------------------------------------------------*/ /* -IDR- * / * * * * * */ ET9INLINE static ET9STATUS ET9LOCALCALL __MGD_Init(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9Assert(pLingInfo); ET9Assert(pLingCmnInfo); pLingCmnInfo->Private.ALdbMGD.bSupported = 0; /* if LDB has been initialized/validated */ if ((dwLdbNum & ET9PLIDMASK) != ET9PLIDNone && pLingInfo->Private.wLDBInitOK == ET9GOODSETUP) { ET9U32 dwAddressTracker = _ET9AW_FindChunk(pLingInfo, ET9MGDCLASS_CHUNK_ID); /* Read point in the LDB. */ /* check to see if LDB has MGD Class chunk */ if (dwAddressTracker) { ET9U8 bVersion = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9MGD_VERSION_OFFSET); if (bVersion != 1) { return ET9STATUS_LDB_VERSION_ERROR; } pLingCmnInfo->Private.ALdbMGD.bClassHuffBase = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9MGD_CLASS_HUFFMAN_OFFSET); pLingCmnInfo->Private.ALdbMGD.bNumFreqBits = _ET9ReadLDBByte(pLingInfo, dwAddressTracker + ET9MGD_CLASS_FREQ_BITS); { ET9U8 bNumClassBits = 8 - pLingCmnInfo->Private.ALdbMGD.bNumFreqBits; /* # of bits to represent the first piece of a class. */ pLingCmnInfo->Private.ALdbMGD.bInvClassHuffBase = (1 << bNumClassBits) - pLingCmnInfo->Private.ALdbMGD.bClassHuffBase; } dwAddressTracker += ET9MGD_NUM_CLASSES_OFFSET; pLingCmnInfo->Private.ALdbMGD.wNumClasses = _ET9ReadLDBWord2(pLingInfo, dwAddressTracker); dwAddressTracker += 2; __LdbMgdSectionInit(pLingInfo, _ET9AWMGD_SUFFIX, &dwAddressTracker); __LdbMgdSectionInit(pLingInfo, _ET9AWMGD_STEM, &dwAddressTracker); /* Check for a suffix section. */ dwAddressTracker = _ET9AW_FindChunk(pLingInfo, ET9APPEND_CHUNK_ID); ET9Assert(dwAddressTracker); /* A DB w/ a class/freq section and no suffix section is broken. */ if (!dwAddressTracker) { return ET9STATUS_CORRUPT_DB; } pLingCmnInfo->Private.ALdbMGD.dwSuffixAddress = dwAddressTracker + ET9MGD_SUFFIX_DATA_OFFSET; pLingCmnInfo->Private.ALdbMGD.bSupported = 1; } } return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9BOOL ET9FARCALL __MGD_ET9AW_IsLanguageMGD(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum) { if (__AssureActiveLDB(pLingInfo, dwLdbNum)) { return 0; } return pLingInfo->pLingCmnInfo->Private.ALdbMGD.bSupported; } #else #define __MGD_Init(pLingInfo,dwLdbNum) ET9STATUS_NONE #endif /* ET9_MGD_MODULE */ /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9BOOL ET9FARCALL _ET9AW_IsPrimaryMGD(ET9AWLingInfo * const pLingInfo) { #ifdef ET9_MGD_MODULE if (__ET9AWLdbBasicValidityCheck(pLingInfo, 0) != ET9STATUS_NONE) { return 0; } return __MGD_ET9AW_IsLanguageMGD(pLingInfo, pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingInfo->pLingCmnInfo->dwSecondLdbNum : pLingInfo->pLingCmnInfo->dwFirstLdbNum); #else ET9_UNUSED(pLingInfo); return 0; #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ ET9BOOL ET9FARCALL _ET9AW_IsUsingMGD(ET9AWLingInfo * const pLingInfo) { #ifdef ET9_MGD_MODULE if (_ET9AW_IsPrimaryMGD(pLingInfo)) { return 1; /* If the first is MGD, don't need to check the 2nd. */ } if (!ET9AW_GetBilingualSupported(pLingInfo)) { return 0; } return __MGD_ET9AW_IsLanguageMGD(pLingInfo, pLingInfo->pLingCmnInfo->Base.pWordSymbInfo->Private.bSwitchLanguage ? pLingInfo->pLingCmnInfo->dwFirstLdbNum : pLingInfo->pLingCmnInfo->dwSecondLdbNum); #else ET9_UNUSED(pLingInfo); return 0; #endif } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9U8 ET9LOCALCALL __ET9AWLdbGetChunkCount(ET9AWLingInfo * const pLingInfo) { ET9U8 bChunkCt = 0; ET9U8 bVer = 0; __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_CHUNK_COUNT_BYTE, 1, &bChunkCt); __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_LDBLAYOUTVER, 1, &bVer); if (bVer <= 3) { bChunkCt >>= 2; } return bChunkCt; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * */ static ET9U32 ET9LOCALCALL __ET9AWLdbGetLDBEndAddress(ET9AWLingInfo * const pLingInfo) { ET9U32 retAddress = 0; ET9U8 bPosCt = 0; ET9U32 iSymDecodeCtByte = 0; ET9U8 bSymDecCt[2] = {0}; ET9U16 wSymDecCt = 0; ET9U8 bIntervalOffset[3] = {0}; ET9U32 iLastIntervalOffset = 0; __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_BODY, 1, &bPosCt); iSymDecodeCtByte = ET9LDBOFFSET_BODY + 4 * bPosCt + 1; __ET9ReadLDBData(pLingInfo, iSymDecodeCtByte, 2, bSymDecCt); wSymDecCt = (ET9U16)((((ET9U16)bSymDecCt[0] << 8) & 0xFF00) | (bSymDecCt[1]& 0x00FF)); iLastIntervalOffset = iSymDecodeCtByte + 2 * (1 + wSymDecCt + 255) + 3 * bPosCt; __ET9ReadLDBData(pLingInfo, iLastIntervalOffset, 3, bIntervalOffset); retAddress = (ET9U32)((((ET9U32)bIntervalOffset[0] << 16) & 0xFF0000) | (((ET9U32)bIntervalOffset[1] << 8 ) & 0xFF00) | (bIntervalOffset[2] & 0xFF)); return retAddress - 1; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ static ET9U32 ET9LOCALCALL __ET9AWLdbGetChunkSize(ET9AWLingInfo * const pLingInfo, const ET9U32 dwChunkStartAddress) { ET9U32 dwChunkSize = 0; ET9U8 bSize[3] = {0}; __ET9ReadLDBData(pLingInfo, dwChunkStartAddress + 1, 3, bSize); dwChunkSize = (ET9U32)((((ET9U32)bSize[0] << 16) & 0xFF0000) | (((ET9U32)bSize[1] << 8) & 0xFF00) | (bSize[2] & 0xFF)); return dwChunkSize; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * */ ET9U32 ET9FARCALL _ET9AW_FindChunk(ET9AWLingInfo * const pLingInfo, const ET9U8 bChunk) { ET9U8 bChunkCt; ET9U32 dwAddressTracker; WLOG3(fprintf(pLogFile3, "_ET9AW_FindChunk, bChunk = %d\n", bChunk);) bChunkCt = __ET9AWLdbGetChunkCount(pLingInfo); dwAddressTracker = __ET9AWLdbGetLDBEndAddress(pLingInfo) + 1; while (bChunkCt--) { /* Get start and end address of chunk */ ET9U8 bChunkID; ET9STATUS wStatus = __ET9ReadLDBData(pLingInfo, dwAddressTracker, 1, &bChunkID); if (wStatus) { return 0; } if (bChunkID == bChunk) { return dwAddressTracker; } dwAddressTracker += __ET9AWLdbGetChunkSize(pLingInfo, dwAddressTracker); } return 0; /* failure */ } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * */ static ET9STATUS ET9LOCALCALL __GetOneLdbVersion(ET9AWLingInfo * const pLingInfo, ET9SYMB * psBuf, ET9U16 * const pwBufSize) { ET9STATUS wStatus; ET9U8 byData; ET9U8 bChunkCt = 0; ET9U32 dwAddressTracker = 0; ET9SYMB *psBufIn = psBuf; /* LDB version */ *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'T'; /* Database type */ wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_DATABASETYPE, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* Skip "aa" */ *psBuf++ = (ET9SYMB)'.'; /* LDB type */ wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_LDBLAYOUTVER, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* Skip "bb" */ *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'L'; /* Primary language ID, Secondary language ID, Symbol Class */ wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_PRIMARYLANGID, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* Skip "cc" */ *psBuf++ = (ET9SYMB)'.'; wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_SECONDARYLANGID, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* Skip "dd" */ *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'V'; /* Contents major and minor version, Contents and header deviation. */ wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_CONTENTSMAJORVER, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* Skip "ff" */ *psBuf++ = (ET9SYMB)'.'; wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_CONTENTSMINORVER, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* Skip "gg" */ *psBuf++ = (ET9SYMB)'.'; wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_CONTENTSDEVIATION, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* Skip "hh" */ *psBuf++ = (ET9SYMB)'.'; wStatus = __ET9ReadLDBData(pLingInfo, ET9LDBOFFSET_CONTENTSHEADERDEV, 1, &byData); if (wStatus) { return wStatus; } _ET9BinaryToHex(byData, psBuf); psBuf += 2; /* loop through LDB's chunks */ bChunkCt = __ET9AWLdbGetChunkCount(pLingInfo); dwAddressTracker = __ET9AWLdbGetLDBEndAddress(pLingInfo); while (bChunkCt--) { /* Get start and end address of chunk */ ET9U32 dwChunkStartAddress = dwAddressTracker + 1; ET9U32 dwChunkEndAddress = dwAddressTracker + __ET9AWLdbGetChunkSize(pLingInfo, dwChunkStartAddress); ET9U8 bChunkID; wStatus = __ET9ReadLDBData(pLingInfo, dwChunkStartAddress, 1, &bChunkID); if (bChunkID == ET9ASDB_CHUNK_ID) { /* Get ASDB version */ ET9U32 dwStartOfASDBheader = dwChunkStartAddress; ET9U8 bVersion = 0; *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'A'; *psBuf++ = (ET9SYMB)'S'; *psBuf++ = (ET9SYMB)'v'; wStatus = __ET9ReadLDBData(pLingInfo, dwStartOfASDBheader + 4, 1, &bVersion); if (wStatus) { return wStatus; } _ET9BinaryToHex(bVersion, psBuf); psBuf += 2; } else if (bChunkID == ET9CONTEXT_CHUNK_ID) { /* Get LM version */ ET9U32 dwStartOfLMheader = dwChunkStartAddress; ET9U8 bVersion; *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'L'; *psBuf++ = (ET9SYMB)'M'; *psBuf++ = (ET9SYMB)'v'; bVersion = _ET9ReadLDBByte(pLingInfo, dwStartOfLMheader + ET9ALM_VERSION_OFFSET); _ET9BinaryToHex(bVersion, psBuf); psBuf += 2; } else if (bChunkID == ET9NGRAM_CHUNK_ID) { /* Get ALM version */ ET9U8 bVersion; *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'A'; *psBuf++ = (ET9SYMB)'L'; *psBuf++ = (ET9SYMB)'M'; *psBuf++ = (ET9SYMB)'v'; bVersion = _ET9ReadLDBByte(pLingInfo, dwChunkStartAddress + ET9AWLM_MAJOR_VERSION_OFFSET); _ET9BinaryToHex(bVersion, psBuf); psBuf += 2; *psBuf++ = (ET9SYMB)'.'; bVersion = _ET9ReadLDBByte(pLingInfo, dwChunkStartAddress + ET9AWLM_MINOR_VERSION_OFFSET); _ET9BinaryToHex(bVersion, psBuf); psBuf += 2; } else if (bChunkID == ET9APPEND_CHUNK_ID) { /* Get extra wordlist version */ ET9U32 startOfAppendHeader = dwChunkStartAddress; ET9U8 bVersion; *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'A'; *psBuf++ = (ET9SYMB)'P'; *psBuf++ = (ET9SYMB)'P'; *psBuf++ = (ET9SYMB)'v'; bVersion = _ET9ReadLDBByte(pLingInfo, startOfAppendHeader + 4); _ET9BinaryToHex(bVersion, psBuf); psBuf += 2; } else if (bChunkID == ET9MGDCLASS_CHUNK_ID) { /* Get MGD Class version */ ET9U32 dwStartOfMGDheader = dwChunkStartAddress; ET9U8 bVersion; *psBuf++ = (ET9SYMB)' '; *psBuf++ = (ET9SYMB)'M'; *psBuf++ = (ET9SYMB)'G'; *psBuf++ = (ET9SYMB)'D'; *psBuf++ = (ET9SYMB)'v'; bVersion = _ET9ReadLDBByte(pLingInfo, dwStartOfMGDheader + 4); _ET9BinaryToHex(bVersion, psBuf); psBuf += 2; } dwAddressTracker = dwChunkEndAddress; } *psBuf = 0; *pwBufSize = (ET9U16) (psBuf - psBufIn); return wStatus; } /*---------------------------------------------------------------------------*/ /** * This function gets the version string from the active/current LDB. * * @param pLingInfo Pointer to alphabetic information structure. * @param psLdbVerBuf (out) buffer for the LDB version string. * @param wBufMaxSize Maximum buffer size for the LDB version. * @param pwBufSize (out) actual string length. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWLdbGetVersion(ET9AWLingInfo * const pLingInfo, ET9SYMB * const psLdbVerBuf, const ET9U16 wBufMaxSize, ET9U16 * const pwBufSize) { ET9STATUS wStatus; ET9U8 *pbyVer; ET9U16 wSizeWritten; ET9SYMB *psBuf = psLdbVerBuf; ET9U8 byTemplateStr[] = "XT9 LDB"; wStatus = __ET9AWLdbBasicValidityCheck(pLingInfo, 0); if (wStatus) { return wStatus; } if (psBuf == NULL || pwBufSize == NULL) { return ET9STATUS_INVALID_MEMORY; } if (wBufMaxSize < ET9MAXVERSIONSTR) { return ET9STATUS_NO_MEMORY; } /* Copy template string. */ pbyVer = byTemplateStr; while(*pbyVer) { *psBuf++ = (ET9SYMB)*pbyVer++; } *pwBufSize = 7; wStatus = __AssureActiveLDB(pLingInfo, pLingInfo->pLingCmnInfo->dwFirstLdbNum); if (wStatus) { return wStatus; } wStatus = __GetOneLdbVersion(pLingInfo, psBuf, &wSizeWritten); if (wStatus) { return wStatus; } psBuf += wSizeWritten; *pwBufSize = (ET9U16)(*pwBufSize + wSizeWritten); if (!ET9AW_GetBilingualSupported(pLingInfo)) { return wStatus; } wStatus = __AssureActiveLDB(pLingInfo, pLingInfo->pLingCmnInfo->dwSecondLdbNum); if (wStatus) { return wStatus; } wStatus = __GetOneLdbVersion(pLingInfo, psBuf, &wSizeWritten); if (wStatus) { return wStatus; } psBuf += wSizeWritten; *pwBufSize = (ET9U16)(*pwBufSize + wSizeWritten); return wStatus; } /*---------------------------------------------------------------------------*/ /* -IDR- * * * * * * * * */ ET9STATUS ET9FARCALL _ET9AWLdb_SetActiveLanguage(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum) { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; ET9STATUS wStatus; WLOG3(fprintf(pLogFile3, "_ET9AWLdbSetActiveLanguage, wLdbNum = %08x\n", dwLdbNum);) wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (wStatus) { return wStatus; } /* set current LDB (this value is used even if the LDB isn't) */ pLingCmnInfo->dwLdbNum = dwLdbNum; /* set locale (this value is used even if the LDB isn't) */ _ET9_SetAutoLocale(pLingCmnInfo->Base.pWordSymbInfo, dwLdbNum); /* consider it an LDB fail until proven ok */ pLingInfo->Private.wLDBInitOK = 0; /* */ if (!pLingInfo->ET9AWLdbReadData) { return ET9STATUS_NO_DB_INIT; } /* If no secondary language ID, set it to 1 */ if ((pLingCmnInfo->dwLdbNum & ET9SLIDMASK) == ET9SLIDNone) { pLingCmnInfo->dwLdbNum += ET9SLIDDEFAULT; } /* ok to have no DB, but it's won't be "init ok" */ if ((dwLdbNum & ET9PLIDMASK) == ET9PLIDNone) { return ET9STATUS_NONE; } /* assure LDB access */ wStatus = __InitDirectAccess(); if (wStatus) { return wStatus; } /* init the LDB */ wStatus = __ET9AWLdbInit(pLingInfo); if (wStatus) { return wStatus; } /* LDB ok */ pLingInfo->Private.wLDBInitOK = ET9GOODSETUP; /* check for LDB based AS support */ _ET9AWLdbASInit(pLingInfo, pLingCmnInfo->dwLdbNum, 0); wStatus = __LdbLanguageModelInit(pLingInfo, pLingCmnInfo->dwLdbNum); if (wStatus) { return wStatus; } wStatus = __MGD_Init(pLingInfo, dwLdbNum); if (wStatus) { return wStatus; } #ifdef ET9_DEBUG ++pLingCmnInfo->Private.dwSetActiveLanguageCount; #endif #ifdef ET9_DEBUGLOG3 { ET9U16 wStrLen; ET9SYMB psStr[ET9MAXVERSIONSTR]; if (!__GetOneLdbVersion(pLingInfo, psStr, &wStrLen)) { ET9U16 wIndex; char pcStr[ET9MAXVERSIONSTR + 1]; for (wIndex = 0; wIndex < wStrLen; ++wIndex) { pcStr[wIndex] = (ET9U8)psStr[wIndex]; } pcStr[wIndex] = 0; WLOG3(fprintf(pLogFile3, "_ET9AWLdbSetActiveLanguage, version = %s\n", pcStr);) } } #ifdef ET9_DIRECT_LDB_ACCESS { ET9U32 dwHashValue = 0; __ET9AWLdbHashChunk(pLingInfo, &dwHashValue, ET9LDBOFFSET_BODY, (ET9U32) -1); WLOG3(fprintf(pLogFile3, "_ET9AWLdbSetActiveLanguage, (body) checksum = %u\n", dwHashValue);) } #endif #endif return ET9STATUS_NONE; } /*---------------------------------------------------------------------------*/ /** * This function validates an LDB. * It can be called for an LDB other than the currently active one. * * @param pLingInfo Pointer to alphabetic information structure. * @param dwLdbNum LDB number. * @param ET9AWLdbReadData LDB read callback function. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWLdbValidate(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, ET9DBREADCALLBACK const ET9AWLdbReadData) { ET9STATUS wStatus; ET9U16 wLDBValidID; ET9U32 dwHashValue = 0; if (pLingInfo == NULL || ET9AWLdbReadData == NULL) { return ET9STATUS_INVALID_MEMORY; } if (pLingInfo->Private.wInfoInitOK != ET9GOODSETUP) { return ET9STATUS_NO_INIT; } if ((dwLdbNum & ET9PLIDMASK) == ET9PLIDNone) { return ET9STATUS_LDB_ID_ERROR; } { ET9AWLingCmnInfo * const pLingCmnInfo = pLingInfo->pLingCmnInfo; /* save previous LDB etc */ const ET9DBREADCALLBACK wOldLdbReadCallback = pLingInfo->ET9AWLdbReadData; pLingCmnInfo->dwLdbNum = dwLdbNum; pLingInfo->ET9AWLdbReadData = ET9AWLdbReadData; if ((pLingCmnInfo->dwLdbNum & ET9SLIDMASK) == ET9SLIDNone) { pLingCmnInfo->dwLdbNum += ET9SLIDDEFAULT; } if ((wStatus = __InitDirectAccess()) == ET9STATUS_NONE) { /* get the integrity ID from LDB */ wStatus = __ET9ReadLDBWord(pLingInfo, ET9LDBOFFSET_CHECKSUM, &wLDBValidID); if (!wStatus) { __ET9AWLdbHashChunk(pLingInfo, &dwHashValue, 0, ET9LDBOFFSET_CHECKSUM); __ET9AWLdbHashChunk(pLingInfo, &dwHashValue, ET9LDBOFFSET_CHECKSUM + 2, (ET9U32) -1); wStatus = (ET9STATUS)((wLDBValidID == (ET9U16)dwHashValue) ? ET9STATUS_NONE : ET9STATUS_CORRUPT_DB); } } /* restore previous setup */ pLingInfo->pLingCmnInfo->dwLdbNum = ET9PLIDNone; pLingInfo->ET9AWLdbReadData = wOldLdbReadCallback; pLingInfo->Private.wLDBInitOK = 0; } /* done*/ return wStatus; } /*---------------------------------------------------------------------------*/ /** * This function initializes ET9AWLingInfo structure. * It also sets up the initial language. * * @param pLingInfo Pointer to alphabetic information structure. * @param ET9AWLdbReadData LDB read callback function. * * @return ET9STATUS_NONE on success, otherwise return ET9 error code. */ ET9STATUS ET9FARCALL ET9AWLdbInit(ET9AWLingInfo * const pLingInfo, ET9DBREADCALLBACK const ET9AWLdbReadData) { ET9STATUS wStatus; wStatus = _ET9AWSys_BasicValidityCheck(pLingInfo); if (wStatus) { return wStatus; } pLingInfo->Private.wLDBInitOK = 0; if (!ET9AWLdbReadData) { return ET9STATUS_BAD_PARAM; } /* skip check ET9AWLdbReadData because it will be checked later on */ pLingInfo->ET9AWLdbReadData = ET9AWLdbReadData; /* init find word cache - only necessary for systems that uses the same LDB ID for different versions of the same LDB */ _ET9ClearMem((ET9U8*)&pLingInfo->pLingCmnInfo->Private.sAldbFindWordCache, (ET9U32)sizeof(pLingInfo->pLingCmnInfo->Private.sAldbFindWordCache)); _ET9ClearMem((ET9U8*)&pLingInfo->pLingCmnInfo->Private.sAldbFindAmbigWordCache, (ET9U32)sizeof(pLingInfo->pLingCmnInfo->Private.sAldbFindAmbigWordCache)); /**/ WLOG3(fprintf(pLogFile3, "\nET9AWLdbInit, pLingInfo = %p\n", pLingInfo);) WLOG3(fprintf(pLogFile3, "\n");) WLOG3(fprintf(pLogFile3, " sizeof(ET9AWLingInfo) = %6u\n", sizeof(ET9AWLingInfo));) WLOG3(fprintf(pLogFile3, " sizeof(ET9AWLingCmnInfo) = %6u\n", sizeof(ET9AWLingCmnInfo));) WLOG3(fprintf(pLogFile3, " sizeof(ET9ASpc) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc));) WLOG3(fprintf(pLogFile3, " sizeof(u) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u));) WLOG3(fprintf(pLogFile3, " sizeof(sCmpData) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpData));) WLOG3(fprintf(pLogFile3, " sizeof(ppbFreqRowStore) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpData.ppbFreqRowStore));) WLOG3(fprintf(pLogFile3, " sizeof(ppbCmpResultRowStore) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpData.ppbCmpResultRowStore));) WLOG3(fprintf(pLogFile3, " sizeof(pppbActiveSpc) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpData.pppbActiveSpc));) WLOG3(fprintf(pLogFile3, " sizeof(sCmpDataFlex) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex));) WLOG3(fprintf(pLogFile3, " sizeof(pbLockInfo) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.pbLockInfo));) WLOG3(fprintf(pLogFile3, " sizeof(pbIsFreqPos) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.pbIsFreqPos));) WLOG3(fprintf(pLogFile3, " sizeof(pbIsQualityKey) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.pbIsQualityKey));) WLOG3(fprintf(pLogFile3, " sizeof(pwPrevWordSC) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.pwPrevWordSC));) WLOG3(fprintf(pLogFile3, " sizeof(pbSubstFreqSC) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.pbSubstFreqSC));) WLOG3(fprintf(pLogFile3, " sizeof(ppbFreeDist) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.ppbFreeDist));) WLOG3(fprintf(pLogFile3, " sizeof(ppbEditDist) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.ppbEditDist));) WLOG3(fprintf(pLogFile3, " sizeof(ppbStemDist) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.ppbStemDist));) WLOG3(fprintf(pLogFile3, " sizeof(ppxStemFreq) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.ppxStemFreq));) WLOG3(fprintf(pLogFile3, " sizeof(psLockSymb) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.psLockSymb));) WLOG3(fprintf(pLogFile3, " sizeof(psLockSymbOther) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.psLockSymbOther));) WLOG3(fprintf(pLogFile3, " sizeof(pppbActiveSpc) = %6u\n", sizeof(pLingInfo->pLingCmnInfo->Private.ASpc.u.sCmpDataFlex.pppbActiveSpc));) WLOG3(fprintf(pLogFile3, "\n");) /**/ return ET9STATUS_NONE; } #if 0 /*---------------------------------------------------------------------------*/ /** * Get next symbols prediction. * * @param pLingInfo Pointer to alphabetic information structure. * @param pWordSymbInfo Pointer to word symbol info structure. * * @return Number of found symbols. */ ET9U8 ET9FARCALL _ET9AWLdbGetNextSymbolsPrediction(ET9AWLingInfo *pLingInfo, ET9AWWordSymbInfo *pWordSymbInfo) { ET9U16 wTargetLen; ET9SYMB psTargetWord[ET9MAXLDBWORDSIZE]; ET9SYMB psTargetSyms[ET9MAXLDBWORDSIZE]; ET9U32 pdwTargetSymsFreq[ET9MAXLDBWORDSIZE]; ET9U32 dwWordFreq; ET9UINT nSymbIndex = 0; ET9U16 wWordFreq; ET9U8 bI; ET9U8 bJ; ET9Assert(pLingInfo != NULL); ET9Assert(pWordSymbInfo != NULL); _ET9ClearMem((ET9U8*)psTargetSyms, sizeof(psTargetSyms)); _ET9ClearMem((ET9U8*)pdwTargetSymsFreq, sizeof(pdwTargetSymsFreq)); _ET9ClearMem((ET9U8*)pLingInfo->Private.sPredictSyms, sizeof(pLingInfo->Private.sPredictSyms)); __ET9CompareStart(pLingInfo, pWordSymbInfo, 0, pWordSymbInfo->bNumSymbs, 0 /* bUsingFlex */); __ET9SearchStart(pLingInfo, psTargetWord, &wTargetLen, ET9MAXLDBWORDSIZE); while (!__IsExhausted()) { if (psTargetWord[pWordSymbInfo->bNumSymbs]) { if (__GetWordIndex() >= 0xFFFF) { wWordFreq = 1; } else { wWordFreq = (ET9U16)(0xFFFF / (__GetWordIndex() + 1)); } /* search for existing until hit end */ for (bI = 0; bI < ET9MAXLDBWORDSIZE; ++bI) { if (!psTargetSyms[bI]) { /* create new entry */ psTargetSyms[bI] = psTargetWord[pWordSymbInfo->bNumSymbs]; pdwTargetSymsFreq[bI] = wWordFreq; break; } else if (psTargetSyms[bI] == psTargetWord[pWordSymbInfo->bNumSymbs]) { /* add new freq */ pdwTargetSymsFreq[bI] += wWordFreq; break; } } } __ET9SearchGetNext(); } /* now fill up predict symbols array */ for (bI = 0; bI < ET9PREDICT_SYMS; ++bI) { dwWordFreq = 0; for (bJ = 0; bJ < ET9MAXLDBWORDSIZE; ++bJ) { if (!psTargetSyms[bJ]) { break; } if (pdwTargetSymsFreq[bJ] > dwWordFreq) { dwWordFreq = pdwTargetSymsFreq[bJ]; nSymbIndex = bJ; } } pdwTargetSymsFreq[nSymbIndex] = 0; pLingInfo->Private.sPredictSyms[bI] = psTargetSyms[nSymbIndex]; } return bI; } #endif /*---------------------------------------------------------------------------*/ #ifdef ET9_LIMITED_PRIVATE_ACCESS ET9STATUS ET9FARCALL _PRIVATE_ET9AWLdbFindEntry(ET9AWLingInfo * const pLingInfo, const ET9U32 dwLdbNum, const ET9U8 bInflectOK, ET9SYMB const * const psWord, const ET9U16 wWordLen, ET9U32 * const pdwIndex, ET9U8 * const pbExact, ET9U8 * const pbLowercase) { return _ET9AWLdbFindEntry(pLingInfo, dwLdbNum, bInflectOK, psWord, wWordLen, pdwIndex, pbExact, pbLowercase); } #endif /* ET9_LIMITED_PRIVATE_ACCESS */ /*---------------------------------------------------------------------------*/ /* clean up defines */ #undef __GetPosLock #undef __SetPosLock #undef __IsCodeActive #undef __GetActiveSectionSpc #undef __GetActiveSectionFlexSpc #undef __IsCodeActiveSpc #undef __IsCodeExact #undef __ActivateSymbol #undef __DeactivateCode #undef __GetCodeFreq #endif /* ET9_ALPHABETIC_MODULE */ /*! @} */ /* ----------------------------------< eof >--------------------------------- */