/******************************************************************************* ;******************************************************************************* ;** ** ;** COPYRIGHT 2004-2012 NUANCE COMMUNICATIONS ** ;** ** ;** NUANCE COMMUNICATIONS PROPRIETARY INFORMATION ** ;** ** ;** This software is supplied under the terms of a license agreement ** ;** or non-disclosure agreement with Nuance Communications and may not ** ;** be copied or disclosed except in accordance with the terms of that ** ;** agreement. ** ;** ** ;** ** ;******************************************************************************* ;** ** ;** FileName: et9cprdb.h ** ;** ** ;** Description: Chinese XT9 RDB header file. ** ;** Conforming to the development version of Chinese XT9. ** ;** ** ;******************************************************************************* ;******************************************************************************/ #ifndef ET9CPRDB_H #define ET9CPRDB_H 1 /* don't mangle the function name if compile under C++ */ #if defined(__cplusplus) extern "C" { #endif #include "et9cpsys.h" #include "et9cpspel.h" /*---------------------------------------------------------------------------- * UDB parameters and constants. *----------------------------------------------------------------------------*/ /* constants related to an entry */ #define ET9_CP_NON_FREE_TYPE_MASK (0x80) /* whether the obj is non-free type */ #define ET9_CP_RDB_TYPE_MASK (0x40) /* whether the obj is RDB type */ #define ET9_CP_AUDB_TYPE_MASK (0x20) /* whether the obj is AUDB type */ #define ET9_CP_MIN_WORD_SIZE 1 /* min word size of an entry */ #define ET9_CP_WORD_SIZE_MASK (0x1F) /* mask for word size */ #define ET9_CP_FREE_SIZE_FLAG (0x40) /* whether the free obj uses 2 bytes for entry size */ #define ET9_CP_MAX_FREE_SIZE (0x3FFF)/* max size of a free entry */ #define ET9_CP_FREE_SIZE_UPPER_MASK (ET9_CP_FREE_SIZE_FLAG - 1) /* mask for upper byte of a free entry's size */ /* bit masks for get entry operation */ #define ET9_CP_GET_TYPE 0x01 #define ET9_CP_GET_ENTRY_SIZE 0x02 #define ET9_CP_GET_FREQ 0x04 #define ET9_CP_GET_ID 0x08 #define ET9_CP_GET_CHECKSUM 0x10 #define ET9_CP_MAX_UPDATE_COUNT ((ET9U16)500) /* max update count before rescale */ #define ET9_CP_MAXFREQ ((ET9U16)0x1FFF) /* max frequency */ #define ET9_CP_MINFREQ ((ET9U16)1) /* min frequency */ #define ET9_CP_AUDB_INITFREQ ((ET9U16)60) /* initial freq for AUDB type */ #define ET9_CP_AUDB_LEN_THRESHOLD ((ET9U16)8) /* max length to auto add full sentence as AUDB type */ #define ET9_CP_RDB_INITFREQ ((ET9U16)64) /* initial freq for RDB words */ #define ET9_CP_UDB_INITFREQ ((ET9U16)400) /* initial freq for UDB words */ #define ET9_CP_MDB_INITFREQ ((ET9U16)60) /* initial freq for MDB words */ #define ET9_CP_FREQ_RISE ((ET9U16)88) /* freq rise for one usage */ #define ET9_CP_UDB_RESCALE_FACTOR 32 /* freq rescale factor */ #define ET9_CP_UDB_CUTOFF_LEVELS 5 /* number of estimated cutoff freq levels */ #define ET9_CP_UDB_MIN_CUTOFF ((ET9U16)10) /* min cutoff freq */ #define ET9_CP_UDB_MIN_FREE_FACTOR 20 /* min factor (1/20) of free space before overcrowded */ #define ET9_CP_UDB_MIN_CLEANUP_FACTOR 20 /* min factor (1/20) to clean during clean up */ /* constants for UDB export and import */ typedef enum { ET9_CP_TUDB_VERSION_NONE = 0, ET9_CP_TUDB_VERSION_1 = 1, ET9_CP_TUDB_VERSION_MAX } ET9_CP_TUDB_VERSION; #define ET9_CP_TUDB_CURRENT_VERSION ET9_CP_TUDB_VERSION_1 #define ET9_CP_U8_ASCII_STR_SIZE 2 /* size of Hex ASCII string converted from a U8 value */ #define ET9_CP_U16_ASCII_STR_SIZE 4 /* size of Hex ASCII string converted from a U16 value */ #define ET9_CP_U32_ASCII_STR_SIZE 8 /* size of Hex ASCII string converted from a U32 value */ #define ET9_CP_MAX_UTF8_CHAR_SIZE 3 #define ET9_CP_TUDB_TYPE_SIZE 1 #define ET9_CP_TUDB_MODE_SIZE 1 #define ET9_CP_TUDB_FREQ_SIZE (ET9_CP_U16_ASCII_STR_SIZE + 1) #define ET9_CP_TUDB_MAX_PINYIN_SIZE ( (ET9_CP_MAX_PINYIN_SYL_SIZE + 1) * ET9CPMAXUDBPHRASESIZE + 1) #define ET9_CP_TUDB_MAX_BPMF_SIZE ( (ET9_CP_MAX_BPMF_SYL_SIZE * ET9_CP_MAX_UTF8_CHAR_SIZE + 1) * ET9CPMAXUDBPHRASESIZE + 1) #if (ET9_CP_TUDB_MAX_BPMF_SIZE < ET9_CP_TUDB_MAX_PINYIN_SIZE) #define ET9_CP_TUDB_MAX_SPELL_SIZE ET9_CP_TUDB_MAX_PINYIN_SIZE #else #define ET9_CP_TUDB_MAX_SPELL_SIZE ET9_CP_TUDB_MAX_BPMF_SIZE #endif #define ET9_CP_TUDB_MAX_PHRASE_SIZE (ET9_CP_MAX_UTF8_CHAR_SIZE * ET9CPMAXUDBPHRASESIZE + 1) #define ET9_CP_TUDB_MAX_HEADER_SIZE (ET9_CP_U8_ASCII_STR_SIZE + 1 \ + ET9MAXVERSIONSTR + 1 \ + ET9MAXVERSIONSTR + 1) /* TUDB Header: TUDB version ASCII string + Ldb version string + Core version string */ #define ET9_CP_TUDB_MAX_ENTRY_SIZE (ET9_CP_TUDB_TYPE_SIZE \ + ET9_CP_TUDB_MODE_SIZE \ + ET9_CP_TUDB_FREQ_SIZE \ + ET9_CP_TUDB_MAX_PHRASE_SIZE \ + ET9_CP_TUDB_MAX_SPELL_SIZE) #define ET9_CP_TUDB_USER_TYPE 'U' /* User-defined phrases */ #define ET9_CP_TUDB_AUTO_TYPE 'A' /* Auto-created phrases */ #define ET9_CP_TUDB_EOL_MARK 'E' /* End-of-list mark */ #define ET9_CP_TUDB_PINYIN_MODE 'P' /* spelling is Pinyin */ #define ET9_CP_TUDB_BPMF_MODE 'B' /* spelling is BPMF */ #define ET9_CP_TUDB_STROKE_MODE 'S' /* spelling is Stroke */ #define ET9_CP_TUDB_SYL_DELIMITER '.' #define ET9_CP_TUDB_FIELD_DELIMITER '\t' #define ET9_CP_TUDB_ENTRY_DELIMITER '\n' /* Given a == hi byte, b == lo byte; compose a word integral (ET9U16). */ #define ET9_CP_MAKEWORD(a, b) ((ET9U16)((ET9U8)(b) | ((ET9U16)((ET9U8)(a)) << 8))) #define ET9_CP_LOBYTE(w) ( (ET9U8)(w) ) #define ET9_CP_HIBYTE(w) ( (ET9U8)( (ET9U16)(w) >> 8 ) ) /* entry size = 3 + 2 * (word size) */ #define ET9_CP_WordSizeToEntrySize(bWordSize) ((ET9U8)(3 + ((bWordSize) << 1))) /* word size = ((entry size) - 3) / 2 */ #define ET9_CP_EntrySizeToWordSize(wEntrySize) ((ET9U8)(((wEntrySize) - 3) >> 1)) /* next zone */ #define ET9_CP_NextZone(b) ((b + 1) % ET9_CP_UDBZONEMAX) /*---------------------------------------------------------------------------- * Constants and Macros about UDB header info. *----------------------------------------------------------------------------*/ #define ET9_CP_UDBBPMFZONES ET9CPBpmfLetterCount /* based on BPMF letter */ #define ET9_CP_UDBPINYINZONES ET9CPPinyinLetterCount /* based on Pinyin letter */ #if (ET9_CP_UDBBPMFZONES > ET9_CP_UDBPINYINZONES) #define ET9_CP_UDBPHONETICZONES ET9_CP_UDBBPMFZONES #else #define ET9_CP_UDBPHONETICZONES ET9_CP_UDBPINYINZONES #endif #define ET9_CP_UDBSTROKEZONES 7 /* based on SID even distribution. */ #define ET9_CP_UDBZONEMAX (ET9_CP_UDBPHONETICZONES + ET9_CP_UDBSTROKEZONES) #define ET9_CP_UDB_TOTAL_SIZE_SIZE 2 /* Total size in bytes of the database */ #define ET9_CP_UDB_DATA_CHECK_SIZE 2 /* Additive checksum of database */ #define ET9_CP_UDB_DIRTY_COUNT_SIZE 4 /* Write counter for the last write. (multi-thread support) */ #define ET9_CP_UDB_UPDATE_COUNT_SIZE 2 /* Counter incremented when UDB is updated */ #define ET9_CP_UDB_FREE_BYTES_SIZE 2 /* Total number of free bytes in the UDB */ #define ET9_CP_UDB_CUTOFF_FREQ_SIZE 2 /* Cutoff freqency for cleaning up UDB entries when overcrowded */ #define ET9_CP_UDB_LDB_COMPAT_SIZE 2 /* Store LDB compatibility info */ #define ET9_CP_UDB_ZONE_OFFSET_SIZE 2 /* Offsets to beginning of each zone. */ #define ET9_CP_UDB_ZONE_WORDCOUNT_SIZE 2 /* Word counts in each zone. */ #define ET9_CP_UDB_TOTAL_SIZE_OFFSET 0 #define ET9_CP_UDB_DATA_CHECK_OFFSET (ET9_CP_UDB_TOTAL_SIZE_OFFSET + ET9_CP_UDB_TOTAL_SIZE_SIZE) #define ET9_CP_UDB_DIRTY_COUNT_OFFSET (ET9_CP_UDB_DATA_CHECK_OFFSET + ET9_CP_UDB_DATA_CHECK_SIZE) #define ET9_CP_UDB_UPDATE_COUNT_OFFSET (ET9_CP_UDB_DIRTY_COUNT_OFFSET + ET9_CP_UDB_DIRTY_COUNT_SIZE) #define ET9_CP_UDB_FREE_BYTES_OFFSET (ET9_CP_UDB_UPDATE_COUNT_OFFSET + ET9_CP_UDB_UPDATE_COUNT_SIZE) #define ET9_CP_UDB_CUTOFF_FREQ_OFFSET (ET9_CP_UDB_FREE_BYTES_OFFSET + ET9_CP_UDB_FREE_BYTES_SIZE) #define ET9_CP_UDB_LDB_COMPAT_OFFSET (ET9_CP_UDB_CUTOFF_FREQ_OFFSET + ET9_CP_UDB_CUTOFF_FREQ_SIZE) #define ET9_CP_UDB_ZONE_OFFSET_OFFSET (ET9_CP_UDB_LDB_COMPAT_OFFSET + ET9_CP_UDB_LDB_COMPAT_SIZE) #define ET9_CP_UDB_ZONE_WORDCOUNT_OFFSET (ET9_CP_UDB_ZONE_OFFSET_OFFSET + ET9_CP_UDB_ZONE_OFFSET_SIZE * ET9_CP_UDBZONEMAX) #define ET9_CP_UDB_HEADER_SIZE (ET9_CP_UDB_ZONE_WORDCOUNT_OFFSET + ET9_CP_UDB_ZONE_WORDCOUNT_SIZE * ET9_CP_UDBZONEMAX) /* Macros to get UDB header fields */ #define ET9_CP_UdbTotalSize(udb) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_TOTAL_SIZE_OFFSET) ) #define ET9_CP_UdbDataCheck(udb) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_DATA_CHECK_OFFSET) ) #define ET9_CP_UdbDirtyCount(udb) (ET9_CP_ReadU32( (udb) + ET9_CP_UDB_DIRTY_COUNT_OFFSET) ) #define ET9_CP_UdbUpdateCount(udb) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_UPDATE_COUNT_OFFSET) ) #define ET9_CP_UdbFreeBytes(udb) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_FREE_BYTES_OFFSET) ) #define ET9_CP_UdbCutoffFreq(udb) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_CUTOFF_FREQ_OFFSET) ) #define ET9_CP_UdbLdbCompat(udb) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_LDB_COMPAT_OFFSET) ) #define ET9_CP_UdbZoneOffset(udb, i) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_ZONE_OFFSET_OFFSET + ET9_CP_UDB_ZONE_OFFSET_SIZE * (i) ) ) #define ET9_CP_UdbZoneWordCount(udb, i) (ET9_CP_ReadU16( (udb) + ET9_CP_UDB_ZONE_WORDCOUNT_OFFSET + ET9_CP_UDB_ZONE_WORDCOUNT_SIZE * (i) ) ) /* The pointer to UDB data area */ #define ET9_CP_UdbDataStart(udb) ( (udb) + ET9_CP_UDB_HEADER_SIZE) /* The pointer to the byte following the end of UDB */ #define ET9_CP_UdbDataEnd(udb, size) ( (udb) + size ) /* The actual number of bytes in the UDB data area */ #define ET9_CP_UdbDataBytes(size) ( (ET9U16)(size - ET9_CP_UDB_HEADER_SIZE) ) #define ET9_CP_UdbPtrToOffset(udb, ptr) ( (ET9U16)( (ptr) - ET9_CP_UdbDataStart(udb) ) ) #define ET9_CP_UdbOffsetToPtr(udb, offset) (ET9_CP_UdbDataStart(udb) + (offset)) /* Test this thread's cached write count (dwDirtyCount) against the count from UDB/MDB. * * return : 1 = Another thread has changed UDB/MDB since we last read, * current search results may be invalid * (Note: caller should return ET9STATUS_NEED_SELLIST_BUILD to integration layer) * * 0 = UDB and MDB hasn't changed since we last read */ #define ET9_CP_IsUdbChangedByOtherThread(pET9CPLingInfo) \ ( ( (pET9CPLingInfo)->Udb.pDb && (pET9CPLingInfo)->Udb.dwDirtyCount != ET9_CP_UdbDirtyCount( (pET9CPLingInfo)->Udb.pDb ) ) \ || ( (pET9CPLingInfo)->Mdb.pDb && (pET9CPLingInfo)->Mdb.dwDirtyCount != ET9_CP_UdbDirtyCount( (pET9CPLingInfo)->Mdb.pDb ) ) ) /*---------------------------------------------------------------------------- * Define the RDB structure and variable types. *----------------------------------------------------------------------------*/ typedef enum ET9_CP_EntryType_e { ET9_CP_FREE = 0, ET9_CP_RDBPHONETIC = 1, ET9_CP_RDBSTROKE = 2, ET9_CP_AUDBPHONETIC = 3, ET9_CP_AUDBSTROKE = 4, ET9_CP_UDBPHONETIC = 5, ET9_CP_UDBSTROKE = 6 } ET9_CP_EntryType; #define ET9_CP_ENTRY_TYPE_MASK_FREE (1 << ET9_CP_FREE) #define ET9_CP_ENTRY_TYPE_MASK_RDBPHONETIC (1 << ET9_CP_RDBPHONETIC) #define ET9_CP_ENTRY_TYPE_MASK_RDBSTROKE (1 << ET9_CP_RDBSTROKE) #define ET9_CP_ENTRY_TYPE_MASK_AUDBPHONETIC (1 << ET9_CP_AUDBPHONETIC) #define ET9_CP_ENTRY_TYPE_MASK_AUDBSTROKE (1 << ET9_CP_AUDBSTROKE) #define ET9_CP_ENTRY_TYPE_MASK_UDBPHONETIC (1 << ET9_CP_UDBPHONETIC) #define ET9_CP_ENTRY_TYPE_MASK_UDBSTROKE (1 << ET9_CP_UDBSTROKE) #define ET9_CP_ENTRY_TYPE_MASK_RDB (ET9_CP_ENTRY_TYPE_MASK_RDBPHONETIC | ET9_CP_ENTRY_TYPE_MASK_RDBSTROKE) #define ET9_CP_ENTRY_TYPE_MASK_AUDB (ET9_CP_ENTRY_TYPE_MASK_AUDBPHONETIC | ET9_CP_ENTRY_TYPE_MASK_AUDBSTROKE) #define ET9_CP_ENTRY_TYPE_MASK_UDB (ET9_CP_ENTRY_TYPE_MASK_UDBPHONETIC | ET9_CP_ENTRY_TYPE_MASK_UDBSTROKE) #define ET9_CP_ENTRY_TYPE_MASK_ALL (ET9_CP_ENTRY_TYPE_MASK_RDB | ET9_CP_ENTRY_TYPE_MASK_AUDB | ET9_CP_ENTRY_TYPE_MASK_UDB) typedef struct ET9_CP_RUdbObj_s { ET9_CP_EntryType eType; /* Entry type */ /* ET9U8 bLang; reserved for future use */ ET9U16 wEntrySize; /* bytes occupied by the entry == 3 + 2w (where w == word length) */ ET9U16 wFreq; /* word frequency (ET9_CP_FREE entry: undefined) */ ET9U16 pwID[ET9CPMAXUDBPHRASESIZE]; /* Phonetic entry: Phonetic ID, Stroke entry: Stroke ID, ET9_CP_FREE entry: undefined */ } ET9_CP_RUdbObj; /*---------------------------------------------------------------------------- * Declare prototypes for RDB functions internal to the core *----------------------------------------------------------------------------*/ /* check if Mdb phrases should be given high priority in the selection list */ #define ET9_CP_MdbIsHighPriority(pLing) (pLing->bMdbHiPriority) /*------------------------------------------------------------------------ * Function : ET9_CP_MoveUdbPR * * Synopsis : Move a UDB pointer to the right by the given amount. * Wrap around at UDB end. * * Input: : pUdb = pointer to UDB * pbThis = pointer to start of the move * wNumMoves = number of bytes of the move amount * * Output : none * * Return : result pointer after the move *-----------------------------------------------------------------------*/ #if defined(ET9_DEBUG) || defined(WIN32_PLATFORM_WFSP) || defined(WIN32_PLATFORM_PSPC) ET9U8 ET9FARDATA * ET9FARCALL ET9_CP_MoveUdbPR(const ET9_CP_UdbInfo *pUdbInfo, const ET9U8 ET9FARDATA * pbThis, ET9U16 wNumMoves); #else /* release version for optimization */ #define ET9_CP_MoveUdbPR(pUdbInfo, pbThis, wNumMoves) \ ( (pbThis) + (wNumMoves) - ( ( (pbThis) + (wNumMoves) >= ET9_CP_UdbDataEnd( (pUdbInfo)->pDb, (pUdbInfo)->wDbSize ) ) ? ET9_CP_UdbDataBytes( (pUdbInfo)->wDbSize ) : 0 ) ) #endif /* ET9_DEBUG or windows mobile */ ET9U16 ET9FARCALL ET9_CP_GetEntryInfo(const ET9_CP_UdbInfo *pUdbInfo, const ET9U8 ET9FARDATA * pbData, ET9_CP_RUdbObj *pObj, ET9U8 bOption); /* Description : Check if the UDB is compatible with the LDB. * Return : ET9STATUS_NO_RUDB if has a UDB and the UDB is incompatible with LDB. */ ET9STATUS ET9FARCALL ET9_CP_CheckUdbCompat(ET9CPLingInfo *pET9CPLingInfo, ET9_CP_UdbInfo *pUdbInfo); ET9BOOL ET9FARCALL ET9_CP_UdbUsePhrase(ET9CPLingInfo *pET9CPLingInfo, ET9_CP_UdbInfo *pUdbInfo, ET9CPPhrase *pPhrase, ET9_CP_IDEncode eEncode, ET9U16 wFreq, ET9BOOL bIsAuto, ET9BOOL bForceAdd); void ET9FARCALL ET9_CP_GetUdbPhrases(ET9CPLingInfo *pET9CPLingInfo, ET9_CP_UdbInfo *pUdbInfo, ET9BOOL bNeedPartialSyl, ET9BOOL bNeedPartialPhrase, const ET9U16 *pwPrefix, ET9U8 bPrefixLen, ET9_CP_SpellMatch *pMatchType, ET9_CP_SpellData *pSpellData, ET9BOOL bIsSID, const ET9U8 *pbToneMask, ET9BOOL bValidateCntxPrefix, ET9_CP_PhraseBuf *pPhraseBuf); ET9BOOL ET9FARCALL ET9_CP_UdbDeleteUIDMatch(ET9CPLingInfo *pET9CPLingInfo, ET9_CP_UdbInfo *pUdbInfo, ET9CPPhrase *psUIDPhrase, ET9_CP_IDEncode eEncode, ET9U16 *pwAltID, ET9U8 bAltCount, ET9U16 wTargetTypeMask); void ET9FARCALL ET9_CP_UdbWriteBuf(ET9CPLingInfo *pLingInfo, ET9CPDBWRITECALLBACK pfDbWrite, ET9U8 ET9FARDATA *pDst, const ET9U8 ET9FARDATA *pSrc, ET9UINT nSize); /* End don't mangle the function name if compile under C++ */ #if defined(__cplusplus) } #endif #endif /* #ifndef ET9CPRDB_H */ /* ----------------------------------< eof >--------------------------------- */