From a8a914d47df5a2e98d743d2ae213bb8625b8ba3f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 13:39:01 +0800 Subject: [PATCH 1/8] minor changes --- source/dnode/mgmt/main/inc/dnd.h | 35 +++++++++++++++++---------- source/dnode/mgmt/main/inc/dndInt.h | 20 +++++++--------- source/dnode/mgmt/main/src/dndEnv.c | 37 ----------------------------- source/dnode/mgmt/main/src/dndInt.c | 16 ++++++++++++- source/dnode/mgmt/main/src/dndMsg.c | 29 +++++++++++++++++++--- 5 files changed, 72 insertions(+), 65 deletions(-) diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index 255a739add..f3a409f257 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -134,33 +134,42 @@ typedef struct SDnode { SMgmtWrapper wrappers[NODE_MAX]; } SDnode; +// dndFile.h +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); + +// dndInt.h +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); +void dndReleaseWrapper(SMgmtWrapper *pWrapper); + +// dndMonitor.h +void dndSendMonitorReport(SDnode *pDnode); + +// dndMsg.h +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); + +// dndStr.h +const char *dndStatStr(EDndStatus stat); const char *dndNodeLogStr(ENodeType ntype); const char *dndNodeProcStr(ENodeType ntype); const char *dndEventStr(EDndEvent ev); -EDndStatus dndGetStatus(SDnode *pDnode); -void dndSetStatus(SDnode *pDnode, EDndStatus stat); -void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -void dndSendMonitorReport(SDnode *pDnode); +// dndTransport.h int32_t dndInitServer(SDnode *pDnode); void dndCleanupServer(SDnode *pDnode); int32_t dndInitClient(SDnode *pDnode); void dndCleanupClient(SDnode *pDnode); -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); -int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); -int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); - -SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); -int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); -void dndReleaseWrapper(SMgmtWrapper *pWrapper); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index 612d35d513..0c4ada1df3 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -29,26 +29,24 @@ extern "C" { #endif -// dndInt.c -int32_t dndInit(); -void dndCleanup(); -const char *dndStatStr(EDndStatus stat); -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); +// dndEnv.h +int32_t dndInit(); +void dndCleanup(); -// dndMsg.c -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); - -// dndExec.c +// dndExec.h int32_t dndOpenNode(SMgmtWrapper *pWrapper); void dndCloseNode(SMgmtWrapper *pWrapper); int32_t dndRun(SDnode *pDnode); -// dndObj.c +// dndInt.c SDnode *dndCreate(const SDnodeOpt *pOption); void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); +// dndMsg.c +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); + // dndTransport.c int32_t dndInitMsgHandle(SDnode *pDnode); void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/src/dndEnv.c index 8792147822..9f75594335 100644 --- a/source/dnode/mgmt/main/src/dndEnv.c +++ b/source/dnode/mgmt/main/src/dndEnv.c @@ -57,40 +57,3 @@ void dndCleanup() { taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); } - -void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { - pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; - pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; -} - -EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } - -void dndSetStatus(SDnode *pDnode, EDndStatus status) { - if (pDnode->status != status) { - dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); - pDnode->status = status; - } -} - -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = { - .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; - rpcSendResponse(&rpcRsp); -} diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 6f6e21b983..25e4d89c5b 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -188,4 +188,18 @@ void dndReleaseWrapper(SMgmtWrapper *pWrapper) { int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); taosRUnLockLatch(&pWrapper->latch); dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); -} \ No newline at end of file +} + +void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { + pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; + pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; +} + +EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } + +void dndSetStatus(SDnode *pDnode, EDndStatus status) { + if (pDnode->status != status) { + dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); + pDnode->status = status; + } +} diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c index 2ab1e401c7..bb0f8ccb89 100644 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ b/source/dnode/mgmt/main/src/dndMsg.c @@ -66,8 +66,8 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); code = (*msgFp)(pWrapper, pMsg); } else if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, - pRpc->ahandle, pMsg->user); + dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, + pMsg->user); code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); } else { dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, @@ -171,4 +171,27 @@ int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; return -1; } -} \ No newline at end of file +} + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; + rpcSendResponse(&rpcRsp); +} From cf25aca809c13b9ffad81c6db43122276b070091 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 2 Apr 2022 15:08:48 +0800 Subject: [PATCH 2/8] [td-13039] refactor. --- include/common/tcommon.h | 36 +- include/libs/function/function.h | 11 +- source/dnode/vnode/inc/tsdb.h | 2 + source/dnode/vnode/src/tsdb/tsdbRead.c | 72 ++-- source/libs/executor/inc/executorimpl.h | 23 +- source/libs/executor/src/executorimpl.c | 529 +++++------------------- source/libs/function/src/taggfunction.c | 36 +- 7 files changed, 171 insertions(+), 538 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 8332e9d09a..45f608b24e 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -193,20 +193,19 @@ typedef struct SColumn { uint8_t scale; } SColumn; -typedef struct SLimit { - int64_t limit; - int64_t offset; -} SLimit; - -typedef struct SOrder { - uint32_t order; - SColumn col; -} SOrder; - -typedef struct SGroupbyExpr { - SArray* columnInfo; // SArray, group by columns information - bool groupbyTag; // group by tag or column -} SGroupbyExpr; +typedef struct STableBlockDistInfo { + uint16_t rowSize; + uint16_t numOfFiles; + uint32_t numOfTables; + uint64_t totalSize; + uint64_t totalRows; + int32_t maxRows; + int32_t minRows; + int32_t firstSeekTimeUs; + uint32_t numOfRowsInMemTable; + uint32_t numOfSmallBlocks; + SArray *dataBlockInfos; +} STableBlockDistInfo; enum { FUNC_PARAM_TYPE_VALUE = 0x1, @@ -241,15 +240,6 @@ typedef struct SExprInfo { struct tExprNode* pExpr; } SExprInfo; -typedef struct SStateWindow { - SColumn col; -} SStateWindow; - -typedef struct SSessionWindow { - int64_t gap; // gap between two session window(in microseconds) - SColumn col; -} SSessionWindow; - #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 9a518589b0..fe9edc323d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -52,7 +52,12 @@ typedef struct SFuncExecFuncs { FExecFinalize finalize; } SFuncExecFuncs; -#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results +typedef struct SFileBlockInfo { + int32_t numBlocksOfStep; +} SFileBlockInfo; + +#define TSDB_BLOCK_DIST_STEP_ROWS 8 +#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results #define FUNCTION_TYPE_SCALAR 1 #define FUNCTION_TYPE_AGG 2 @@ -101,10 +106,6 @@ typedef struct SFuncExecFuncs { #define FUNCTION_DERIVATIVE 32 #define FUNCTION_BLKINFO 33 -#define FUNCTION_HISTOGRAM 34 -#define FUNCTION_HLL 35 -#define FUNCTION_MODE 36 -#define FUNCTION_SAMPLE 37 #define FUNCTION_COV 38 diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 0d3fcffe7d..1e4a7e5a93 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -171,6 +171,8 @@ tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo); + bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle); /** diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7f2605ae2c..102d6fde90 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -14,7 +14,7 @@ */ #include "tsdbDef.h" -#include +#include "tdatablock.h" #include "os.h" #include "talgo.h" #include "tcompare.h" @@ -31,6 +31,7 @@ #include "tlosertree.h" #include "tsdbDef.h" #include "tmsg.h" +#include "tsdbCommit.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -209,34 +210,34 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load return pLocalIdList; } -//int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { -// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; -// -// int64_t rows = 0; -// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable; -// if (pMemTable == NULL) { return rows; } -// -//// STableData* pMem = NULL; -//// STableData* pIMem = NULL; -// -//// SMemTable* pMemT = pMemRef->snapshot.mem; -//// SMemTable* pIMemT = pMemRef->snapshot.imem; -// -// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); -// for (int32_t i = 0; i < size; ++i) { -// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); -// -//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { -//// pMem = pMemT->tData[pCheckInfo->tableId]; -//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; -//// } -//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { -//// pIMem = pIMemT->tData[pCheckInfo->tableId]; -//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; -//// } -// } -// return rows; -//} +int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; + + int64_t rows = 0; + STsdbMemTable* pMemTable = NULL;//pTsdbReadHandle->pMemTable; + if (pMemTable == NULL) { return rows; } + +// STableData* pMem = NULL; +// STableData* pIMem = NULL; + +// SMemTable* pMemT = pMemRef->snapshot.mem; +// SMemTable* pIMemT = pMemRef->snapshot.imem; + + size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + +// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { +// pMem = pMemT->tData[pCheckInfo->tableId]; +// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; +// } +// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { +// pIMem = pIMemT->tData[pCheckInfo->tableId]; +// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; +// } + } + return rows; +} static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList) { size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); @@ -2261,8 +2262,8 @@ static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { cur->mixBlock = false; cur->blockCompleted = false; } -#if 0 -int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTableBlockInfo) { + +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; @@ -2284,7 +2285,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); + int defaultRows = 4096;//TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { @@ -2345,16 +2346,15 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows; if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows; if (numOfRows < defaultRows) pTableBlockInfo->numOfSmallBlocks+=1; - int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; - SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); - blockInfo->numBlocksOfStep++; +// int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; +// SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); +// blockInfo->numBlocksOfStep++; } } } return code; } -#endif static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) { STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 485ae0a9c0..5a3e25bd68 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -128,6 +128,11 @@ typedef struct { int64_t sumRunTimes; } SOperatorProfResult; +typedef struct SLimit { + int64_t limit; + int64_t offset; +} SLimit; + typedef struct STaskCostInfo { int64_t created; int64_t start; @@ -163,6 +168,11 @@ typedef struct SOperatorCostInfo { uint64_t execCost; } SOperatorCostInfo; +typedef struct SOrder { + uint32_t order; + SColumn col; +} SOrder; + // The basic query information extracted from the SQueryInfo tree to support the // execution of query in a data node. typedef struct STaskAttr { @@ -196,7 +206,6 @@ typedef struct STaskAttr { STimeWindow window; SInterval interval; - SSessionWindow sw; int16_t precision; int16_t numOfOutput; int16_t fillType; @@ -206,13 +215,8 @@ typedef struct STaskAttr { int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. int32_t maxTableColumnWidth; int32_t tagLen; // tag value length of current query - SGroupbyExpr* pGroupbyExpr; SExprInfo* pExpr1; - SExprInfo* pExpr2; - int32_t numOfExpr2; - SExprInfo* pExpr3; - int32_t numOfExpr3; SColumnInfo* tableCols; SColumnInfo* tagColList; @@ -220,8 +224,6 @@ typedef struct STaskAttr { int64_t* fillVal; SSingleColumnFilterInfo* pFilterInfo; - // SFilterInfo *pFilters; - void* tsdb; STableGroupInfo tableGroupInfo; // table list SArray int32_t vgId; @@ -384,7 +386,7 @@ typedef struct SExchangeInfo { } SExchangeInfo; typedef struct STableScanInfo { - void* pTsdbReadHandle; + void* dataReader; int32_t numOfBlocks; // extract basic running information. int32_t numOfSkipped; int32_t numOfBlockStatis; @@ -644,12 +646,13 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3f895b3f1a..149af22249 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -249,7 +249,6 @@ static void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset); -static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); @@ -287,44 +286,6 @@ static int compareRowData(const void* a, const void* b, const void* userData) { return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; } -static void sortGroupResByOrderList(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv, - SSDataBlock* pDataBlock) { - SArray* columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - size_t size = taosArrayGetSize(columnOrderList); - taosArrayDestroy(columnOrderList); - - if (size <= 0) { - return; - } - - int32_t orderId = pRuntimeEnv->pQueryAttr->order.col.colId; - if (orderId <= 0) { - return; - } - - bool found = false; - int16_t dataOffset = 0; - - for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, j); - if (orderId == j) { - found = true; - break; - } - - dataOffset += pColInfoData->info.bytes; - } - - if (found == false) { - return; - } - - int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resSchema.type; - - SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; - taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); -} - // setup the output buffer for each operator SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); @@ -375,17 +336,6 @@ static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput // return (numOfSelectivity > 0 && hasTags); } -static bool isProjQuery(STaskAttr* pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) { - return false; - } - } - - return true; -} - static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) { if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -1411,15 +1361,10 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, - STimeWindow* win) { - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { + bool ascQuery = true; TSKEY curTs = tsCols[pos]; - TSKEY lastTs = *(TSKEY*)pRuntimeEnv->prevRow[0]; + TSKEY lastTs = 0;//*(TSKEY*)pRuntimeEnv->prevRow[0]; // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. // start exactly from this point, no need to do interpolation @@ -1434,27 +1379,24 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlF return true; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, - RESULT_ROW_START_INTERP); + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); return true; } static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfOutput = pOperatorInfo->numOfOutput; + int32_t order = TSDB_ORDER_ASC; + int32_t numOfOutput = pOperatorInfo->numOfOutput; TSKEY actualEndKey = tsCols[endRowIndex]; - - TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr) ? win->ekey : win->skey; + TSKEY key = order ? win->ekey : win->skey; // not ended in current data block, do not invoke interpolation - if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); return false; } @@ -1465,7 +1407,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun return true; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); int32_t nextRowIndex = endRowIndex + step; assert(nextRowIndex >= 0); @@ -1667,10 +1609,9 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; int32_t numOfOutput = pOperatorInfo->numOfOutput; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + bool ascQuery = true; TSKEY* tsCols = NULL; if (pSDataBlock->pDataBlock != NULL) { @@ -1683,7 +1624,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe int32_t startPos = ascQuery ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery); - STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); + STimeWindow win = {0};//getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SResultRow* pResult = NULL; @@ -1699,7 +1640,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + TSKEY ekey = 0;//reviseWindowEkey(pQueryAttr, &win); // forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, // binarySearchForKey, true); @@ -1713,31 +1654,31 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, // prevEndPos); if (startPos < 0) { - if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { - int32_t code = - setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - startPos = pSDataBlock->info.rows - 1; +// if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { +// int32_t code = +// setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, +// tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); +// if (code != TSDB_CODE_SUCCESS || pResult == NULL) { +// longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); +// } +// +// startPos = pSDataBlock->info.rows - 1; // window start(end) key interpolation // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, // forwardStep); doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, // forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - } +// } break; } setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } - if (pQueryAttr->timeWindowInterpo) { - int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0; +// if (pQueryAttr->timeWindowInterpo) { +// int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0; // saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); - } +// } // updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } @@ -2023,28 +1964,6 @@ static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCo return TSDB_CODE_SUCCESS; } -static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock) { - size_t num = taosArrayGetSize(pGroupbyExpr->columnInfo); - for (int32_t k = 0; k < num; ++k) { - SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - continue; - } - - int32_t colId = pColIndex->colId; - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - if (pColInfo->info.colId == colId) { - return i; - } - } - } - - assert(0); - return -1; -} - static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -2264,64 +2183,6 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return NULL; } -static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv* pRuntimeEnv, int32_t numOfTables, SArray* pOperator, - void* merger) { - // qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv)); - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->pQueryAttr = pQueryAttr; - - pRuntimeEnv->pResultRowHashTable = - taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - pRuntimeEnv->pResultRowListSet = - taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = taosMemoryMalloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); - // pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); - pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); - - pRuntimeEnv->prevRow = taosMemoryMalloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); - pRuntimeEnv->tagVal = taosMemoryMalloc(pQueryAttr->tagLen); - - // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pRuntimeEnv->pTableRetrieveTsMap = - taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - - // pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); - - if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || - pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { - goto _clean; - } - - if (pQueryAttr->numOfCols) { - char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*)pRuntimeEnv->prevRow; - pRuntimeEnv->prevRow[0] = start; - for (int32_t i = 1; i < pQueryAttr->numOfCols; ++i) { - pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i - 1].bytes; - } - - if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) { - *(int64_t*)pRuntimeEnv->prevRow[0] = INT64_MIN; - } - } - - // qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv)); - - // group by normal column, sliding window query, interval query are handled by interval query processor - // interval (down sampling operation) - return TSDB_CODE_SUCCESS; - -_clean: - // destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); - taosMemoryFreeClear(pRuntimeEnv->pResultRowHashTable); - taosMemoryFreeClear(pRuntimeEnv->keyBuf); - taosMemoryFreeClear(pRuntimeEnv->prevRow); - taosMemoryFreeClear(pRuntimeEnv->tagVal); - - return TSDB_CODE_QRY_OUT_OF_MEMORY; -} - static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -2400,17 +2261,6 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_Q // return false; //} -static bool isFirstLastRowQuery(STaskAttr* pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functionID == FUNCTION_LAST_ROW) { - return true; - } - } - - return false; -} - static bool isCachedLastQuery(STaskAttr* pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); @@ -2493,18 +2343,6 @@ static bool onlyOneQueryType(STaskAttr* pQueryAttr, int32_t functId, int32_t fun return true; } -static bool onlyFirstQuery(STaskAttr* pQueryAttr) { - return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); -} - -static bool onlyLastQuery(STaskAttr* pQueryAttr) { - return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); -} - -static bool notContainSessionOrStateWindow(STaskAttr* pQueryAttr) { - return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); -} - static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { bool hasFirstLastFunc = false; bool hasOtherFunc = false; @@ -3006,7 +2844,7 @@ int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, *status = BLK_DATA_ALL_NEEDED; - SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); + SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (pCols == NULL) { return terrno; } @@ -3899,76 +3737,6 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S return 0; } -// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, - SExprInfo* pExprInfo) { -#if 0 - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExprInfo1 = &(pExprInfo[i]); - if (pExprInfo1->base.functionId != FUNCTION_STDDEV_DST) { - continue; - } - - SExprBasicInfo* pExpr = &pExprInfo1->base; - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->info.colId == pExpr->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } -#endif -} - -void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, - SExprInfo* pExpr, char* val, int16_t bytes) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -#if 0 - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprBasicInfo* pExpr1 = &pExpr[i].base; - if (pExpr1->functionId != FUNCTION_STDDEV_DST) { - continue; - } - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->info.colId == pExpr1->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } -#endif -} - /* * There are two cases to handle: * @@ -4679,13 +4447,13 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { *newgroup = false; - while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { + while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { if (isTaskKilled(pOperator->pTaskInfo)) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); // todo opt // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -4722,7 +4490,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // The read handle is not initialized yet, since no qualified tables exists - if (pTableScanInfo->pTsdbReadHandle == NULL) { + if (pTableScanInfo->dataReader == NULL) { return NULL; } @@ -4750,11 +4518,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; - // if (pTaskInfo->pTsBuf) { - // bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - // assert(ret); - // } - // if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = 0; } @@ -4790,44 +4553,43 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { STableScanInfo* pTableScanInfo = pOperator->info; *newgroup = false; -#if 0 - STableBlockDist tableBlockDist = {0}; - tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + + STableBlockDistInfo tableBlockDist = {0}; + tableBlockDist.numOfTables = 1; // TODO set the correct number of tables int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { ++numRowSteps; } + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; pBlock->info.numOfCols = 1; - SBufferWriter bw = tbufInitWriter(NULL, false); - blockDistInfoToBinary(&tableBlockDist, &bw); +// SBufferWriter bw = tbufInitWriter(NULL, false); +// blockDistInfoToBinary(&tableBlockDist, &bw); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - int32_t len = (int32_t) tbufTell(&bw); - pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); - - *(int32_t*) pColInfo->pData = len; - memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); - - tbufCloseWriter(&bw); +// int32_t len = (int32_t) tbufTell(&bw); +// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); +// *(int32_t*) pColInfo->pData = len; +// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); +// +// tbufCloseWriter(&bw); SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); pOperator->status = OP_EXEC_DONE; return pBlock; -#endif } static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { @@ -5472,7 +5234,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, } pInfo->pFilterNode = pCondition; - pInfo->pTsdbReadHandle = pTsdbReadHandle; + pInfo->dataReader = pTsdbReadHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->order = order; @@ -5494,7 +5256,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbReadHandle; + pInfo->dataReader = pTsdbReadHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5515,32 +5277,42 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim return pOperator; } -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); +SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) { + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } - pInfo->pTsdbReadHandle = pTsdbReadHandle; + pInfo->dataReader = dataReader; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - SColumnInfoData infoData = {{0}}; - infoData.info.type = TSDB_DATA_TYPE_BINARY; + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_BINARY; infoData.info.bytes = 1024; infoData.info.colId = 0; taosArrayPush(pInfo->block.pDataBlock, &infoData); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableBlockInfoScanOperator"; + pOperator->name = "DataBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - // pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->getNextFn = doBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doBlockInfoScan; + + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } -SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, - SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5579,16 +5351,17 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pInfo->readerHandle = streamReadHandle; pInfo->pRes = pResBlock; - pOperator->name = "StreamBlockScanOperator"; + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doStreamBlockScan; - pOperator->closeFn = operatorDummyCloseFn; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doStreamBlockScan; + pOperator->closeFn = operatorDummyCloseFn; + pOperator->pTaskInfo = pTaskInfo; + return pOperator; } @@ -5922,80 +5695,6 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB return pOperator; } -SArray* getOrderCheckColumns(STaskAttr* pQuery) { - int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - if (pQuery->interval.interval > 0) { - if (pOrderColumns == NULL) { - pOrderColumns = taosArrayInit(1, sizeof(SColIndex)); - } - - SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL}; - taosArrayPush(pOrderColumns, &colIndex); - } - - { - numOfCols = (int32_t)taosArrayGetSize(pOrderColumns); - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base; - int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); - - if (index->colId == pExpr->pParam[0].pCol->colId && - (functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) { - index->colIndex = j; - index->colId = pExpr->resSchema.colId; - } - } - } - } - - return pOrderColumns; -} - -SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { - int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - - bool found = false; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base; - int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); - - // FUNCTION_TAG_DUMMY function needs to be ignored - // if (index->colId == pExpr->pColumns->info.colId && - // ((TSDB_COL_IS_TAG(pExpr->pColumns->flag) && functionId == FUNCTION_TAG) || - // (TSDB_COL_IS_NORMAL_COL(pExpr->pColumns->flag) && functionId == FUNCTION_PRJ))) { - // index->colIndex = j; - // index->colId = pExpr->resSchema.colId; - // found = true; - // break; - // } - } - - assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput); - } - - return pOrderColumns; -} - static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey); static void cleanupAggSup(SAggSupporter* pAggSup); @@ -7044,9 +6743,6 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr return pIntervalInfo->binfo.pRes; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -7062,14 +6758,14 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); +// setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); } pOperator->status = OP_RES_TO_RETURN; - pQueryAttr->order.order = order; // TODO : restore the order +// pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); @@ -8153,7 +7849,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; assert(pQueryAttr->numOfOutput == 1); SExprInfo* pExprInfo = &pOperator->pExpr[0]; @@ -8936,8 +8631,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod return pList; } -int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, - uint64_t queryId, uint64_t taskId) { +int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); @@ -9041,43 +8735,6 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol return TSDB_CODE_SUCCESS; } -static void doUpdateExprColumnIndex(STaskAttr* pQueryAttr) { - assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); - - for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { - SExprBasicInfo* pSqlExprMsg = &pQueryAttr->pExpr1[k].base; - // if (pSqlExprMsg->functionId == FUNCTION_ARITHM) { - // continue; - // } - - // todo opt performance - SColIndex* pColIndex = NULL; /*&pSqlExprMsg->colInfo;*/ - if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfCols; ++f) { - if (pColIndex->colId == pQueryAttr->tableCols[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfCols); - } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { - // do nothing for user-defined constant value result columns - } else { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfTags; ++f) { - if (pColIndex->colId == pQueryAttr->tagColList[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); - } - } -} - void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); @@ -9087,16 +8744,16 @@ void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const float THRESHOLD_RATIO = 0.85f; - if (isProjQuery(pQueryAttr)) { - int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; - if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { - numOfRes = MIN_ROWS_FOR_PRJ_QUERY; - } - - pResultInfo->capacity = numOfRes; - } else { // in case of non-prj query, a smaller output buffer will be used. - pResultInfo->capacity = DEFAULT_MIN_ROWS; - } +// if (isProjQuery(pQueryAttr)) { +// int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; +// if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { +// numOfRes = MIN_ROWS_FOR_PRJ_QUERY; +// } +// +// pResultInfo->capacity = numOfRes; +// } else { // in case of non-prj query, a smaller output buffer will be used. +// pResultInfo->capacity = DEFAULT_MIN_ROWS; +// } pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); pResultInfo->totalRows = 0; diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index af24906c9e..9174420ff4 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -176,26 +176,6 @@ typedef struct SResPair { double avg; } SResPair; -#define TSDB_BLOCK_DIST_STEP_ROWS 16 - -typedef struct STableBlockDist { - uint16_t rowSize; - uint16_t numOfFiles; - uint32_t numOfTables; - uint64_t totalSize; - uint64_t totalRows; - int32_t maxRows; - int32_t minRows; - int32_t firstSeekTimeUs; - uint32_t numOfRowsInMemTable; - uint32_t numOfSmallBlocks; - SArray *dataBlockInfos; -} STableBlockDist; - -typedef struct SFileBlockInfo { - int32_t numBlocksOfStep; -} SFileBlockInfo; - void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { pCell->initialized = false; } @@ -3984,7 +3964,7 @@ static void irate_function(SqlFunctionCtx *pCtx) { } } -static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { +static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDistInfo* pDist) { SBufferReader br = tbufInitReader(data, len, false); pDist->numOfTables = tbufReadUint32(&br); @@ -4024,7 +4004,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi static void blockInfo_func(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); @@ -4036,8 +4016,8 @@ static void blockInfo_func(SqlFunctionCtx* pCtx) { //pResInfo->hasResult = DATA_SET_FLAG; } -static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDist* pSrc) { - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); +static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDistInfo* pSrc) { + STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); assert(pDist != NULL && pSrc != NULL); pDist->numOfTables += pSrc->numOfTables; @@ -4071,7 +4051,7 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock } void block_func_merge(SqlFunctionCtx* pCtx) { - STableBlockDist info = {0}; + STableBlockDistInfo info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4082,7 +4062,7 @@ void block_func_merge(SqlFunctionCtx* pCtx) { //pResInfo->hasResult = DATA_SET_FLAG; } -void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, +void getPercentiles(STableBlockDistInfo *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, double* percents, int32_t* percentiles) { if (totalBlocks == 0) { for (int32_t i = 0; i < numOfPercents; ++i) { @@ -4117,7 +4097,7 @@ void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32 } } -void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { +void generateBlockDistResult(STableBlockDistInfo *pTableBlockDist, char* result) { if (pTableBlockDist == NULL) { return; } @@ -4178,7 +4158,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); pDist->rowSize = (uint16_t)pCtx->param[0].i; generateBlockDistResult(pDist, pCtx->pOutput); From 48312d911558b882c78e9fde03a7430a7e694efa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 15:23:08 +0800 Subject: [PATCH 3/8] handle except --- source/libs/index/src/indexFst.c | 3 +- source/libs/index/src/log | 1445 +++++++++++++++++++++++++++++ source/libs/index/test/fstTest.cc | 73 +- 3 files changed, 1514 insertions(+), 7 deletions(-) create mode 100644 source/libs/index/src/log diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 24bc7a93a2..a146564c9e 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -1251,7 +1251,6 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { taosArrayPush(sws->stack, &s); out += trn.out; node = fstGetNode(sws->fst, trn.addr); - fstNodeDestroy(node); } else { // This is a little tricky. We're in this case if the // given bound is not a prefix of any key in the FST. @@ -1349,7 +1348,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } - FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); + FstSlice slice = fstSliceCreate(buf, isz); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); diff --git a/source/libs/index/src/log b/source/libs/index/src/log new file mode 100644 index 0000000000..0b6cb8ddc2 --- /dev/null +++ b/source/libs/index/src/log @@ -0,0 +1,1445 @@ +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 1) /* +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 2) * Copyright (c) 2019 TAOS Data, Inc. +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 3) * +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 4) * This program is free software: you can use, redistribute, and/or modify +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 5) * it under the terms of the GNU Affero General Public License, version 3 +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 6) * or later ("AGPL"), as published by the Free Software Foundation. +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 7) * +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 8) * This program is distributed in the hope that it will be useful, but WITHOUT +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 9) * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 10) * FITNESS FOR A PARTICULAR PURPOSE. +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 11) * +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 12) * You should have received a copy of the GNU Affero General Public License +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 13) * along with this program. If not, see . +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 14) */ +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 15) +5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 16) #include "indexFst.h" +5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 17) #include "indexFstAutomation.h" +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 18) #include "indexInt.h" +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 19) #include "tchecksum.h" +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 20) #include "tcoding.h" +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 21) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 22) static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 23) CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 24) fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 25) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 26) static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { +20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 27) uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 28) fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 29) return nBytes; +20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 30) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 31) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 32) FstUnFinishedNodes* fstUnFinishedNodesCreate() { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 33) FstUnFinishedNodes* nodes = taosMemoryMalloc(sizeof(FstUnFinishedNodes)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 34) if (nodes == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 35) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 36) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 37) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 38) nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 39) fstUnFinishedNodesPushEmpty(nodes, false); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 40) return nodes; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 41) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 42) static void unFinishedNodeDestroyElem(void* elem) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 43) FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 44) fstBuilderNodeDestroy(b->node); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 45) taosMemoryFree(b->last); +7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 46) b->last = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 47) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 48) void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 49) if (nodes == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 50) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 51) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 52) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 53) taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 54) taosMemoryFree(nodes); +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 55) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 56) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 57) void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 58) FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 59) node->isFinal = isFinal; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 60) node->finalOutput = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 61) node->trans = taosArrayInit(16, sizeof(FstTransition)); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 62) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 63) FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 64) taosArrayPush(nodes->stack, &un); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 65) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 66) FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 67) assert(taosArrayGetSize(nodes->stack) == 1); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 68) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 69) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 70) assert(un->last == NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 71) return un->node; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 72) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 73) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 74) FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 75) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 76) fstBuilderNodeUnfinishedLastCompiled(un, addr); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 77) // taosMemoryFree(un->last); // TODO add func FstLastTransitionFree() +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 78) // un->last = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 79) return un->node; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 80) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 81) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 82) FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 83) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 84) assert(un->last == NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 85) return un->node; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 86) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 87) void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 88) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 89) un->node->isFinal = true; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 90) un->node->finalOutput = out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 91) // un->node->trans = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 92) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 93) void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 94) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 95) fstBuilderNodeUnfinishedLastCompiled(un, addr); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 96) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 97) void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 98) FstSlice* s = &bs; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 99) if (fstSliceIsEmpty(s)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 100) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 101) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 102) size_t sz = taosArrayGetSize(nodes->stack) - 1; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 103) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 104) assert(un->last == NULL); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 105) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 106) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 107) // trn->inp = s->data[s->start]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 108) // trn->out = out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 109) int32_t len = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 110) uint8_t* data = fstSliceData(s, &len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 111) un->last = fstLastTransitionCreate(data[0], out); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 112) +d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 113) for (uint64_t i = 1; i < len; i++) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 114) FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 115) n->isFinal = false; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 116) n->finalOutput = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 117) n->trans = taosArrayInit(16, sizeof(FstTransition)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 118) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 119) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 120) // trn->inp = s->data[i]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 121) // trn->out = out; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 122) FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 123) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 124) FstBuilderNodeUnfinished un = {.node = n, .last = trn}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 125) taosArrayPush(nodes->stack, &un); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 126) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 127) fstUnFinishedNodesPushEmpty(nodes, true); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 128) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 129) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 130) uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 131) FstSlice* s = &bs; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 132) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 133) size_t ssz = taosArrayGetSize(node->stack); // stack size +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 134) uint64_t count = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 135) int32_t lsz; // data len +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 136) uint8_t* data = fstSliceData(s, &lsz); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 137) for (size_t i = 0; i < ssz && i < lsz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 138) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 139) if (un->last->inp == data[i]) { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 140) count++; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 141) } else { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 142) break; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 143) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 144) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 145) return count; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 146) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 147) uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 148) FstSlice* s = &bs; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 149) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 150) size_t lsz = (size_t)(s->end - s->start + 1); // data len +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 151) size_t ssz = taosArrayGetSize(node->stack); // stack size +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 152) *out = in; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 153) uint64_t i = 0; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 154) for (i = 0; i < lsz && i < ssz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 155) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 156) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 157) FstLastTransition* t = un->last; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 158) uint64_t addPrefix = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 159) uint8_t* data = fstSliceData(s, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 160) if (t && t->inp == data[i]) { +92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 161) uint64_t commPrefix = TMIN(t->out, *out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 162) uint64_t tAddPrefix = t->out - commPrefix; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 163) (*out) = (*out) - commPrefix; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 164) t->out = commPrefix; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 165) addPrefix = tAddPrefix; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 166) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 167) break; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 168) } +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 169) if (addPrefix != 0) { +73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 170) if (i + 1 < ssz) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 171) FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 172) fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); +73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 173) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 174) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 175) } +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 176) return i; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 177) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 178) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 179) FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { +93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 180) FstState fs = {.state = EmptyFinal, .val = 0}; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 181) if (addr == EMPTY_ADDRESS) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 182) return fs; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 183) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 184) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 185) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 186) uint8_t v = data[addr]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 187) uint8_t t = (v & 0b11000000) >> 6; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 188) if (t == 0b11) { +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 189) fs.state = OneTransNext; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 190) } else if (t == 0b10) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 191) fs.state = OneTrans; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 192) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 193) fs.state = AnyTrans; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 194) } +93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 195) fs.val = v; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 196) return fs; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 197) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 198) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 199) static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 200) {.state = OneTrans, .val = 0b10000000}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 201) {.state = AnyTrans, .val = 0b00000000}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 202) {.state = EmptyFinal, .val = 0b00000000}}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 203) // debug +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 204) static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 205) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 206) FstState fstStateCreate(State state) { +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 207) uint8_t idx = (uint8_t)state; +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 208) return fstStateDict[idx]; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 209) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 210) // compile +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 211) void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 212) FstState s = fstStateCreate(OneTransNext); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 213) fstStateSetCommInput(&s, inp); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 214) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 215) bool null = false; +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 216) uint8_t v = fstStateCommInput(&s, &null); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 217) if (null) { +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 218) // w->write_all(&[inp]) +7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 219) fstCountingWriterWrite(w, &inp, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 220) } +7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 221) fstCountingWriterWrite(w, &(s.val), 1); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 222) // w->write_all(&[s.val]) +7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 223) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 224) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 225) void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 226) Output out = trn->out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 227) uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 228) uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 229) PackSizes packSizes = 0; +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 230) +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 231) FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 232) FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 233) fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 234) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 235) FstState st = fstStateCreate(OneTrans); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 236) +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 237) fstStateSetCommInput(&st, trn->inp); +22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 238) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 239) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 240) uint8_t inp = fstStateCommInput(&st, &null); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 241) if (null == true) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 242) fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 243) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 244) fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 245) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 246) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 247) void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 248) size_t sz = taosArrayGetSize(node->trans); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 249) assert(sz <= 256); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 250) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 251) uint8_t tSize = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 252) uint8_t oSize = packSize(node->finalOutput); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 253) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 254) // finalOutput.is_zero() +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 255) bool anyOuts = (node->finalOutput != 0); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 256) for (size_t i = 0; i < sz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 257) FstTransition* t = taosArrayGet(node->trans, i); +92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 258) tSize = TMAX(tSize, packDeltaSize(addr, t->addr)); +92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 259) oSize = TMAX(oSize, packSize(t->out)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 260) anyOuts = anyOuts || (t->out != 0); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 261) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 262) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 263) PackSizes packSizes = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 264) if (anyOuts) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 265) FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 266) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 267) FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 268) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 269) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 270) FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 271) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 272) FstState st = fstStateCreate(AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 273) fstStateSetFinalState(&st, node->isFinal); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 274) fstStateSetStateNtrans(&st, (uint8_t)sz); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 275) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 276) if (anyOuts) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 277) if (FST_BUILDER_NODE_IS_FINAL(node)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 278) fstCountingWriterPackUintIn(w, node->finalOutput, oSize); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 279) } +d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 280) for (int32_t i = sz - 1; i >= 0; i--) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 281) FstTransition* t = taosArrayGet(node->trans, i); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 282) fstCountingWriterPackUintIn(w, t->out, oSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 283) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 284) } +d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 285) for (int32_t i = sz - 1; i >= 0; i--) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 286) FstTransition* t = taosArrayGet(node->trans, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 287) fstPackDeltaIn(w, addr, t->addr, tSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 288) } +d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 289) for (int32_t i = sz - 1; i >= 0; i--) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 290) FstTransition* t = taosArrayGet(node->trans, i); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 291) fstCountingWriterWrite(w, (char*)&t->inp, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 292) // fstPackDeltaIn(w, addr, t->addr, tSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 293) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 294) if (sz > TRANS_INDEX_THRESHOLD) { +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 295) // A value of 255 indicates that no transition exists for the byte +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 296) // at that index. (Except when there are 256 transitions.) Namely, +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 297) // any value greater than or equal to the number of transitions in +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 298) // this node indicates an absent transition. +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 299) uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); +984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 300) memset(index, 255, sizeof(uint8_t) * 256); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 301) /// for (uint8_t i = 0; i < 256; i++) { +984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 302) // index[i] = 255; +984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 303) ///} +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 304) for (size_t i = 0; i < sz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 305) FstTransition* t = taosArrayGet(node->trans, i); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 306) index[t->inp] = i; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 307) // fstPackDeltaIn(w, addr, t->addr, tSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 308) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 309) fstCountingWriterWrite(w, (char*)index, 256); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 310) taosMemoryFree(index); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 311) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 312) fstCountingWriterWrite(w, (char*)&packSizes, 1); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 313) bool null = false; +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 314) fstStateStateNtrans(&st, &null); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 315) if (null == true) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 316) // 256 can't be represented in a u8, so we abuse the fact that +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 317) // the # of transitions can never be 1 here, since 1 is always +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 318) // encoded in the state byte. +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 319) uint8_t v = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 320) if (sz == 256) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 321) fstCountingWriterWrite(w, (char*)&v, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 322) } else { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 323) fstCountingWriterWrite(w, (char*)&sz, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 324) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 325) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 326) fstCountingWriterWrite(w, (char*)(&(st.val)), 1); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 327) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 328) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 329) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 330) // set_comm_input +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 331) void fstStateSetCommInput(FstState* s, uint8_t inp) { +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 332) assert(s->state == OneTransNext || s->state == OneTrans); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 333) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 334) uint8_t val; +9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 335) COMMON_INDEX(inp, 0b111111, val); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 336) s->val = (s->val & fstStateDict[s->state].val) | val; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 337) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 338) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 339) // comm_input +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 340) uint8_t fstStateCommInput(FstState* s, bool* null) { +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 341) assert(s->state == OneTransNext || s->state == OneTrans); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 342) uint8_t v = s->val & 0b00111111; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 343) if (v == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 344) *null = true; +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 345) return v; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 346) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 347) // v = 0 indicate that common_input is None +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 348) return v == 0 ? 0 : COMMON_INPUT(v); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 349) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 350) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 351) // input_len +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 352) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 353) uint64_t fstStateInputLen(FstState* s) { +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 354) assert(s->state == OneTransNext || s->state == OneTrans); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 355) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 356) fstStateCommInput(s, &null); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 357) return null ? 1 : 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 358) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 359) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 360) // end_addr +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 361) uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { +4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 362) assert(s->state == OneTransNext); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 363) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 364) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 365) uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { +4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 366) assert(s->state == OneTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 367) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 368) - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 369) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 370) uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 371) uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 372) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 373) uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 374) return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 375) - fstStateTotalTransSize(state, version, sizes, nTrans) - nTrans * oSizes // output values +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 376) - finalOsize; // final output +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 377) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 378) // input +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 379) uint8_t fstStateInput(FstState* s, FstNode* node) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 380) assert(s->state == OneTransNext || s->state == OneTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 381) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 382) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 383) uint8_t inp = fstStateCommInput(s, &null); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 384) uint8_t* data = fstSliceData(slice, NULL); +9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 385) return null == false ? inp : data[node->start - 1]; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 386) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 387) uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 388) assert(s->state == AnyTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 389) FstSlice* slice = &node->data; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 390) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 391) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 392) - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 393) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 394) uint8_t* data = fstSliceData(slice, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 395) return data[at]; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 396) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 397) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 398) // trans_addr +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 399) CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 400) assert(s->state == OneTransNext || s->state == OneTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 401) FstSlice* slice = &node->data; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 402) if (s->state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 403) return (CompiledAddr)(node->end) - 1; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 404) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 405) PackSizes sizes = node->sizes; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 406) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 407) uint64_t i = node->start - fstStateInputLen(s) - 1 // PackSizes +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 408) - tSizes; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 409) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 410) // refactor error logic +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 411) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 412) return unpackDelta(data + i, tSizes, node->end); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 413) } +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 414) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 415) CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 416) assert(s->state == AnyTrans); +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 417) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 418) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 419) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 420) uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 421) node->nTrans - (i * tSizes) - tSizes; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 422) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 423) return unpackDelta(data + at, tSizes, node->end); +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 424) } +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 425) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 426) // sizes +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 427) PackSizes fstStateSizes(FstState* s, FstSlice* slice) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 428) assert(s->state == OneTrans || s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 429) uint64_t i; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 430) if (s->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 431) i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 432) } else { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 433) i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 434) } +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 435) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 436) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 437) return (PackSizes)(*(data + i)); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 438) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 439) // Output +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 440) Output fstStateOutput(FstState* s, FstNode* node) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 441) assert(s->state == OneTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 442) +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 443) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 444) if (oSizes == 0) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 445) return 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 446) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 447) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 448) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 449) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 450) uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 451) uint8_t* data = fstSliceData(slice, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 452) return unpackUint64(data + i, oSizes); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 453) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 454) Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 455) assert(s->state == AnyTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 456) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 457) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 458) if (oSizes == 0) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 459) return 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 460) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 461) FstSlice* slice = &node->data; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 462) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 463) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 464) - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 465) +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 466) return unpackUint64(data + at, oSizes); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 467) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 468) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 469) // anyTrans specify function +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 470) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 471) void fstStateSetFinalState(FstState* s, bool yes) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 472) assert(s->state == AnyTrans); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 473) if (yes) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 474) s->val |= 0b01000000; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 475) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 476) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 477) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 478) bool fstStateIsFinalState(FstState* s) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 479) assert(s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 480) return (s->val & 0b01000000) == 0b01000000; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 481) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 482) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 483) void fstStateSetStateNtrans(FstState* s, uint8_t n) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 484) assert(s->state == AnyTrans); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 485) if (n <= 0b00111111) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 486) s->val = (s->val & 0b11000000) | n; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 487) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 488) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 489) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 490) // state_ntrans +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 491) uint8_t fstStateStateNtrans(FstState* s, bool* null) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 492) assert(s->state == AnyTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 493) *null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 494) uint8_t n = s->val & 0b00111111; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 495) +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 496) if (n == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 497) *null = true; // None +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 498) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 499) return n; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 500) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 501) uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 502) assert(s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 503) uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 504) return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 505) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 506) uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 507) assert(s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 508) return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 509) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 510) uint64_t fstStateNtransLen(FstState* s) { +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 511) assert(s->state == AnyTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 512) bool null = false; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 513) fstStateStateNtrans(s, &null); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 514) return null == true ? 1 : 0; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 515) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 516) uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 517) bool null = false; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 518) uint8_t n = fstStateStateNtrans(s, &null); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 519) if (null != true) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 520) return n; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 521) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 522) int32_t len; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 523) uint8_t* data = fstSliceData(slice, &len); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 524) n = data[len - 2]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 525) // n = data[slice->end - 1]; // data[data.len() - 2] +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 526) return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 527) // then it is encoded in the state byte +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 528) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 529) Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 530) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 531) if (oSizes == 0 || !fstStateIsFinalState(s)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 532) return 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 533) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 534) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 535) uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 536) - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 537) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 538) return unpackUint64(data + at, (uint8_t)oSizes); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 539) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 540) uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 541) assert(s->state == AnyTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 542) FstSlice* slice = &node->data; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 543) if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 544) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 545) - fstStateTransIndexSize(s, node->version, node->nTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 546) int32_t dlen = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 547) uint8_t* data = fstSliceData(slice, &dlen); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 548) uint64_t i = data[at + b]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 549) // uint64_t i = slice->data[slice->start + at + b]; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 550) if (i >= node->nTrans) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 551) *null = true; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 552) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 553) return i; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 554) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 555) uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 556) - node->nTrans; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 557) uint64_t end = start + node->nTrans; +73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 558) FstSlice t = fstSliceCopy(slice, start, end - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 559) int32_t len = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 560) uint8_t* data = fstSliceData(&t, &len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 561) int i = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 562) for (; i < len; i++) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 563) uint8_t v = data[i]; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 564) if (v == b) { +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 565) fstSliceDestroy(&t); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 566) return node->nTrans - i - 1; // bug +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 567) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 568) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 569) if (i == len) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 570) *null = true; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 571) } +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 572) fstSliceDestroy(&t); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 573) } +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 574) +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 575) return 0; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 576) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 577) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 578) // fst node function +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 579) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 580) FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 581) FstNode* n = (FstNode*)taosMemoryMalloc(sizeof(FstNode)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 582) if (n == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 583) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 584) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 585) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 586) FstState st = fstStateCreateFrom(slice, addr); +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 587) +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 588) if (st.state == EmptyFinal) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 589) n->data = fstSliceCreate(NULL, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 590) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 591) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 592) n->start = EMPTY_ADDRESS; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 593) n->end = EMPTY_ADDRESS; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 594) n->isFinal = true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 595) n->nTrans = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 596) n->sizes = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 597) n->finalOutput = 0; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 598) } else if (st.state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 599) n->data = fstSliceCopy(slice, 0, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 600) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 601) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 602) n->start = addr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 603) n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 604) n->isFinal = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 605) n->sizes = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 606) n->nTrans = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 607) n->finalOutput = 0; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 608) } else if (st.state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 609) FstSlice data = fstSliceCopy(slice, 0, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 610) PackSizes sz = fstStateSizes(&st, &data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 611) n->data = data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 612) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 613) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 614) n->start = addr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 615) n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 616) n->isFinal = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 617) n->nTrans = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 618) n->sizes = sz; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 619) n->finalOutput = 0; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 620) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 621) FstSlice data = fstSliceCopy(slice, 0, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 622) uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 623) uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 624) n->data = data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 625) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 626) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 627) n->start = addr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 628) n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 629) n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 630) n->nTrans = nTrans; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 631) n->sizes = sz; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 632) n->finalOutput = +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 633) fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 634) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 635) return n; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 636) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 637) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 638) // debug state transition +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 639) static const char* fstNodeState(FstNode* node) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 640) FstState* st = &node->state; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 641) return fstStateStr[st->state]; +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 642) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 643) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 644) void fstNodeDestroy(FstNode* node) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 645) fstSliceDestroy(&node->data); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 646) taosMemoryFree(node); +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 647) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 648) FstTransitions* fstNodeTransitions(FstNode* node) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 649) FstTransitions* t = taosMemoryMalloc(sizeof(FstTransitions)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 650) if (NULL == t) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 651) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 652) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 653) FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 654) t->range = range; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 655) t->node = node; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 656) return t; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 657) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 658) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 659) // Returns the transition at index `i`. +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 660) bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 661) bool s = true; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 662) FstState* st = &node->state; +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 663) if (st->state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 664) trn->inp = fstStateInput(st, node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 665) trn->out = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 666) trn->addr = fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 667) } else if (st->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 668) trn->inp = fstStateInput(st, node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 669) trn->out = fstStateOutput(st, node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 670) trn->addr = fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 671) } else if (st->state == AnyTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 672) trn->inp = fstStateInputForAnyTrans(st, node, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 673) trn->out = fstStateOutputForAnyTrans(st, node, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 674) trn->addr = fstStateTransAddrForAnyTrans(st, node, i); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 675) } else { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 676) s = false; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 677) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 678) return s; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 679) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 680) +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 681) // Returns the transition address of the `i`th transition +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 682) bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 683) bool s = true; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 684) FstState* st = &node->state; +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 685) if (st->state == OneTransNext) { +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 686) assert(i == 0); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 687) fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 688) } else if (st->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 689) assert(i == 0); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 690) fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 691) } else if (st->state == AnyTrans) { +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 692) fstStateTransAddrForAnyTrans(st, node, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 693) } else if (FST_STATE_EMPTY_FINAL(node)) { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 694) s = false; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 695) } else { +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 696) assert(0); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 697) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 698) return s; +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 699) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 700) +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 701) // Finds the `i`th transition corresponding to the given input byte. +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 702) // If no transition for this byte exists, then `false` is returned. +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 703) bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 704) bool s = true; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 705) FstState* st = &node->state; +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 706) if (st->state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 707) if (fstStateInput(st, node) == b) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 708) *res = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 709) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 710) s = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 711) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 712) } else if (st->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 713) if (fstStateInput(st, node) == b) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 714) *res = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 715) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 716) s = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 717) } +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 718) } else if (st->state == AnyTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 719) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 720) uint64_t out = fstStateFindInput(st, node, b, &null); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 721) if (null == false) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 722) *res = out; +2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 723) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 724) s = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 725) } +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 726) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 727) return s; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 728) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 729) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 730) bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 731) size_t sz = taosArrayGetSize(builderNode->trans); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 732) assert(sz < 256); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 733) if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 734) return true; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 735) } else if (sz != 1 || builderNode->isFinal) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 736) fstStateCompileForAnyTrans(w, addr, builderNode); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 737) // AnyTrans->Compile(w, addr, node); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 738) } else { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 739) FstTransition* tran = taosArrayGet(builderNode->trans, 0); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 740) if (tran->addr == lastAddr && tran->out == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 741) fstStateCompileForOneTransNext(w, addr, tran->inp); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 742) // OneTransNext::compile(w, lastAddr, tran->inp); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 743) return true; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 744) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 745) fstStateCompileForOneTrans(w, addr, tran); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 746) // OneTrans::Compile(w, lastAddr, *tran); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 747) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 748) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 749) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 750) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 751) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 752) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 753) bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 754) return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 755) } +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 756) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 757) FstBuilder* fstBuilderCreate(void* w, FstType ty) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 758) FstBuilder* b = taosMemoryMalloc(sizeof(FstBuilder)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 759) if (NULL == b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 760) return b; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 761) } +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 762) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 763) b->wrt = fstCountingWriterCreate(w); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 764) b->unfinished = fstUnFinishedNodesCreate(); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 765) b->registry = fstRegistryCreate(10000, 2); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 766) b->last = fstSliceCreate(NULL, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 767) b->lastAddr = NONE_ADDRESS; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 768) b->len = 0; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 769) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 770) char buf64[8] = {0}; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 771) void* pBuf64 = buf64; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 772) taosEncodeFixedU64(&pBuf64, VERSION); +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 773) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 774) +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 775) pBuf64 = buf64; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 776) memset(buf64, 0, sizeof(buf64)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 777) taosEncodeFixedU64(&pBuf64, ty); +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 778) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 779) +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 780) return b; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 781) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 782) void fstBuilderDestroy(FstBuilder* b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 783) if (b == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 784) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 785) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 786) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 787) fstCountingWriterDestroy(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 788) fstUnFinishedNodesDestroy(b->unfinished); +260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 789) fstRegistryDestroy(b->registry); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 790) fstSliceDestroy(&b->last); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 791) taosMemoryFree(b); +260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 792) } +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 793) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 794) bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 795) OrderType t = fstBuilderCheckLastKey(b, bs, true); +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 796) if (t == Ordered) { +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 797) // add log info +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 798) fstBuilderInsertOutput(b, bs, in); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 799) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 800) } +ebcb9be39fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-22 12:20:59 +0800 801) indexInfo("fst write key must be ordered"); +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 802) return false; +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 803) } +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 804) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 805) void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 806) FstSlice* s = &bs; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 807) if (fstSliceIsEmpty(s)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 808) b->len = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 809) fstUnFinishedNodesSetRootOutput(b->unfinished, in); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 810) return; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 811) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 812) // if (in != 0) { //if let Some(in) = in +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 813) // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 814) //} else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 815) // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 816) // out = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 817) //} +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 818) Output out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 819) uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 820) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 821) if (prefixLen == FST_SLICE_LEN(s)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 822) assert(out == 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 823) return; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 824) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 825) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 826) b->len += 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 827) fstBuilderCompileFrom(b, prefixLen); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 828) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 829) FstSlice sub = fstSliceCopy(s, prefixLen, s->end); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 830) fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 831) fstSliceDestroy(&sub); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 832) return; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 833) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 834) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 835) OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 836) FstSlice* input = &bs; +716c0045f8a source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 17:29:59 +0800 837) if (fstSliceIsEmpty(&b->last)) { +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 838) fstSliceDestroy(&b->last); +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 839) // deep copy or not +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 840) b->last = fstSliceDeepCopy(&bs, input->start, input->end); +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 841) } else { +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 842) int comp = fstSliceCompare(&b->last, &bs); +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 843) if (comp == 0 && ckDup) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 844) return DuplicateKey; +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 845) } else if (comp == 1) { +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 846) return OutOfOrdered; +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 847) } +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 848) // deep copy or not +d8d6c04fd50 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 16:18:35 +0800 849) fstSliceDestroy(&b->last); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 850) b->last = fstSliceDeepCopy(&bs, input->start, input->end); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 851) } +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 852) return Ordered; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 853) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 854) void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 855) CompiledAddr addr = NONE_ADDRESS; +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 856) while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 857) FstBuilderNode* bn = NULL; +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 858) if (addr == NONE_ADDRESS) { +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 859) bn = fstUnFinishedNodesPopEmpty(b->unfinished); +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 860) } else { +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 861) bn = fstUnFinishedNodesPopFreeze(b->unfinished, addr); +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 862) } +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 863) addr = fstBuilderCompile(b, bn); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 864) +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 865) fstBuilderNodeDestroy(bn); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 866) assert(addr != NONE_ADDRESS); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 867) // fstBuilderNodeDestroy(n); +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 868) } +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 869) fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 870) return; +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 871) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 872) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 873) CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 874) if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 875) return EMPTY_ADDRESS; +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 876) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 877) FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 878) if (entry->state == FOUND) { +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 879) CompiledAddr ret = entry->addr; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 880) fstRegistryEntryDestroy(entry); +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 881) return ret; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 882) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 883) CompiledAddr startAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt)); +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 884) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 885) fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 886) b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 887) if (entry->state == NOTFOUND) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 888) FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 889) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 890) fstRegistryEntryDestroy(entry); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 891) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 892) return b->lastAddr; +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 893) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 894) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 895) void* fstBuilderInsertInner(FstBuilder* b) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 896) fstBuilderCompileFrom(b, 0); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 897) FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 898) CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 899) fstBuilderNodeDestroy(rootNode); +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 900) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 901) char buf64[8] = {0}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 902) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 903) void* pBuf64 = buf64; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 904) taosEncodeFixedU64(&pBuf64, b->len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 905) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 906) +d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 907) pBuf64 = buf64; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 908) taosEncodeFixedU64(&pBuf64, rootAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 909) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 910) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 911) char buf32[4] = {0}; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 912) void* pBuf32 = buf32; +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 913) uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 914) taosEncodeFixedU32(&pBuf32, sum); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 915) fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 916) +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 917) fstCountingWriterFlush(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 918) // fstCountingWriterDestroy(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 919) // b->wrt = NULL; +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 920) return b->wrt; +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 921) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 922) void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); } +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 923) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 924) FstSlice fstNodeAsSlice(FstNode* node) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 925) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 926) FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 927) return s; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 928) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 929) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 930) FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 931) FstLastTransition* trn = taosMemoryMalloc(sizeof(FstLastTransition)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 932) if (trn == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 933) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 934) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 935) +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 936) trn->inp = inp; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 937) trn->out = out; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 938) return trn; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 939) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 940) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 941) void fstLastTransitionDestroy(FstLastTransition* trn) { taosMemoryFree(trn); } +22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 942) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 943) void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 944) FstLastTransition* trn = unNode->last; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 945) if (trn == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 946) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 947) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 948) FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 949) taosArrayPush(unNode->node->trans, &t); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 950) fstLastTransitionDestroy(trn); +7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 951) unNode->last = NULL; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 952) return; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 953) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 954) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 955) void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 956) if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 957) unNode->node->finalOutput += out; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 958) } +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 959) size_t sz = taosArrayGetSize(unNode->node->trans); +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 960) for (size_t i = 0; i < sz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 961) FstTransition* trn = taosArrayGet(unNode->node->trans, i); +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 962) trn->out += out; +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 963) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 964) if (unNode->last) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 965) unNode->last->out += out; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 966) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 967) return; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 968) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 969) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 970) Fst* fstCreate(FstSlice* slice) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 971) int32_t slen; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 972) char* buf = fstSliceData(slice, &slen); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 973) if (slen < 36) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 974) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 975) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 976) uint64_t len = slen; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 977) uint64_t skip = 0; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 978) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 979) uint64_t version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 980) taosDecodeFixedU64(buf, &version); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 981) skip += sizeof(version); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 982) if (version == 0 || version > VERSION) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 983) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 984) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 985) +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 986) uint64_t type; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 987) taosDecodeFixedU64(buf + skip, &type); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 988) skip += sizeof(type); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 989) +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 990) uint32_t checkSum = 0; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 991) len -= sizeof(checkSum); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 992) taosDecodeFixedU32(buf + len, &checkSum); +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 993) if (taosCheckChecksum(buf, len, checkSum)) { +f9d83c044dd source/libs/index/src/index_fst.c (yihaoDeng 2022-01-10 23:42:58 +0800 994) indexError("index file is corrupted"); +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 995) // verify fst +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 996) return NULL; +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 997) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 998) CompiledAddr rootAddr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 999) len -= sizeof(rootAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1000) taosDecodeFixedU64(buf + len, &rootAddr); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1001) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1002) uint64_t fstLen; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1003) len -= sizeof(fstLen); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1004) taosDecodeFixedU64(buf + len, &fstLen); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1005) // TODO(validate root addr) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1006) Fst* fst = (Fst*)taosMemoryCalloc(1, sizeof(Fst)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1007) if (fst == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1008) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1009) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1010) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1011) fst->meta = (FstMeta*)taosMemoryMalloc(sizeof(FstMeta)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1012) if (NULL == fst->meta) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1013) goto FST_CREAT_FAILED; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1014) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1015) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1016) fst->meta->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1017) fst->meta->rootAddr = rootAddr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1018) fst->meta->ty = type; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1019) fst->meta->len = fstLen; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1020) fst->meta->checkSum = checkSum; +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1021) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1022) FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1023) *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1024) fst->data = s; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1025) +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1026) taosThreadMutexInit(&fst->mtx, NULL); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1027) return fst; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1028) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1029) FST_CREAT_FAILED: +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1030) taosMemoryFree(fst->meta); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1031) taosMemoryFree(fst); +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1032) +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1033) return NULL; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1034) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1035) void fstDestroy(Fst* fst) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1036) if (fst) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1037) taosMemoryFree(fst->meta); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1038) fstSliceDestroy(fst->data); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1039) taosMemoryFree(fst->data); +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1040) taosThreadMutexDestroy(&fst->mtx); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1041) } +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1042) taosMemoryFree(fst); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1043) } +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1044) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1045) bool fstGet(Fst* fst, FstSlice* b, Output* out) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1046) // dec lock range +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1047) // taosThreadMutexLock(&fst->mtx); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1048) FstNode* root = fstGetRoot(fst); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1049) Output tOut = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1050) int32_t len; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1051) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1052) uint8_t* data = fstSliceData(b, &len); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1053) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1054) SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1055) taosArrayPush(nodes, &root); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1056) for (uint32_t i = 0; i < len; i++) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1057) uint8_t inp = data[i]; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1058) Output res = 0; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1059) if (false == fstNodeFindInput(root, inp, &res)) { +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1060) // taosThreadMutexUnlock(&fst->mtx); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1061) return false; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1062) } +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1063) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1064) FstTransition trn; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1065) fstNodeGetTransitionAt(root, res, &trn); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1066) tOut += trn.out; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1067) root = fstGetNode(fst, trn.addr); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1068) taosArrayPush(nodes, &root); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1069) } +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1070) if (!FST_NODE_IS_FINAL(root)) { +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1071) // taosThreadMutexUnlock(&fst->mtx); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1072) return false; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1073) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1074) tOut = tOut + FST_NODE_FINAL_OUTPUT(root); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1075) } +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 1076) +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1077) for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1078) FstNode** node = (FstNode**)taosArrayGet(nodes, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1079) fstNodeDestroy(*node); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1080) } +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1081) taosArrayDestroy(nodes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1082) // fst->root = NULL; +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1083) // taosThreadMutexUnlock(&fst->mtx); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1084) *out = tOut; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1085) return true; +f07045c1d8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 17:28:58 +0800 1086) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1087) FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1088) // refactor later +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1089) return fstStreamBuilderCreate(fst, ctx); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1090) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1091) StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1092) if (sb == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1093) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1094) } +26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1095) return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); +26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1096) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1097) FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1098) // refactor later +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1099) return fstStreamBuilderCreate(fst, ctx); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1100) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1101) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1102) FstNode* fstGetRoot(Fst* fst) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1103) CompiledAddr rAddr = fstGetRootAddr(fst); +2eb0053e5ad source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 22:19:56 +0800 1104) return fstGetNode(fst, rAddr); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1105) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1106) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1107) FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1108) // refactor later +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1109) return fstNodeCreate(fst->meta->version, addr, fst->data); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1110) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1111) FstType fstGetType(Fst* fst) { return fst->meta->ty; } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1112) CompiledAddr fstGetRootAddr(Fst* fst) { return fst->meta->rootAddr; } +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1113) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1114) Output fstEmptyFinalOutput(Fst* fst, bool* null) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1115) Output res = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1116) FstNode* node = fstGetRoot(fst); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1117) if (FST_NODE_IS_FINAL(node)) { +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1118) *null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1119) res = FST_NODE_FINAL_OUTPUT(node); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1120) } else { +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1121) *null = true; +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1122) } +40fc4b6cf38 source/libs/index/src/index_fst.c (yihaoDeng 2022-01-02 17:50:40 +0800 1123) fstNodeDestroy(node); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1124) return res; +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1125) } +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1126) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1127) bool fstVerify(Fst* fst) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1128) uint32_t len, checkSum = fst->meta->checkSum; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1129) uint8_t* data = fstSliceData(fst->data, &len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1130) TSCKSUM initSum = 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1131) if (!taosCheckChecksumWhole(data, len)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1132) return false; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1133) } +a09dcbdd85f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:59:51 +0800 1134) return true; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1135) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1136) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1137) // data bound function +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1138) FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1139) FstBoundWithData* b = taosMemoryCalloc(1, sizeof(FstBoundWithData)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1140) if (b == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1141) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1142) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1143) +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1144) if (data != NULL) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1145) b->data = fstSliceCopy(data, data->start, data->end); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1146) } else { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1147) b->data = fstSliceCreate(NULL, 0); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1148) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1149) b->type = type; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1150) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1151) return b; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1152) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1153) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1154) bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1155) int comp = fstSliceCompare(slice, &bound->data); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1156) if (bound->type == Included) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1157) return comp > 0 ? true : false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1158) } else if (bound->type == Excluded) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1159) return comp >= 0 ? true : false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1160) } else { +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1161) return false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1162) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1163) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1164) bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1165) if (bound->type == Unbounded) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1166) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1167) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1168) return fstSliceIsEmpty(&bound->data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1169) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1170) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1171) +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1172) bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1173) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1174) void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1175) +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1176) StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1177) FstBoundWithData* max) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1178) StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1179) if (sws == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1180) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1181) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1182) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1183) sws->fst = fst; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1184) sws->aut = automation; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1185) sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1186) +0c5f2d1da95 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 21:41:41 +0800 1187) sws->emptyOutput.null = true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1188) sws->emptyOutput.out = 0; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1189) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1190) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1191) sws->endAt = max; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1192) streamWithStateSeekMin(sws, min); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1193) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1194) return sws; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1195) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1196) void streamWithStateDestroy(StreamWithState* sws) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1197) if (sws == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1198) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1199) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1200) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1201) taosArrayDestroy(sws->inp); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1202) taosArrayDestroyEx(sws->stack, streamStateDestroy); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1203) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1204) taosMemoryFree(sws); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1205) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1206) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1207) bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1208) AutomationCtx* aut = sws->aut; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1209) if (fstBoundWithDataIsEmpty(min)) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1210) if (fstBoundWithDataIsIncluded(min)) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1211) sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1212) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1213) StreamState s = {.node = fstGetRoot(sws->fst), +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1214) .trans = 0, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1215) .out = {.null = false, .out = 0}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1216) .autState = automFuncs[aut->type].start(aut)}; // auto.start callback +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1217) taosArrayPush(sws->stack, &s); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1218) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1219) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1220) FstSlice* key = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1221) bool inclusize = false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1222) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1223) if (min->type == Included) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1224) key = &min->data; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1225) inclusize = true; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1226) } else if (min->type == Excluded) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1227) key = &min->data; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1228) } else { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1229) return false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1230) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1231) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1232) FstNode* node = fstGetRoot(sws->fst); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1233) Output out = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1234) // void* autState = sws->aut->start(); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1235) void* autState = automFuncs[aut->type].start(aut); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1236) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1237) int32_t len; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1238) uint8_t* data = fstSliceData(key, &len); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1239) for (uint32_t i = 0; i < len; i++) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1240) uint8_t b = data[i]; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1241) uint64_t res = 0; +2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1242) bool find = fstNodeFindInput(node, b, &res); +2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1243) if (find == true) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1244) FstTransition trn; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1245) fstNodeGetTransitionAt(node, res, &trn); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1246) void* preState = autState; +cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1247) // autState = sws->aut->accept(preState, b); +cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1248) autState = automFuncs[aut->type].accept(aut, preState, b); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1249) taosArrayPush(sws->inp, &b); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1250) StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1251) taosArrayPush(sws->stack, &s); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1252) out += trn.out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1253) node = fstGetNode(sws->fst, trn.addr); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1254) } else { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1255) // This is a little tricky. We're in this case if the +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1256) // given bound is not a prefix of any key in the FST. +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1257) // Since this is a minimum bound, we need to find the +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1258) // first transition in this node that proceeds the current +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1259) // input byte. +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1260) FstTransitions* trans = fstNodeTransitions(node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1261) uint64_t i = 0; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1262) for (i = trans->range.start; i < trans->range.end; i++) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1263) FstTransition trn; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1264) if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1265) break; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1266) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1267) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1268) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1269) StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1270) taosArrayPush(sws->stack, &s); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1271) return true; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1272) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1273) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1274) uint32_t sz = taosArrayGetSize(sws->stack); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1275) if (sz != 0) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1276) StreamState* s = taosArrayGet(sws->stack, sz - 1); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1277) if (inclusize) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1278) s->trans -= 1; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1279) taosArrayPop(sws->inp); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1280) } else { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1281) FstNode* n = s->node; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1282) uint64_t trans = s->trans; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1283) FstTransition trn; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1284) fstNodeGetTransitionAt(n, trans - 1, &trn); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1285) StreamState s = { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1286) .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1287) taosArrayPush(sws->stack, &s); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1288) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1289) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1290) return false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1291) } +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1292) +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1293) return false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1294) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1295) StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1296) AutomationCtx* aut = sws->aut; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1297) FstOutput output = sws->emptyOutput; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1298) if (output.null == false) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1299) FstSlice emptySlice = fstSliceCreate(NULL, 0); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1300) if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1301) taosArrayDestroyEx(sws->stack, streamStateDestroy); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1302) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1303) return NULL; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1304) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1305) void* start = automFuncs[aut->type].start(aut); +cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1306) if (automFuncs[aut->type].isMatch(aut, start)) { +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1307) FstSlice s = fstSliceCreate(NULL, 0); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1308) return swsResultCreate(&s, output, callback == NULL ? NULL : callback(start)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1309) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1310) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1311) SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1312) while (taosArrayGetSize(sws->stack) > 0) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1313) StreamState* p = (StreamState*)taosArrayPop(sws->stack); +f0110af30ec source/libs/index/src/index_fst.c (yihaoDeng 2021-12-16 22:04:47 +0800 1314) if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1315) if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1316) taosArrayPop(sws->inp); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1317) } +6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1318) streamStateDestroy(p); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1319) continue; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1320) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1321) FstTransition trn; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1322) fstNodeGetTransitionAt(p->node, p->trans, &trn); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1323) +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1324) Output out = p->out.out + trn.out; +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1325) void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1326) void* tState = (callback == NULL) ? NULL : callback(nextState); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1327) bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1328) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1329) FstNode* nextNode = fstGetNode(sws->fst, trn.addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1330) taosArrayPush(nodes, &nextNode); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1331) taosArrayPush(sws->inp, &(trn.inp)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1332) +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1333) if (FST_NODE_IS_FINAL(nextNode)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1334) // void *eofState = sws->aut->acceptEof(nextState); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1335) void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1336) if (eofState != NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1337) isMatch = automFuncs[aut->type].isMatch(aut, eofState); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1338) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1339) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1340) StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1341) taosArrayPush(sws->stack, &s1); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1342) +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1343) StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1344) taosArrayPush(sws->stack, &s2); +6f3c49ee73d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-14 16:17:38 +0800 1345) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1346) size_t isz = taosArrayGetSize(sws->inp); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1347) uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1348) for (uint32_t i = 0; i < isz; i++) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1349) buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1350) } +00000000000 source/libs/index/src/indexFst.c (Not Committed Yet 2022-04-02 15:19:56 +0800 1351) FstSlice slice = fstSliceCreate(buf, isz); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1352) if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1353) taosArrayDestroyEx(sws->stack, streamStateDestroy); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1354) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1355) taosMemoryFreeClear(buf); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1356) fstSliceDestroy(&slice); +06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1357) taosArrayDestroy(nodes); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1358) return NULL; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1359) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1360) if (FST_NODE_IS_FINAL(nextNode) && isMatch) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1361) FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1362) StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1363) taosMemoryFreeClear(buf); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1364) fstSliceDestroy(&slice); +06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1365) taosArrayDestroy(nodes); +6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1366) nodes = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1367) return result; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1368) } +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1369) taosMemoryFreeClear(buf); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1370) fstSliceDestroy(&slice); +6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1371) }; +06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1372) taosArrayDestroy(nodes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1373) return NULL; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1374) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1375) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1376) StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1377) StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1378) if (result == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1379) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1380) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1381) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1382) result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1383) result->out = fOut; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1384) result->state = state; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1385) return result; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1386) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1387) void swsResultDestroy(StreamWithStateResult* result) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1388) if (NULL == result) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1389) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1390) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1391) +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1392) fstSliceDestroy(&result->data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1393) startWithStateValueDestroy(result->state); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1394) taosMemoryFree(result); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1395) } +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1396) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1397) void streamStateDestroy(void* s) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1398) if (NULL == s) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1399) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1400) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1401) StreamState* ss = (StreamState*)s; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1402) fstNodeDestroy(ss->node); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1403) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1404) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1405) FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1406) FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1407) if (NULL == b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1408) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1409) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1410) +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1411) b->fst = fst; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1412) b->aut = aut; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1413) b->min = fstBoundStateCreate(Unbounded, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1414) b->max = fstBoundStateCreate(Unbounded, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1415) return b; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1416) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1417) void fstStreamBuilderDestroy(FstStreamBuilder* b) { +7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1418) fstSliceDestroy(&b->min->data); +7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1419) fstSliceDestroy(&b->max->data); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1420) taosMemoryFreeClear(b->min); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1421) taosMemoryFreeClear(b->max); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1422) taosMemoryFree(b); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1423) } +30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1424) void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1425) if (b == NULL) { +30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1426) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1427) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1428) if (type == GE) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1429) b->min->type = Included; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1430) fstSliceDestroy(&(b->min->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1431) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1432) } else if (type == GT) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1433) b->min->type = Excluded; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1434) fstSliceDestroy(&(b->min->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1435) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1436) } else if (type == LE) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1437) b->max->type = Included; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1438) fstSliceDestroy(&(b->max->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1439) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1440) } else if (type == LT) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1441) b->max->type = Excluded; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1442) fstSliceDestroy(&(b->max->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1443) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1444) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1445) } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index eff53108cd..c45f655746 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -510,6 +510,68 @@ void checkFstCheckIteratorRange2() { } delete m; } +void checkFstCheckIteratorRange3() { + FstWriter* fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int count = 2; + // Performance_fstWriteRecords(fw); + int64_t e = taosGetTimestampUs(); + + std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + + fw->Put("ab", 1); + fw->Put("b", 2); + fw->Put("cdd", 3); + fw->Put("cde", 3); + fw->Put("ddd", 4); + fw->Put("ed", 5); + delete fw; + + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "b", GE, "", (RangeType)10, result); + assert(result.size() == 5); + automCtxDestroy(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "", (RangeType)20, "ab", LE, result); + assert(result.size() == 1); + automCtxDestroy(ctx); + // taosMemoryFree(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "", (RangeType)30, "ab", LT, result); + assert(result.size() == 0); + automCtxDestroy(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "ed", GT, "ed", (RangeType)40, result); + assert(result.size() == 0); + automCtxDestroy(ctx); + } + delete m; +} void fst_get(Fst* fst) { for (int i = 0; i < 10000; i++) { @@ -573,11 +635,12 @@ int main(int argc, char* argv[]) { // path suid colName ver // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); //} - checkFstCheckIterator1(); - checkFstCheckIterator2(); - checkFstCheckIteratorPrefix(); - checkFstCheckIteratorRange1(); - checkFstCheckIteratorRange2(); + // checkFstCheckIterator1(); + // checkFstCheckIterator2(); + // checkFstCheckIteratorPrefix(); + // checkFstCheckIteratorRange1(); + // checkFstCheckIteratorRange2(); + checkFstCheckIteratorRange3(); // checkFstLongTerm(); // checkFstPrefixSearch(); From 3a3fef91f0828ecbb54fbfbf83b0233a53dfc381 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 15:25:44 +0800 Subject: [PATCH 4/8] handle except --- source/libs/index/src/log | 1445 ------------------------------------- 1 file changed, 1445 deletions(-) delete mode 100644 source/libs/index/src/log diff --git a/source/libs/index/src/log b/source/libs/index/src/log deleted file mode 100644 index 0b6cb8ddc2..0000000000 --- a/source/libs/index/src/log +++ /dev/null @@ -1,1445 +0,0 @@ -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 1) /* -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 2) * Copyright (c) 2019 TAOS Data, Inc. -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 3) * -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 4) * This program is free software: you can use, redistribute, and/or modify -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 5) * it under the terms of the GNU Affero General Public License, version 3 -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 6) * or later ("AGPL"), as published by the Free Software Foundation. -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 7) * -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 8) * This program is distributed in the hope that it will be useful, but WITHOUT -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 9) * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 10) * FITNESS FOR A PARTICULAR PURPOSE. -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 11) * -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 12) * You should have received a copy of the GNU Affero General Public License -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 13) * along with this program. If not, see . -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 14) */ -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 15) -5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 16) #include "indexFst.h" -5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 17) #include "indexFstAutomation.h" -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 18) #include "indexInt.h" -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 19) #include "tchecksum.h" -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 20) #include "tcoding.h" -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 21) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 22) static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 23) CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 24) fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 25) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 26) static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { -20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 27) uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 28) fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 29) return nBytes; -20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 30) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 31) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 32) FstUnFinishedNodes* fstUnFinishedNodesCreate() { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 33) FstUnFinishedNodes* nodes = taosMemoryMalloc(sizeof(FstUnFinishedNodes)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 34) if (nodes == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 35) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 36) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 37) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 38) nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 39) fstUnFinishedNodesPushEmpty(nodes, false); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 40) return nodes; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 41) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 42) static void unFinishedNodeDestroyElem(void* elem) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 43) FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 44) fstBuilderNodeDestroy(b->node); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 45) taosMemoryFree(b->last); -7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 46) b->last = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 47) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 48) void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 49) if (nodes == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 50) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 51) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 52) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 53) taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 54) taosMemoryFree(nodes); -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 55) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 56) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 57) void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 58) FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 59) node->isFinal = isFinal; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 60) node->finalOutput = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 61) node->trans = taosArrayInit(16, sizeof(FstTransition)); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 62) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 63) FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 64) taosArrayPush(nodes->stack, &un); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 65) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 66) FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 67) assert(taosArrayGetSize(nodes->stack) == 1); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 68) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 69) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 70) assert(un->last == NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 71) return un->node; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 72) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 73) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 74) FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 75) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 76) fstBuilderNodeUnfinishedLastCompiled(un, addr); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 77) // taosMemoryFree(un->last); // TODO add func FstLastTransitionFree() -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 78) // un->last = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 79) return un->node; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 80) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 81) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 82) FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 83) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 84) assert(un->last == NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 85) return un->node; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 86) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 87) void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 88) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 89) un->node->isFinal = true; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 90) un->node->finalOutput = out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 91) // un->node->trans = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 92) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 93) void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 94) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 95) fstBuilderNodeUnfinishedLastCompiled(un, addr); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 96) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 97) void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 98) FstSlice* s = &bs; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 99) if (fstSliceIsEmpty(s)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 100) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 101) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 102) size_t sz = taosArrayGetSize(nodes->stack) - 1; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 103) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 104) assert(un->last == NULL); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 105) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 106) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 107) // trn->inp = s->data[s->start]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 108) // trn->out = out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 109) int32_t len = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 110) uint8_t* data = fstSliceData(s, &len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 111) un->last = fstLastTransitionCreate(data[0], out); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 112) -d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 113) for (uint64_t i = 1; i < len; i++) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 114) FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 115) n->isFinal = false; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 116) n->finalOutput = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 117) n->trans = taosArrayInit(16, sizeof(FstTransition)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 118) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 119) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 120) // trn->inp = s->data[i]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 121) // trn->out = out; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 122) FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 123) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 124) FstBuilderNodeUnfinished un = {.node = n, .last = trn}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 125) taosArrayPush(nodes->stack, &un); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 126) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 127) fstUnFinishedNodesPushEmpty(nodes, true); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 128) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 129) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 130) uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 131) FstSlice* s = &bs; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 132) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 133) size_t ssz = taosArrayGetSize(node->stack); // stack size -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 134) uint64_t count = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 135) int32_t lsz; // data len -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 136) uint8_t* data = fstSliceData(s, &lsz); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 137) for (size_t i = 0; i < ssz && i < lsz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 138) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 139) if (un->last->inp == data[i]) { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 140) count++; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 141) } else { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 142) break; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 143) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 144) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 145) return count; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 146) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 147) uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 148) FstSlice* s = &bs; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 149) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 150) size_t lsz = (size_t)(s->end - s->start + 1); // data len -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 151) size_t ssz = taosArrayGetSize(node->stack); // stack size -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 152) *out = in; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 153) uint64_t i = 0; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 154) for (i = 0; i < lsz && i < ssz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 155) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 156) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 157) FstLastTransition* t = un->last; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 158) uint64_t addPrefix = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 159) uint8_t* data = fstSliceData(s, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 160) if (t && t->inp == data[i]) { -92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 161) uint64_t commPrefix = TMIN(t->out, *out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 162) uint64_t tAddPrefix = t->out - commPrefix; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 163) (*out) = (*out) - commPrefix; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 164) t->out = commPrefix; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 165) addPrefix = tAddPrefix; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 166) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 167) break; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 168) } -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 169) if (addPrefix != 0) { -73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 170) if (i + 1 < ssz) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 171) FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 172) fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); -73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 173) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 174) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 175) } -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 176) return i; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 177) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 178) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 179) FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { -93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 180) FstState fs = {.state = EmptyFinal, .val = 0}; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 181) if (addr == EMPTY_ADDRESS) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 182) return fs; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 183) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 184) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 185) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 186) uint8_t v = data[addr]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 187) uint8_t t = (v & 0b11000000) >> 6; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 188) if (t == 0b11) { -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 189) fs.state = OneTransNext; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 190) } else if (t == 0b10) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 191) fs.state = OneTrans; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 192) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 193) fs.state = AnyTrans; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 194) } -93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 195) fs.val = v; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 196) return fs; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 197) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 198) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 199) static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 200) {.state = OneTrans, .val = 0b10000000}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 201) {.state = AnyTrans, .val = 0b00000000}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 202) {.state = EmptyFinal, .val = 0b00000000}}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 203) // debug -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 204) static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 205) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 206) FstState fstStateCreate(State state) { -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 207) uint8_t idx = (uint8_t)state; -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 208) return fstStateDict[idx]; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 209) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 210) // compile -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 211) void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 212) FstState s = fstStateCreate(OneTransNext); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 213) fstStateSetCommInput(&s, inp); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 214) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 215) bool null = false; -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 216) uint8_t v = fstStateCommInput(&s, &null); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 217) if (null) { -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 218) // w->write_all(&[inp]) -7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 219) fstCountingWriterWrite(w, &inp, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 220) } -7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 221) fstCountingWriterWrite(w, &(s.val), 1); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 222) // w->write_all(&[s.val]) -7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 223) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 224) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 225) void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 226) Output out = trn->out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 227) uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 228) uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 229) PackSizes packSizes = 0; -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 230) -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 231) FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 232) FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 233) fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 234) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 235) FstState st = fstStateCreate(OneTrans); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 236) -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 237) fstStateSetCommInput(&st, trn->inp); -22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 238) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 239) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 240) uint8_t inp = fstStateCommInput(&st, &null); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 241) if (null == true) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 242) fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 243) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 244) fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 245) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 246) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 247) void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 248) size_t sz = taosArrayGetSize(node->trans); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 249) assert(sz <= 256); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 250) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 251) uint8_t tSize = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 252) uint8_t oSize = packSize(node->finalOutput); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 253) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 254) // finalOutput.is_zero() -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 255) bool anyOuts = (node->finalOutput != 0); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 256) for (size_t i = 0; i < sz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 257) FstTransition* t = taosArrayGet(node->trans, i); -92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 258) tSize = TMAX(tSize, packDeltaSize(addr, t->addr)); -92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 259) oSize = TMAX(oSize, packSize(t->out)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 260) anyOuts = anyOuts || (t->out != 0); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 261) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 262) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 263) PackSizes packSizes = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 264) if (anyOuts) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 265) FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 266) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 267) FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 268) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 269) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 270) FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 271) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 272) FstState st = fstStateCreate(AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 273) fstStateSetFinalState(&st, node->isFinal); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 274) fstStateSetStateNtrans(&st, (uint8_t)sz); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 275) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 276) if (anyOuts) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 277) if (FST_BUILDER_NODE_IS_FINAL(node)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 278) fstCountingWriterPackUintIn(w, node->finalOutput, oSize); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 279) } -d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 280) for (int32_t i = sz - 1; i >= 0; i--) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 281) FstTransition* t = taosArrayGet(node->trans, i); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 282) fstCountingWriterPackUintIn(w, t->out, oSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 283) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 284) } -d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 285) for (int32_t i = sz - 1; i >= 0; i--) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 286) FstTransition* t = taosArrayGet(node->trans, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 287) fstPackDeltaIn(w, addr, t->addr, tSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 288) } -d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 289) for (int32_t i = sz - 1; i >= 0; i--) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 290) FstTransition* t = taosArrayGet(node->trans, i); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 291) fstCountingWriterWrite(w, (char*)&t->inp, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 292) // fstPackDeltaIn(w, addr, t->addr, tSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 293) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 294) if (sz > TRANS_INDEX_THRESHOLD) { -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 295) // A value of 255 indicates that no transition exists for the byte -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 296) // at that index. (Except when there are 256 transitions.) Namely, -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 297) // any value greater than or equal to the number of transitions in -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 298) // this node indicates an absent transition. -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 299) uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); -984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 300) memset(index, 255, sizeof(uint8_t) * 256); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 301) /// for (uint8_t i = 0; i < 256; i++) { -984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 302) // index[i] = 255; -984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 303) ///} -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 304) for (size_t i = 0; i < sz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 305) FstTransition* t = taosArrayGet(node->trans, i); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 306) index[t->inp] = i; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 307) // fstPackDeltaIn(w, addr, t->addr, tSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 308) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 309) fstCountingWriterWrite(w, (char*)index, 256); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 310) taosMemoryFree(index); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 311) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 312) fstCountingWriterWrite(w, (char*)&packSizes, 1); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 313) bool null = false; -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 314) fstStateStateNtrans(&st, &null); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 315) if (null == true) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 316) // 256 can't be represented in a u8, so we abuse the fact that -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 317) // the # of transitions can never be 1 here, since 1 is always -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 318) // encoded in the state byte. -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 319) uint8_t v = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 320) if (sz == 256) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 321) fstCountingWriterWrite(w, (char*)&v, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 322) } else { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 323) fstCountingWriterWrite(w, (char*)&sz, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 324) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 325) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 326) fstCountingWriterWrite(w, (char*)(&(st.val)), 1); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 327) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 328) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 329) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 330) // set_comm_input -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 331) void fstStateSetCommInput(FstState* s, uint8_t inp) { -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 332) assert(s->state == OneTransNext || s->state == OneTrans); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 333) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 334) uint8_t val; -9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 335) COMMON_INDEX(inp, 0b111111, val); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 336) s->val = (s->val & fstStateDict[s->state].val) | val; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 337) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 338) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 339) // comm_input -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 340) uint8_t fstStateCommInput(FstState* s, bool* null) { -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 341) assert(s->state == OneTransNext || s->state == OneTrans); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 342) uint8_t v = s->val & 0b00111111; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 343) if (v == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 344) *null = true; -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 345) return v; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 346) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 347) // v = 0 indicate that common_input is None -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 348) return v == 0 ? 0 : COMMON_INPUT(v); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 349) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 350) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 351) // input_len -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 352) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 353) uint64_t fstStateInputLen(FstState* s) { -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 354) assert(s->state == OneTransNext || s->state == OneTrans); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 355) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 356) fstStateCommInput(s, &null); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 357) return null ? 1 : 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 358) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 359) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 360) // end_addr -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 361) uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { -4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 362) assert(s->state == OneTransNext); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 363) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 364) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 365) uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { -4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 366) assert(s->state == OneTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 367) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 368) - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 369) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 370) uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 371) uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 372) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 373) uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 374) return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 375) - fstStateTotalTransSize(state, version, sizes, nTrans) - nTrans * oSizes // output values -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 376) - finalOsize; // final output -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 377) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 378) // input -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 379) uint8_t fstStateInput(FstState* s, FstNode* node) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 380) assert(s->state == OneTransNext || s->state == OneTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 381) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 382) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 383) uint8_t inp = fstStateCommInput(s, &null); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 384) uint8_t* data = fstSliceData(slice, NULL); -9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 385) return null == false ? inp : data[node->start - 1]; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 386) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 387) uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 388) assert(s->state == AnyTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 389) FstSlice* slice = &node->data; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 390) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 391) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 392) - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 393) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 394) uint8_t* data = fstSliceData(slice, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 395) return data[at]; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 396) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 397) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 398) // trans_addr -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 399) CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 400) assert(s->state == OneTransNext || s->state == OneTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 401) FstSlice* slice = &node->data; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 402) if (s->state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 403) return (CompiledAddr)(node->end) - 1; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 404) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 405) PackSizes sizes = node->sizes; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 406) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 407) uint64_t i = node->start - fstStateInputLen(s) - 1 // PackSizes -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 408) - tSizes; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 409) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 410) // refactor error logic -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 411) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 412) return unpackDelta(data + i, tSizes, node->end); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 413) } -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 414) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 415) CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 416) assert(s->state == AnyTrans); -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 417) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 418) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 419) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 420) uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 421) node->nTrans - (i * tSizes) - tSizes; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 422) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 423) return unpackDelta(data + at, tSizes, node->end); -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 424) } -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 425) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 426) // sizes -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 427) PackSizes fstStateSizes(FstState* s, FstSlice* slice) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 428) assert(s->state == OneTrans || s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 429) uint64_t i; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 430) if (s->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 431) i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 432) } else { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 433) i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 434) } -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 435) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 436) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 437) return (PackSizes)(*(data + i)); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 438) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 439) // Output -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 440) Output fstStateOutput(FstState* s, FstNode* node) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 441) assert(s->state == OneTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 442) -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 443) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 444) if (oSizes == 0) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 445) return 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 446) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 447) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 448) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 449) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 450) uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 451) uint8_t* data = fstSliceData(slice, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 452) return unpackUint64(data + i, oSizes); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 453) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 454) Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 455) assert(s->state == AnyTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 456) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 457) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 458) if (oSizes == 0) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 459) return 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 460) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 461) FstSlice* slice = &node->data; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 462) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 463) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 464) - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 465) -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 466) return unpackUint64(data + at, oSizes); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 467) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 468) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 469) // anyTrans specify function -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 470) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 471) void fstStateSetFinalState(FstState* s, bool yes) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 472) assert(s->state == AnyTrans); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 473) if (yes) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 474) s->val |= 0b01000000; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 475) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 476) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 477) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 478) bool fstStateIsFinalState(FstState* s) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 479) assert(s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 480) return (s->val & 0b01000000) == 0b01000000; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 481) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 482) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 483) void fstStateSetStateNtrans(FstState* s, uint8_t n) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 484) assert(s->state == AnyTrans); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 485) if (n <= 0b00111111) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 486) s->val = (s->val & 0b11000000) | n; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 487) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 488) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 489) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 490) // state_ntrans -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 491) uint8_t fstStateStateNtrans(FstState* s, bool* null) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 492) assert(s->state == AnyTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 493) *null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 494) uint8_t n = s->val & 0b00111111; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 495) -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 496) if (n == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 497) *null = true; // None -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 498) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 499) return n; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 500) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 501) uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 502) assert(s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 503) uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 504) return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 505) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 506) uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 507) assert(s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 508) return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 509) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 510) uint64_t fstStateNtransLen(FstState* s) { -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 511) assert(s->state == AnyTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 512) bool null = false; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 513) fstStateStateNtrans(s, &null); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 514) return null == true ? 1 : 0; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 515) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 516) uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 517) bool null = false; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 518) uint8_t n = fstStateStateNtrans(s, &null); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 519) if (null != true) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 520) return n; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 521) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 522) int32_t len; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 523) uint8_t* data = fstSliceData(slice, &len); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 524) n = data[len - 2]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 525) // n = data[slice->end - 1]; // data[data.len() - 2] -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 526) return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 527) // then it is encoded in the state byte -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 528) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 529) Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 530) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 531) if (oSizes == 0 || !fstStateIsFinalState(s)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 532) return 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 533) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 534) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 535) uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 536) - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 537) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 538) return unpackUint64(data + at, (uint8_t)oSizes); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 539) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 540) uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 541) assert(s->state == AnyTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 542) FstSlice* slice = &node->data; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 543) if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 544) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 545) - fstStateTransIndexSize(s, node->version, node->nTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 546) int32_t dlen = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 547) uint8_t* data = fstSliceData(slice, &dlen); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 548) uint64_t i = data[at + b]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 549) // uint64_t i = slice->data[slice->start + at + b]; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 550) if (i >= node->nTrans) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 551) *null = true; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 552) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 553) return i; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 554) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 555) uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 556) - node->nTrans; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 557) uint64_t end = start + node->nTrans; -73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 558) FstSlice t = fstSliceCopy(slice, start, end - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 559) int32_t len = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 560) uint8_t* data = fstSliceData(&t, &len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 561) int i = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 562) for (; i < len; i++) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 563) uint8_t v = data[i]; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 564) if (v == b) { -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 565) fstSliceDestroy(&t); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 566) return node->nTrans - i - 1; // bug -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 567) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 568) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 569) if (i == len) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 570) *null = true; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 571) } -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 572) fstSliceDestroy(&t); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 573) } -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 574) -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 575) return 0; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 576) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 577) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 578) // fst node function -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 579) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 580) FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 581) FstNode* n = (FstNode*)taosMemoryMalloc(sizeof(FstNode)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 582) if (n == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 583) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 584) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 585) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 586) FstState st = fstStateCreateFrom(slice, addr); -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 587) -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 588) if (st.state == EmptyFinal) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 589) n->data = fstSliceCreate(NULL, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 590) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 591) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 592) n->start = EMPTY_ADDRESS; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 593) n->end = EMPTY_ADDRESS; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 594) n->isFinal = true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 595) n->nTrans = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 596) n->sizes = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 597) n->finalOutput = 0; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 598) } else if (st.state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 599) n->data = fstSliceCopy(slice, 0, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 600) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 601) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 602) n->start = addr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 603) n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 604) n->isFinal = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 605) n->sizes = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 606) n->nTrans = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 607) n->finalOutput = 0; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 608) } else if (st.state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 609) FstSlice data = fstSliceCopy(slice, 0, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 610) PackSizes sz = fstStateSizes(&st, &data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 611) n->data = data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 612) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 613) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 614) n->start = addr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 615) n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 616) n->isFinal = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 617) n->nTrans = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 618) n->sizes = sz; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 619) n->finalOutput = 0; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 620) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 621) FstSlice data = fstSliceCopy(slice, 0, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 622) uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 623) uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 624) n->data = data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 625) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 626) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 627) n->start = addr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 628) n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 629) n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 630) n->nTrans = nTrans; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 631) n->sizes = sz; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 632) n->finalOutput = -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 633) fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 634) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 635) return n; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 636) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 637) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 638) // debug state transition -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 639) static const char* fstNodeState(FstNode* node) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 640) FstState* st = &node->state; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 641) return fstStateStr[st->state]; -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 642) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 643) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 644) void fstNodeDestroy(FstNode* node) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 645) fstSliceDestroy(&node->data); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 646) taosMemoryFree(node); -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 647) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 648) FstTransitions* fstNodeTransitions(FstNode* node) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 649) FstTransitions* t = taosMemoryMalloc(sizeof(FstTransitions)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 650) if (NULL == t) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 651) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 652) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 653) FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 654) t->range = range; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 655) t->node = node; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 656) return t; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 657) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 658) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 659) // Returns the transition at index `i`. -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 660) bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 661) bool s = true; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 662) FstState* st = &node->state; -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 663) if (st->state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 664) trn->inp = fstStateInput(st, node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 665) trn->out = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 666) trn->addr = fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 667) } else if (st->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 668) trn->inp = fstStateInput(st, node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 669) trn->out = fstStateOutput(st, node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 670) trn->addr = fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 671) } else if (st->state == AnyTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 672) trn->inp = fstStateInputForAnyTrans(st, node, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 673) trn->out = fstStateOutputForAnyTrans(st, node, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 674) trn->addr = fstStateTransAddrForAnyTrans(st, node, i); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 675) } else { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 676) s = false; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 677) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 678) return s; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 679) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 680) -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 681) // Returns the transition address of the `i`th transition -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 682) bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 683) bool s = true; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 684) FstState* st = &node->state; -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 685) if (st->state == OneTransNext) { -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 686) assert(i == 0); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 687) fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 688) } else if (st->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 689) assert(i == 0); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 690) fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 691) } else if (st->state == AnyTrans) { -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 692) fstStateTransAddrForAnyTrans(st, node, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 693) } else if (FST_STATE_EMPTY_FINAL(node)) { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 694) s = false; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 695) } else { -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 696) assert(0); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 697) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 698) return s; -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 699) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 700) -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 701) // Finds the `i`th transition corresponding to the given input byte. -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 702) // If no transition for this byte exists, then `false` is returned. -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 703) bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 704) bool s = true; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 705) FstState* st = &node->state; -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 706) if (st->state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 707) if (fstStateInput(st, node) == b) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 708) *res = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 709) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 710) s = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 711) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 712) } else if (st->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 713) if (fstStateInput(st, node) == b) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 714) *res = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 715) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 716) s = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 717) } -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 718) } else if (st->state == AnyTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 719) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 720) uint64_t out = fstStateFindInput(st, node, b, &null); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 721) if (null == false) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 722) *res = out; -2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 723) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 724) s = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 725) } -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 726) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 727) return s; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 728) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 729) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 730) bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 731) size_t sz = taosArrayGetSize(builderNode->trans); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 732) assert(sz < 256); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 733) if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 734) return true; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 735) } else if (sz != 1 || builderNode->isFinal) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 736) fstStateCompileForAnyTrans(w, addr, builderNode); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 737) // AnyTrans->Compile(w, addr, node); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 738) } else { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 739) FstTransition* tran = taosArrayGet(builderNode->trans, 0); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 740) if (tran->addr == lastAddr && tran->out == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 741) fstStateCompileForOneTransNext(w, addr, tran->inp); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 742) // OneTransNext::compile(w, lastAddr, tran->inp); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 743) return true; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 744) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 745) fstStateCompileForOneTrans(w, addr, tran); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 746) // OneTrans::Compile(w, lastAddr, *tran); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 747) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 748) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 749) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 750) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 751) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 752) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 753) bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 754) return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 755) } -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 756) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 757) FstBuilder* fstBuilderCreate(void* w, FstType ty) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 758) FstBuilder* b = taosMemoryMalloc(sizeof(FstBuilder)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 759) if (NULL == b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 760) return b; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 761) } -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 762) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 763) b->wrt = fstCountingWriterCreate(w); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 764) b->unfinished = fstUnFinishedNodesCreate(); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 765) b->registry = fstRegistryCreate(10000, 2); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 766) b->last = fstSliceCreate(NULL, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 767) b->lastAddr = NONE_ADDRESS; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 768) b->len = 0; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 769) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 770) char buf64[8] = {0}; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 771) void* pBuf64 = buf64; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 772) taosEncodeFixedU64(&pBuf64, VERSION); -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 773) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 774) -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 775) pBuf64 = buf64; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 776) memset(buf64, 0, sizeof(buf64)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 777) taosEncodeFixedU64(&pBuf64, ty); -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 778) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 779) -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 780) return b; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 781) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 782) void fstBuilderDestroy(FstBuilder* b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 783) if (b == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 784) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 785) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 786) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 787) fstCountingWriterDestroy(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 788) fstUnFinishedNodesDestroy(b->unfinished); -260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 789) fstRegistryDestroy(b->registry); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 790) fstSliceDestroy(&b->last); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 791) taosMemoryFree(b); -260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 792) } -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 793) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 794) bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 795) OrderType t = fstBuilderCheckLastKey(b, bs, true); -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 796) if (t == Ordered) { -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 797) // add log info -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 798) fstBuilderInsertOutput(b, bs, in); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 799) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 800) } -ebcb9be39fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-22 12:20:59 +0800 801) indexInfo("fst write key must be ordered"); -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 802) return false; -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 803) } -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 804) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 805) void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 806) FstSlice* s = &bs; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 807) if (fstSliceIsEmpty(s)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 808) b->len = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 809) fstUnFinishedNodesSetRootOutput(b->unfinished, in); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 810) return; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 811) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 812) // if (in != 0) { //if let Some(in) = in -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 813) // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 814) //} else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 815) // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 816) // out = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 817) //} -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 818) Output out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 819) uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 820) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 821) if (prefixLen == FST_SLICE_LEN(s)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 822) assert(out == 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 823) return; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 824) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 825) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 826) b->len += 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 827) fstBuilderCompileFrom(b, prefixLen); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 828) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 829) FstSlice sub = fstSliceCopy(s, prefixLen, s->end); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 830) fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 831) fstSliceDestroy(&sub); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 832) return; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 833) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 834) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 835) OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 836) FstSlice* input = &bs; -716c0045f8a source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 17:29:59 +0800 837) if (fstSliceIsEmpty(&b->last)) { -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 838) fstSliceDestroy(&b->last); -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 839) // deep copy or not -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 840) b->last = fstSliceDeepCopy(&bs, input->start, input->end); -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 841) } else { -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 842) int comp = fstSliceCompare(&b->last, &bs); -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 843) if (comp == 0 && ckDup) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 844) return DuplicateKey; -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 845) } else if (comp == 1) { -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 846) return OutOfOrdered; -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 847) } -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 848) // deep copy or not -d8d6c04fd50 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 16:18:35 +0800 849) fstSliceDestroy(&b->last); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 850) b->last = fstSliceDeepCopy(&bs, input->start, input->end); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 851) } -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 852) return Ordered; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 853) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 854) void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 855) CompiledAddr addr = NONE_ADDRESS; -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 856) while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 857) FstBuilderNode* bn = NULL; -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 858) if (addr == NONE_ADDRESS) { -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 859) bn = fstUnFinishedNodesPopEmpty(b->unfinished); -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 860) } else { -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 861) bn = fstUnFinishedNodesPopFreeze(b->unfinished, addr); -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 862) } -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 863) addr = fstBuilderCompile(b, bn); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 864) -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 865) fstBuilderNodeDestroy(bn); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 866) assert(addr != NONE_ADDRESS); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 867) // fstBuilderNodeDestroy(n); -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 868) } -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 869) fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 870) return; -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 871) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 872) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 873) CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 874) if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 875) return EMPTY_ADDRESS; -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 876) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 877) FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 878) if (entry->state == FOUND) { -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 879) CompiledAddr ret = entry->addr; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 880) fstRegistryEntryDestroy(entry); -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 881) return ret; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 882) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 883) CompiledAddr startAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt)); -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 884) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 885) fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 886) b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 887) if (entry->state == NOTFOUND) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 888) FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 889) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 890) fstRegistryEntryDestroy(entry); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 891) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 892) return b->lastAddr; -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 893) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 894) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 895) void* fstBuilderInsertInner(FstBuilder* b) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 896) fstBuilderCompileFrom(b, 0); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 897) FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 898) CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 899) fstBuilderNodeDestroy(rootNode); -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 900) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 901) char buf64[8] = {0}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 902) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 903) void* pBuf64 = buf64; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 904) taosEncodeFixedU64(&pBuf64, b->len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 905) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 906) -d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 907) pBuf64 = buf64; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 908) taosEncodeFixedU64(&pBuf64, rootAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 909) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 910) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 911) char buf32[4] = {0}; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 912) void* pBuf32 = buf32; -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 913) uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 914) taosEncodeFixedU32(&pBuf32, sum); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 915) fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 916) -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 917) fstCountingWriterFlush(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 918) // fstCountingWriterDestroy(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 919) // b->wrt = NULL; -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 920) return b->wrt; -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 921) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 922) void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); } -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 923) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 924) FstSlice fstNodeAsSlice(FstNode* node) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 925) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 926) FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 927) return s; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 928) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 929) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 930) FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 931) FstLastTransition* trn = taosMemoryMalloc(sizeof(FstLastTransition)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 932) if (trn == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 933) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 934) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 935) -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 936) trn->inp = inp; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 937) trn->out = out; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 938) return trn; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 939) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 940) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 941) void fstLastTransitionDestroy(FstLastTransition* trn) { taosMemoryFree(trn); } -22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 942) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 943) void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 944) FstLastTransition* trn = unNode->last; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 945) if (trn == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 946) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 947) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 948) FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 949) taosArrayPush(unNode->node->trans, &t); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 950) fstLastTransitionDestroy(trn); -7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 951) unNode->last = NULL; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 952) return; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 953) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 954) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 955) void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 956) if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 957) unNode->node->finalOutput += out; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 958) } -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 959) size_t sz = taosArrayGetSize(unNode->node->trans); -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 960) for (size_t i = 0; i < sz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 961) FstTransition* trn = taosArrayGet(unNode->node->trans, i); -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 962) trn->out += out; -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 963) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 964) if (unNode->last) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 965) unNode->last->out += out; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 966) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 967) return; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 968) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 969) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 970) Fst* fstCreate(FstSlice* slice) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 971) int32_t slen; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 972) char* buf = fstSliceData(slice, &slen); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 973) if (slen < 36) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 974) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 975) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 976) uint64_t len = slen; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 977) uint64_t skip = 0; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 978) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 979) uint64_t version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 980) taosDecodeFixedU64(buf, &version); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 981) skip += sizeof(version); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 982) if (version == 0 || version > VERSION) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 983) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 984) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 985) -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 986) uint64_t type; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 987) taosDecodeFixedU64(buf + skip, &type); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 988) skip += sizeof(type); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 989) -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 990) uint32_t checkSum = 0; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 991) len -= sizeof(checkSum); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 992) taosDecodeFixedU32(buf + len, &checkSum); -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 993) if (taosCheckChecksum(buf, len, checkSum)) { -f9d83c044dd source/libs/index/src/index_fst.c (yihaoDeng 2022-01-10 23:42:58 +0800 994) indexError("index file is corrupted"); -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 995) // verify fst -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 996) return NULL; -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 997) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 998) CompiledAddr rootAddr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 999) len -= sizeof(rootAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1000) taosDecodeFixedU64(buf + len, &rootAddr); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1001) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1002) uint64_t fstLen; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1003) len -= sizeof(fstLen); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1004) taosDecodeFixedU64(buf + len, &fstLen); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1005) // TODO(validate root addr) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1006) Fst* fst = (Fst*)taosMemoryCalloc(1, sizeof(Fst)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1007) if (fst == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1008) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1009) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1010) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1011) fst->meta = (FstMeta*)taosMemoryMalloc(sizeof(FstMeta)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1012) if (NULL == fst->meta) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1013) goto FST_CREAT_FAILED; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1014) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1015) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1016) fst->meta->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1017) fst->meta->rootAddr = rootAddr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1018) fst->meta->ty = type; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1019) fst->meta->len = fstLen; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1020) fst->meta->checkSum = checkSum; -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1021) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1022) FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1023) *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1024) fst->data = s; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1025) -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1026) taosThreadMutexInit(&fst->mtx, NULL); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1027) return fst; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1028) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1029) FST_CREAT_FAILED: -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1030) taosMemoryFree(fst->meta); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1031) taosMemoryFree(fst); -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1032) -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1033) return NULL; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1034) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1035) void fstDestroy(Fst* fst) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1036) if (fst) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1037) taosMemoryFree(fst->meta); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1038) fstSliceDestroy(fst->data); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1039) taosMemoryFree(fst->data); -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1040) taosThreadMutexDestroy(&fst->mtx); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1041) } -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1042) taosMemoryFree(fst); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1043) } -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1044) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1045) bool fstGet(Fst* fst, FstSlice* b, Output* out) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1046) // dec lock range -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1047) // taosThreadMutexLock(&fst->mtx); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1048) FstNode* root = fstGetRoot(fst); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1049) Output tOut = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1050) int32_t len; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1051) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1052) uint8_t* data = fstSliceData(b, &len); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1053) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1054) SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1055) taosArrayPush(nodes, &root); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1056) for (uint32_t i = 0; i < len; i++) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1057) uint8_t inp = data[i]; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1058) Output res = 0; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1059) if (false == fstNodeFindInput(root, inp, &res)) { -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1060) // taosThreadMutexUnlock(&fst->mtx); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1061) return false; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1062) } -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1063) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1064) FstTransition trn; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1065) fstNodeGetTransitionAt(root, res, &trn); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1066) tOut += trn.out; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1067) root = fstGetNode(fst, trn.addr); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1068) taosArrayPush(nodes, &root); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1069) } -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1070) if (!FST_NODE_IS_FINAL(root)) { -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1071) // taosThreadMutexUnlock(&fst->mtx); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1072) return false; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1073) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1074) tOut = tOut + FST_NODE_FINAL_OUTPUT(root); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1075) } -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 1076) -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1077) for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1078) FstNode** node = (FstNode**)taosArrayGet(nodes, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1079) fstNodeDestroy(*node); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1080) } -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1081) taosArrayDestroy(nodes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1082) // fst->root = NULL; -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1083) // taosThreadMutexUnlock(&fst->mtx); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1084) *out = tOut; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1085) return true; -f07045c1d8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 17:28:58 +0800 1086) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1087) FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1088) // refactor later -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1089) return fstStreamBuilderCreate(fst, ctx); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1090) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1091) StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1092) if (sb == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1093) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1094) } -26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1095) return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); -26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1096) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1097) FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1098) // refactor later -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1099) return fstStreamBuilderCreate(fst, ctx); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1100) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1101) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1102) FstNode* fstGetRoot(Fst* fst) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1103) CompiledAddr rAddr = fstGetRootAddr(fst); -2eb0053e5ad source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 22:19:56 +0800 1104) return fstGetNode(fst, rAddr); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1105) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1106) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1107) FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1108) // refactor later -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1109) return fstNodeCreate(fst->meta->version, addr, fst->data); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1110) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1111) FstType fstGetType(Fst* fst) { return fst->meta->ty; } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1112) CompiledAddr fstGetRootAddr(Fst* fst) { return fst->meta->rootAddr; } -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1113) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1114) Output fstEmptyFinalOutput(Fst* fst, bool* null) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1115) Output res = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1116) FstNode* node = fstGetRoot(fst); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1117) if (FST_NODE_IS_FINAL(node)) { -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1118) *null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1119) res = FST_NODE_FINAL_OUTPUT(node); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1120) } else { -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1121) *null = true; -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1122) } -40fc4b6cf38 source/libs/index/src/index_fst.c (yihaoDeng 2022-01-02 17:50:40 +0800 1123) fstNodeDestroy(node); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1124) return res; -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1125) } -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1126) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1127) bool fstVerify(Fst* fst) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1128) uint32_t len, checkSum = fst->meta->checkSum; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1129) uint8_t* data = fstSliceData(fst->data, &len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1130) TSCKSUM initSum = 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1131) if (!taosCheckChecksumWhole(data, len)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1132) return false; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1133) } -a09dcbdd85f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:59:51 +0800 1134) return true; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1135) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1136) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1137) // data bound function -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1138) FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1139) FstBoundWithData* b = taosMemoryCalloc(1, sizeof(FstBoundWithData)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1140) if (b == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1141) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1142) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1143) -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1144) if (data != NULL) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1145) b->data = fstSliceCopy(data, data->start, data->end); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1146) } else { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1147) b->data = fstSliceCreate(NULL, 0); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1148) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1149) b->type = type; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1150) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1151) return b; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1152) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1153) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1154) bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1155) int comp = fstSliceCompare(slice, &bound->data); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1156) if (bound->type == Included) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1157) return comp > 0 ? true : false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1158) } else if (bound->type == Excluded) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1159) return comp >= 0 ? true : false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1160) } else { -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1161) return false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1162) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1163) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1164) bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1165) if (bound->type == Unbounded) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1166) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1167) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1168) return fstSliceIsEmpty(&bound->data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1169) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1170) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1171) -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1172) bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1173) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1174) void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1175) -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1176) StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1177) FstBoundWithData* max) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1178) StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1179) if (sws == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1180) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1181) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1182) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1183) sws->fst = fst; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1184) sws->aut = automation; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1185) sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1186) -0c5f2d1da95 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 21:41:41 +0800 1187) sws->emptyOutput.null = true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1188) sws->emptyOutput.out = 0; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1189) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1190) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1191) sws->endAt = max; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1192) streamWithStateSeekMin(sws, min); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1193) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1194) return sws; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1195) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1196) void streamWithStateDestroy(StreamWithState* sws) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1197) if (sws == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1198) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1199) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1200) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1201) taosArrayDestroy(sws->inp); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1202) taosArrayDestroyEx(sws->stack, streamStateDestroy); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1203) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1204) taosMemoryFree(sws); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1205) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1206) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1207) bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1208) AutomationCtx* aut = sws->aut; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1209) if (fstBoundWithDataIsEmpty(min)) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1210) if (fstBoundWithDataIsIncluded(min)) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1211) sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1212) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1213) StreamState s = {.node = fstGetRoot(sws->fst), -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1214) .trans = 0, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1215) .out = {.null = false, .out = 0}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1216) .autState = automFuncs[aut->type].start(aut)}; // auto.start callback -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1217) taosArrayPush(sws->stack, &s); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1218) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1219) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1220) FstSlice* key = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1221) bool inclusize = false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1222) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1223) if (min->type == Included) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1224) key = &min->data; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1225) inclusize = true; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1226) } else if (min->type == Excluded) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1227) key = &min->data; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1228) } else { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1229) return false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1230) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1231) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1232) FstNode* node = fstGetRoot(sws->fst); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1233) Output out = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1234) // void* autState = sws->aut->start(); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1235) void* autState = automFuncs[aut->type].start(aut); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1236) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1237) int32_t len; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1238) uint8_t* data = fstSliceData(key, &len); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1239) for (uint32_t i = 0; i < len; i++) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1240) uint8_t b = data[i]; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1241) uint64_t res = 0; -2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1242) bool find = fstNodeFindInput(node, b, &res); -2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1243) if (find == true) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1244) FstTransition trn; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1245) fstNodeGetTransitionAt(node, res, &trn); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1246) void* preState = autState; -cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1247) // autState = sws->aut->accept(preState, b); -cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1248) autState = automFuncs[aut->type].accept(aut, preState, b); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1249) taosArrayPush(sws->inp, &b); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1250) StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1251) taosArrayPush(sws->stack, &s); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1252) out += trn.out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1253) node = fstGetNode(sws->fst, trn.addr); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1254) } else { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1255) // This is a little tricky. We're in this case if the -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1256) // given bound is not a prefix of any key in the FST. -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1257) // Since this is a minimum bound, we need to find the -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1258) // first transition in this node that proceeds the current -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1259) // input byte. -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1260) FstTransitions* trans = fstNodeTransitions(node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1261) uint64_t i = 0; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1262) for (i = trans->range.start; i < trans->range.end; i++) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1263) FstTransition trn; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1264) if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1265) break; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1266) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1267) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1268) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1269) StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1270) taosArrayPush(sws->stack, &s); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1271) return true; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1272) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1273) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1274) uint32_t sz = taosArrayGetSize(sws->stack); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1275) if (sz != 0) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1276) StreamState* s = taosArrayGet(sws->stack, sz - 1); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1277) if (inclusize) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1278) s->trans -= 1; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1279) taosArrayPop(sws->inp); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1280) } else { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1281) FstNode* n = s->node; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1282) uint64_t trans = s->trans; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1283) FstTransition trn; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1284) fstNodeGetTransitionAt(n, trans - 1, &trn); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1285) StreamState s = { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1286) .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1287) taosArrayPush(sws->stack, &s); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1288) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1289) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1290) return false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1291) } -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1292) -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1293) return false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1294) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1295) StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1296) AutomationCtx* aut = sws->aut; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1297) FstOutput output = sws->emptyOutput; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1298) if (output.null == false) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1299) FstSlice emptySlice = fstSliceCreate(NULL, 0); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1300) if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1301) taosArrayDestroyEx(sws->stack, streamStateDestroy); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1302) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1303) return NULL; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1304) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1305) void* start = automFuncs[aut->type].start(aut); -cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1306) if (automFuncs[aut->type].isMatch(aut, start)) { -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1307) FstSlice s = fstSliceCreate(NULL, 0); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1308) return swsResultCreate(&s, output, callback == NULL ? NULL : callback(start)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1309) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1310) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1311) SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1312) while (taosArrayGetSize(sws->stack) > 0) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1313) StreamState* p = (StreamState*)taosArrayPop(sws->stack); -f0110af30ec source/libs/index/src/index_fst.c (yihaoDeng 2021-12-16 22:04:47 +0800 1314) if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1315) if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1316) taosArrayPop(sws->inp); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1317) } -6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1318) streamStateDestroy(p); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1319) continue; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1320) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1321) FstTransition trn; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1322) fstNodeGetTransitionAt(p->node, p->trans, &trn); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1323) -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1324) Output out = p->out.out + trn.out; -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1325) void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1326) void* tState = (callback == NULL) ? NULL : callback(nextState); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1327) bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1328) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1329) FstNode* nextNode = fstGetNode(sws->fst, trn.addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1330) taosArrayPush(nodes, &nextNode); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1331) taosArrayPush(sws->inp, &(trn.inp)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1332) -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1333) if (FST_NODE_IS_FINAL(nextNode)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1334) // void *eofState = sws->aut->acceptEof(nextState); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1335) void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1336) if (eofState != NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1337) isMatch = automFuncs[aut->type].isMatch(aut, eofState); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1338) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1339) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1340) StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1341) taosArrayPush(sws->stack, &s1); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1342) -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1343) StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1344) taosArrayPush(sws->stack, &s2); -6f3c49ee73d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-14 16:17:38 +0800 1345) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1346) size_t isz = taosArrayGetSize(sws->inp); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1347) uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1348) for (uint32_t i = 0; i < isz; i++) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1349) buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1350) } -00000000000 source/libs/index/src/indexFst.c (Not Committed Yet 2022-04-02 15:19:56 +0800 1351) FstSlice slice = fstSliceCreate(buf, isz); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1352) if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1353) taosArrayDestroyEx(sws->stack, streamStateDestroy); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1354) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1355) taosMemoryFreeClear(buf); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1356) fstSliceDestroy(&slice); -06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1357) taosArrayDestroy(nodes); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1358) return NULL; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1359) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1360) if (FST_NODE_IS_FINAL(nextNode) && isMatch) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1361) FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1362) StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1363) taosMemoryFreeClear(buf); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1364) fstSliceDestroy(&slice); -06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1365) taosArrayDestroy(nodes); -6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1366) nodes = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1367) return result; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1368) } -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1369) taosMemoryFreeClear(buf); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1370) fstSliceDestroy(&slice); -6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1371) }; -06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1372) taosArrayDestroy(nodes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1373) return NULL; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1374) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1375) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1376) StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1377) StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1378) if (result == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1379) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1380) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1381) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1382) result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1383) result->out = fOut; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1384) result->state = state; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1385) return result; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1386) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1387) void swsResultDestroy(StreamWithStateResult* result) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1388) if (NULL == result) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1389) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1390) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1391) -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1392) fstSliceDestroy(&result->data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1393) startWithStateValueDestroy(result->state); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1394) taosMemoryFree(result); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1395) } -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1396) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1397) void streamStateDestroy(void* s) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1398) if (NULL == s) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1399) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1400) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1401) StreamState* ss = (StreamState*)s; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1402) fstNodeDestroy(ss->node); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1403) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1404) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1405) FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1406) FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1407) if (NULL == b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1408) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1409) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1410) -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1411) b->fst = fst; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1412) b->aut = aut; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1413) b->min = fstBoundStateCreate(Unbounded, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1414) b->max = fstBoundStateCreate(Unbounded, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1415) return b; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1416) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1417) void fstStreamBuilderDestroy(FstStreamBuilder* b) { -7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1418) fstSliceDestroy(&b->min->data); -7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1419) fstSliceDestroy(&b->max->data); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1420) taosMemoryFreeClear(b->min); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1421) taosMemoryFreeClear(b->max); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1422) taosMemoryFree(b); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1423) } -30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1424) void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1425) if (b == NULL) { -30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1426) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1427) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1428) if (type == GE) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1429) b->min->type = Included; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1430) fstSliceDestroy(&(b->min->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1431) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1432) } else if (type == GT) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1433) b->min->type = Excluded; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1434) fstSliceDestroy(&(b->min->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1435) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1436) } else if (type == LE) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1437) b->max->type = Included; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1438) fstSliceDestroy(&(b->max->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1439) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1440) } else if (type == LT) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1441) b->max->type = Excluded; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1442) fstSliceDestroy(&(b->max->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1443) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1444) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1445) } From ba5503da680deacb72ec4b543ff777365956f601 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Apr 2022 03:27:07 -0400 Subject: [PATCH 5/8] insert using implement --- include/libs/nodes/querynodes.h | 1 - include/util/tdef.h | 2 +- source/libs/command/src/command.c | 13 +++- source/libs/parser/inc/parInsertData.h | 7 +- source/libs/parser/src/parInsert.c | 19 ++++-- source/libs/parser/src/parInsertData.c | 71 +++++++++++--------- source/libs/parser/test/parserInsertTest.cpp | 15 ++++- 7 files changed, 81 insertions(+), 47 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index dec709535f..ca4646a5ca 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -277,7 +277,6 @@ typedef struct SVnodeModifOpStmt { ENodeType nodeType; ENodeType sqlNodeType; SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint32_t insertType; // insert data from [file|sql statement| bound statement] const char* sql; // current sql statement position diff --git a/include/util/tdef.h b/include/util/tdef.h index b1bebcee46..be88afcf66 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -307,7 +307,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_DAYS_PER_FILE (1 * 1440) // unit minute +#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 7ffbd4f80a..4d4ac6c1e4 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -16,7 +16,16 @@ #include "command.h" #include "tdatablock.h" -// #define SET_VARSTR(pData, val, pOffset) +static int32_t getSchemaBytes(const SSchema* pSchema) { + switch (pSchema->type) { + case TSDB_DATA_TYPE_BINARY: + return (pSchema->bytes - VARSTR_HEADER_SIZE); + case TSDB_DATA_TYPE_NCHAR: + return (pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + default: + return pSchema->bytes; + } +} static void buildRspData(const STableMeta* pMeta, char* pData) { int32_t* pColSizes = (int32_t*)pData; @@ -50,7 +59,7 @@ static void buildRspData(const STableMeta* pMeta, char* pData) { // Length pData += BitmapLen(numOfRows); for (int32_t i = 0; i < numOfRows; ++i) { - *(int32_t*)pData = pMeta->schema[i].bytes; + *(int32_t*)pData = getSchemaBytes(pMeta->schema + i); pData += sizeof(int32_t); } pColSizes[2] = sizeof(int32_t) * numOfRows; diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index 5894fc88cc..ee17de50e0 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -77,7 +77,7 @@ typedef struct STableDataBlocks { STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; bool cloned; - + int32_t createTbReqLen; SParsedDataColInfo boundColumnInfo; SRowBuilder rowBuilder; } STableDataBlocks; @@ -118,6 +118,7 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? dataBuf->pTableMeta->uid : dataBuf->pTableMeta->suid); pBlocks->uid = dataBuf->pTableMeta->uid; pBlocks->sversion = dataBuf->pTableMeta->sversion; + pBlocks->schemaLen = dataBuf->createTbReqLen; if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -136,7 +137,7 @@ void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks); + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d7eb31b6cd..d9cf0b39b5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -753,7 +753,7 @@ static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, S } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision, const SName* pName) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const SName* pName) { if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -763,9 +763,9 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pCxt->tags.boundColumns[i]]; - param.schema = pSchema; - CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); + SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 + param.schema = pTagSchema; + CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); @@ -791,6 +791,7 @@ static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, S if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } + pBackup->uid = tGenIdPI64(); return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); } @@ -833,7 +834,11 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_NK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision, &name)); + CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, &name)); + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_NK_RP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); + } return TSDB_CODE_SUCCESS; } @@ -1015,7 +1020,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { STableDataBlocks *dataBuf = NULL; CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq)); if (TK_NK_LP == sToken.type) { // pSql -> field1_name, ...) @@ -1046,7 +1051,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // merge according to vgId if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } return buildOutput(pCxt); } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index f70e514b5a..088b25d544 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -149,8 +149,28 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star return TSDB_CODE_SUCCESS; } +static int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { + int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq); + if (pBlocks->nAllocSize - pBlocks->size < len) { + pBlocks->nAllocSize += len + pBlocks->rowSize; + char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); + if (pTmp != NULL) { + pBlocks->pData = pTmp; + memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); + } else { + pBlocks->nAllocSize -= len + pBlocks->rowSize; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + char* pBuf = pBlocks->pData + pBlocks->size; + tSerializeSVCreateTbReq((void**)&pBuf, pCreateTbReq); + pBlocks->size += len; + pBlocks->createTbReqLen = len; + return TSDB_CODE_SUCCESS; +} + int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -163,6 +183,13 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3 return ret; } + if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctbCfg.pTag) { + ret = buildCreateTbMsg(*dataBlocks, pCreateTbReq); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + } + taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); if (pBlockList) { taosArrayPush(pBlockList, dataBlocks); @@ -294,7 +321,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey int32_t extendedRowSize = getExtendedRowSize(dataBuf); SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char * pBlockData = pBlocks->data; + char * pBlockData = pBlocks->data + pBlocks->schemaLen; int n = 0; while (n < nRows) { pBlkKeyTuple->skey = TD_ROW_KEY((STSRow *)pBlockData); @@ -340,44 +367,26 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey } // Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, int8_t schemaAttached, bool isRawPayload) { +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, bool isRawPayload) { // TODO: optimize this function, handle the case while binary is not presented STableMeta* pTableMeta = pTableDataBlock->pTableMeta; STableComInfo tinfo = getTableInfo(pTableMeta); SSchema* pSchema = getTableColumnSchema(pTableMeta); + int32_t nonDataLen = sizeof(SSubmitBlk) + pTableDataBlock->createTbReqLen; SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); - pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk); + memcpy(pDataBlock, pTableDataBlock->pData, nonDataLen); + pDataBlock = (char*)pDataBlock + nonDataLen; int32_t flen = 0; // original total length of row - - // schema needs to be included into the submit data block - if (schemaAttached) { - int32_t numOfCols = getNumOfColumns(pTableDataBlock->pTableMeta); - for(int32_t j = 0; j < numOfCols; ++j) { - STColumn* pCol = (STColumn*) pDataBlock; - pCol->colId = htons(pSchema[j].colId); - pCol->type = pSchema[j].type; - pCol->bytes = htons(pSchema[j].bytes); - pCol->offset = 0; - - pDataBlock = (char*)pDataBlock + sizeof(STColumn); + if (isRawPayload) { + for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { flen += TYPE_BYTES[pSchema[j].type]; } - - int32_t schemaSize = sizeof(STColumn) * numOfCols; - pBlock->schemaLen = schemaSize; - } else { - if (isRawPayload) { - for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { - flen += TYPE_BYTES[pSchema[j].type]; - } - } - pBlock->schemaLen = 0; } + pBlock->schemaLen = pTableDataBlock->createTbReqLen; - char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); + char* p = pTableDataBlock->pData + nonDataLen; pBlock->dataLen = 0; int32_t numOfRows = pBlock->numOfRows; @@ -414,7 +423,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB return pBlock->dataLen + pBlock->schemaLen; } -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { +int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks) { const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); @@ -429,7 +438,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); @@ -474,7 +483,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, isRawPayload); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index 90e2ba3db2..d292fcf8b0 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -62,7 +62,7 @@ protected: void dumpReslut() { SVnodeModifOpStmt* pStmt = getVnodeModifStmt(res_); size_t num = taosArrayGetSize(pStmt->pDataBlocks); - cout << "schemaAttache:" << (int32_t)pStmt->schemaAttache << ", payloadType:" << (int32_t)pStmt->payloadType << ", insertType:" << pStmt->insertType << ", numOfVgs:" << num << endl; + cout << "payloadType:" << (int32_t)pStmt->payloadType << ", insertType:" << pStmt->insertType << ", numOfVgs:" << num << endl; for (size_t i = 0; i < num; ++i) { SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, i); cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; @@ -81,7 +81,6 @@ protected: void checkReslut(int32_t numOfTables, int16_t numOfRows1, int16_t numOfRows2 = -1) { SVnodeModifOpStmt* pStmt = getVnodeModifStmt(res_); - ASSERT_EQ(pStmt->schemaAttache, 0); ASSERT_EQ(pStmt->payloadType, PAYLOAD_TYPE_KV); ASSERT_EQ(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); size_t num = taosArrayGetSize(pStmt->pDataBlocks); @@ -168,6 +167,18 @@ TEST_F(InsertTest, multiTableMultiRowTest) { checkReslut(2, 3, 2); } +// INSERT INTO +// tb1_name USING st1_name [(tag1_name, ...)] TAGS (tag1_value, ...) VALUES (field1_value, ...) +// tb2_name USING st2_name [(tag1_name, ...)] TAGS (tag1_value, ...) VALUES (field1_value, ...) +TEST_F(InsertTest, autoCreateTableTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 using st1 tags(1, 'wxy') values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 3); +} + TEST_F(InsertTest, toleranceTest) { setDatabase("root", "test"); From 29fd5ec11ecf901fa212b93c1d772b4a7a1d00e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 2 Apr 2022 15:42:45 +0800 Subject: [PATCH 6/8] [td-14393] fix bug. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 20 +++++++++++++++----- source/libs/executor/src/executorimpl.c | 5 +++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 102d6fde90..4a3e764b13 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2268,6 +2268,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); // find the start data block in file @@ -2288,6 +2289,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* int defaultRows = 4096;//TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; + bool ascTraverse = ASCENDING_TRAVERSE(pTsdbReadHandle->order); + while (true) { numOfBlocks = 0; tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); @@ -2300,8 +2303,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + if ((ascTraverse && win.skey > pTsdbReadHandle->window.ekey) || (!ascTraverse && win.ekey < pTsdbReadHandle->window.ekey)) { tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle, pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); @@ -2343,9 +2345,17 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* int32_t numOfRows = pBlock[j].numOfRows; pTableBlockInfo->totalRows += numOfRows; - if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows; - if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows; - if (numOfRows < defaultRows) pTableBlockInfo->numOfSmallBlocks+=1; + if (numOfRows > pTableBlockInfo->maxRows) { + pTableBlockInfo->maxRows = numOfRows; + } + + if (numOfRows < pTableBlockInfo->minRows) { + pTableBlockInfo->minRows = numOfRows; + } + + if (numOfRows < defaultRows) { + pTableBlockInfo->numOfSmallBlocks += 1; + } // int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; // SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); // blockInfo->numBlocksOfStep++; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 149af22249..8e7e3b6495 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4564,6 +4564,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; @@ -4585,8 +4586,8 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { // // tbufCloseWriter(&bw); - SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); - pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); +// SArray* g = GET_TABLEGROUP(pOperator->, 0); +// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); pOperator->status = OP_EXEC_DONE; return pBlock; From bf08e3482df22f080254506edbf866b3fe7bf8c1 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 2 Apr 2022 16:45:43 +0800 Subject: [PATCH 7/8] [add cases] --- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/db/create_all_options.sim | 486 ++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 tests/script/tsim/db/create_all_options.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index d8fbfce55f..b756976394 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -5,6 +5,7 @@ ./test.sh -f tsim/user/basic1.sim # ---- db +./test.sh -f tsim/db/create_all_options.sim ./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim new file mode 100644 index 0000000000..7f39474f4d --- /dev/null +++ b/tests/script/tsim/db/create_all_options.sim @@ -0,0 +1,486 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$loop_cnt = 0 +check_dnode_ready_1: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $data00 != 1 then + return -1 +endi +if $data01 != localhost:7100 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready_1 +endi +if $data14 != ready then + goto check_dnode_ready_1 +endi +if $data24 != ready then + goto check_dnode_ready_1 +endi + +print ============= create database with all options +#database_option: { +# | BLOCKS value [3~1000, default: 6] +# | CACHE value [default: 16] +# | CACHELAST value [0, 1, 2, 3, default: 0] +# | COMP [0 | 1 | 2, default: 2] +# | DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +# | FSYNC value [0 ~ 180000 ms, default: 3000] +# | MAXROWS value [200~10000, default: 4096] +# | MINROWS value [10~1000, default: 100] +# | KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] +# | PRECISION ['ms' | 'us' | 'ns', default: ms] +# | QUORUM value [1 | 2, default: 1] +# | REPLICA value [1 | 3, default: 1] +# | TTL value [1d ~ , default: 1] +# | WAL value [1 | 2, default: 1] +# | VGROUPS value [default: 2] +# | SINGLE_STABLE [0 | 1, default: ] +# | STREAM_MODE [0 | 1, default: ] +# +#$data0_db : name +#$data1_db : create_time +#$data2_db : vgroups +#$data3_db : ntables +#$data4_db : replica +#$data5_db : quorum +#$data6_db : days +#$data7_db : keep +#$data8_db : cache +#$data9_db : blocks +#$data10_db : minrows +#$data11_db : maxrows +#$data12_db : wal +#$data13_db : fsync +#$data14_db : comp +#$data15_db : cachelast +#$data16_db : precision + +print ====> create database db, with default +sql create database db +sql show databases +print rows: $rows +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $rows != 2 then + return -1 +endi +if $data0_db != db then # name + return -1 +endi +if $data2_db != 2 then # vgroups + return -1 +endi +if $data3_db != 0 then # ntables + return -1 +endi +if $data4_db != 1 then # replica + return -1 +endi +if $data5_db != 1 then # quorum + return -1 +endi +if $data6_db != 14400 then # days + return -1 +endi +if $data7_db != 5256000,5256000,5256000 then # keep + return -1 +endi +if $data8_db != 16 then # cache + return -1 +endi +if $data9_db != 6 then # blocks + return -1 +endi +if $data10_db != 100 then # minrows + return -1 +endi +if $data11_db != 4096 then # maxrows + return -1 +endi +if $data12_db != 1 then # wal + return -1 +endi +if $data13_db != 3000 then # fsync + return -1 +endi +if $data14_db != 2 then # comp + return -1 +endi +if $data15_db != 0 then # cachelast + return -1 +endi +if $data16_db != ms then # precision + return -1 +endi +sql drop database db + +print ====> BLOCKS value [3~1000, default: 6] +sql create database db BLOCKS 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data9_db != 3 then + return -1 +endi +sql drop database db + +sql create database db BLOCKS 1000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data9_db != 1000 then + return -1 +endi +sql drop database db +sql_error create database db BLOCKS 2 +sql_error create database db BLOCKS 0 +sql_error create database db BLOCKS -1 + +print ====> CACHE value [default: 16] +sql create database db CACHE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data8_db != 1 then + return -1 +endi +sql drop database db + +sql create database db CACHE 128 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data8_db != 128 then + return -1 +endi +sql drop database db + +print ====> CACHELAST value [0, 1, 2, 3, default: 0] +sql create database db CACHELAST 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 1 then + return -1 +endi +sql drop database db + +sql create database db CACHELAST 2 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 2 then + return -1 +endi +sql drop database db + +sql create database db CACHELAST 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 3 then + return -1 +endi +sql drop database db +sql_error create database db CACHELAST 4 +sql_error create database db CACHELAST -1 + +print ====> COMP [0 | 1 | 2, default: 2] +sql create database db COMP 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data14_db != 1 then + return -1 +endi +sql drop database db + +sql create database db COMP 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data14_db != 0 then + return -1 +endi +sql drop database db +sql_error create database db COMP 3 +sql_error create database db COMP -1 + +#print ====> DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +#print ====> KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] +#sql create database db DAYS 60m KEEP 60m +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 60 then +# return -1 +#endi +#if $data7_db != 60,60,60 then +# return -1 +#endi +#sql drop database db +#sql create database db DAYS 60m KEEP 1d +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 60 then +# return -1 +#endi +#if $data7_db != 1440,1440,1440 then +# return -1 +#endi +#sql create database db DAYS 3650d KEEP 365000d +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 5256000 then +# return -1 +#endi +#if $data7_db != 525600000,525600000,525600000 then +# return -1 +#endi +#sql drop database db +#sql_error create database db DAYS -59m +#sql_error create database db DAYS 59m +#sql_error create database db DAYS 5256001m +#sql_error create database db DAYS 3651d +#sql_error create database db KEEP -59m +#sql_error create database db KEEP 14399m +#sql_error create database db KEEP 525600001m +#sql_error create database db KEEP 365001d + +print ====> FSYNC value [0 ~ 180000 ms, default: 3000] +sql create database db FSYNC 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data13_db != 0 then + return -1 +endi +sql drop database db + +sql create database db FSYNC 180000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data13_db != 180000 then + return -1 +endi +sql drop database db +sql_error create database db FSYNC 180001 +sql_error create database db FSYNC -1 + +print ====> MAXROWS value [200~10000, default: 4096], MINROWS value [10~1000, default: 100] +sql create database db MAXROWS 10000 MINROWS 1000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data10_db != 1000 then + return -1 +endi +if $data11_db != 10000 then + return -1 +endi +sql drop database db + +sql create database db MAXROWS 200 MINROWS 10 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data10_db != 10 then + return -1 +endi +if $data11_db != 200 then + return -1 +endi +sql drop database db +sql_error create database db MAXROWS -1 +sql_error create database db MAXROWS 0 +sql_error create database db MAXROWS 199 +sql_error create database db MAXROWS 10001 +sql_error create database db MINROWS -1 +sql_error create database db MINROWS 0 +sql_error create database db MINROWS 9 +sql_error create database db MINROWS 1001 +sql_error create database db MAXROWS 500 MINROWS 1000 + +print ====> PRECISION ['ms' | 'us' | 'ns', default: ms] +sql create database db PRECISION 'us' +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data16_db != us then + return -1 +endi +sql drop database db + +sql create database db PRECISION 'ns' +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data16_db != ns then + return -1 +endi +sql drop database db +sql_error create database db PRECISION 'as' +sql_error create database db PRECISION -1 + +print ====> QUORUM value [1 | 2, default: 1] +#sql create database db QUORUM 2 +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data5_db != 2 then +# return -1 +#endi +#sql drop database db + +sql create database db QUORUM 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data5_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db QUORUM 3 +sql_error create database db QUORUM 0 +sql_error create database db QUORUM -1 + +print ====> REPLICA value [1 | 3, default: 1] +sql create database db REPLICA 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data4_db != 3 then + return -1 +endi +sql drop database db + +sql create database db REPLICA 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data4_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db REPLICA 2 +sql_error create database db REPLICA 0 +sql_error create database db REPLICA -1 +sql_error create database db REPLICA 4 + +print ====> TTL value [1d ~ , default: 1] +sql create database db TTL 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db TTL 10 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXX_db != 10 then +# return -1 +#endi +sql drop database db +sql_error create database db TTL 0 +sql_error create database db TTL -1 + +print ====> WAL value [1 | 2, default: 1] +sql create database db WAL 2 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data12_db != 2 then + return -1 +endi +sql drop database db + +sql create database db WAL 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data12_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db WAL 3 +sql_error create database db WAL -1 +sql_error create database db WAL 0 + +print ====> VGROUPS value [1~4096, default: 2] +sql create database db VGROUPS 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data2_db != 1 then + return -1 +endi +sql drop database db + +sql create database db VGROUPS 16 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data2_db != 16 then + return -1 +endi +sql drop database db +sql_error create database db VGROUPS 4097 +sql_error create database db VGROUPS -1 +sql_error create database db VGROUPS 0 + +print ====> SINGLE_STABLE [0 | 1, default: ] +sql create database db SINGLE_STABLE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db SINGLE_STABLE 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXXX_db != 0 then +# return -1 +#endi +sql drop database db +sql_error create database db SINGLE_STABLE 2 +sql_error create database db SINGLE_STABLE -1 + +print ====> STREAM_MODE [0 | 1, default: ] +sql create database db STREAM_MODE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db STREAM_MODE 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXX_db != 0 then +# return -1 +#endi +sql drop database db +sql_error create database db STREAM_MODE 2 +sql_error create database db STREAM_MODE -1 + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT From 6f77eeccce1dca6df5e23e2d94cfdad14b3bb2fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 2 Apr 2022 16:59:02 +0800 Subject: [PATCH 8/8] [td-13039] support stddev. --- source/libs/function/inc/builtinsimpl.h | 5 + source/libs/function/src/builtins.c | 7 +- source/libs/function/src/builtinsimpl.c | 226 ++++++++++++++++++++++-- 3 files changed, 224 insertions(+), 14 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index f7fccb29f7..ab7bfc7767 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -38,9 +38,14 @@ void minFunction(SqlFunctionCtx* pCtx); void maxFunction(SqlFunctionCtx *pCtx); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); void stddevFunction(SqlFunctionCtx* pCtx); void stddevFinalize(SqlFunctionCtx* pCtx); +bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +void percentileFunction(SqlFunctionCtx *pCtx); + bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); void firstFunction(SqlFunctionCtx *pCtx); void lastFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 348c7a4368..909eac9532 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -68,9 +68,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .classification = FUNC_MGT_AGG_FUNC, .checkFunc = stubCheckAndGetResultType, .getEnvFunc = getStddevFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize }, { .name = "percentile", @@ -434,6 +434,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } + case FUNCTION_TYPE_STDDEV: case FUNCTION_TYPE_SIN: case FUNCTION_TYPE_COS: case FUNCTION_TYPE_TAN: diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 610e5a0bb2..0dc2989f77 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,6 +14,7 @@ */ #include "builtinsimpl.h" +#include "tpercentile.h" #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" @@ -453,6 +454,7 @@ bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { } typedef struct SStddevRes { + double result; int64_t count; union {double quadraticDSum; int64_t quadraticISum;}; union {double dsum; int64_t isum;}; @@ -463,38 +465,129 @@ bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } +bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SStddevRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); + memset(pRes, 0, sizeof(SStddevRes)); + return true; +} + void stddevFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; - int32_t type = pInput->pData[0]->info.type; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; + int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); -// } else { // computing based on the true data block - SColumnInfoData* pCol = pInput->pData[0]; + // computing based on the true data block + SColumnInfoData* pCol = pInput->pData[0]; - int32_t start = pInput->startRowIndex; - int32_t numOfRows = pInput->numOfRows; + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; - switch(type) { - case TSDB_DATA_TYPE_INT: { - int32_t* plist = (int32_t*)pCol->pData; + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* plist = (int8_t*)pCol->pData; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; } + numOfElem += 1; pStddevRes->count += 1; - pStddevRes->isum += plist[i]; + pStddevRes->isum += plist[i]; pStddevRes->quadraticISum += plist[i] * plist[i]; } + + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* plist = (int16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; } break; } + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t* plist = (int64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + float* plist = (float*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* plist = (double*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; + } + break; + } + + default: + break; + } + // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); } @@ -503,11 +596,122 @@ void stddevFinalize(SqlFunctionCtx* pCtx) { functionFinalize(pCtx); SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - double res = pStddevRes->quadraticISum/pStddevRes->count - (pStddevRes->isum / pStddevRes->count) * (pStddevRes->isum / pStddevRes->count); + double avg = pStddevRes->isum / ((double) pStddevRes->count); + pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg); } +typedef struct SPercentileInfo { + tMemBucket *pMemBucket; + int32_t stage; + double minval; + double maxval; + int64_t numOfElems; +} SPercentileInfo; +bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SPercentileInfo); + return true; +} +bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + // in the first round, get the min-max value of all involved data + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); + SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); + pInfo->numOfElems = 0; + + return true; +} + +void percentileFunction(SqlFunctionCtx *pCtx) { + int32_t notNullElems = 0; +#if 0 + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + pInfo->stage += 1; + + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + return; + } else { + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } + } + + // the first stage, only acquire the min/max value + if (pInfo->stage == 0) { + if (pCtx->preAggVals.isSet) { + double tmin = 0.0, tmax = 0.0; + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_FLOAT_TYPE(pCtx->inputType)) { + tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); + tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max); + } else { + assert(true); + } + + if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { + SET_DOUBLE_VAL(&pInfo->minval, tmin); + } + + if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { + SET_DOUBLE_VAL(&pInfo->maxval, tmax); + } + + pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); + } else { + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + double v = 0; + GET_TYPED_DATA(v, double, pCtx->inputType, data); + + if (v < GET_DOUBLE_VAL(&pInfo->minval)) { + SET_DOUBLE_VAL(&pInfo->minval, v); + } + + if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { + SET_DOUBLE_VAL(&pInfo->maxval, v); + } + + pInfo->numOfElems += 1; + } + } + + return; + } + + // the second stage, calculate the true percentile value + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + notNullElems += 1; + tMemBucketPut(pInfo->pMemBucket, data, 1); + } + + SET_VAL(pCtx, notNullElems, 1); + pResInfo->hasResult = DATA_SET_FLAG; +#endif + +} bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0);