From f520de0dab9a2924091eba27f8e0be411f1f1830 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 May 2023 10:07:01 +0800 Subject: [PATCH 01/25] refactor: adjust the module dependency. --- source/dnode/vnode/src/tq/tqRestore.c | 5 +++-- source/libs/executor/CMakeLists.txt | 3 +-- source/libs/stream/src/streamExec.c | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index ff9f95d5fa..6e3036a66b 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -61,9 +61,10 @@ static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { // seek the stored version and extract data from WAL int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); if (pTask->chkInfo.currentVer < firstVer) { + tqWarn("vgId:%d s-task:%s ver:%"PRId64" earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId, + pTask->id.idStr, pTask->chkInfo.currentVer, firstVer, firstVer); + pTask->chkInfo.currentVer = firstVer; - tqWarn("vgId:%d s-task:%s ver earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId, - pTask->id.idStr, firstVer, pTask->chkInfo.currentVer); // todo need retry if failed int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 8b3d04e32c..4adff71d85 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,10 +1,9 @@ aux_source_directory(src EXECUTOR_SRC) -#add_library(executor ${EXECUTOR_SRC}) add_library(executor STATIC ${EXECUTOR_SRC}) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes index stream + PRIVATE os util common function parser planner qcom vnode scalar nodes index ) target_include_directories( diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 55474541ed..637d661343 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -163,7 +163,6 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t code = 0; ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - void* exec = pTask->exec.pExecutor; qSetStreamOpOpen(exec); @@ -404,7 +403,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { { // set input - void* pExecutor = pTask->exec.pExecutor; + void* pExecutor = pTask->exec.pExecutor; const SStreamQueueItem* pItem = pInput; if (pItem->type == STREAM_INPUT__GET_RES) { From c15b1e35203aa9e47809b69f40d542a0860d9712 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 23 May 2023 11:46:02 +0800 Subject: [PATCH 02/25] opt compile --- include/libs/index/index.h | 1 + include/libs/scalar/filter.h | 10 ++++++ source/dnode/vnode/inc/vnode.h | 47 ++++++++++--------------- source/dnode/vnode/src/meta/metaQuery.c | 27 +++++++++----- source/libs/executor/src/executil.c | 40 +++++++++++---------- source/libs/index/CMakeLists.txt | 1 - source/libs/index/src/indexFilter.c | 4 +-- 7 files changed, 71 insertions(+), 59 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 0d31ca2f68..207197ccda 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -212,6 +212,7 @@ typedef struct SIndexMetaArg { void* idx; void* ivtIdx; uint64_t suid; + int (*metaFilterFunc)(void* metaEx, void* param, SArray* result); } SIndexMetaArg; typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index f20ba287de..7c5ca628cd 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -55,6 +55,16 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, SNode **pOtherCond); +typedef struct SMetaFltParam { + tb_uid_t suid; + int16_t cid; + int16_t type; + void *val; + bool reverse; + bool equal; + int (*filterFunc)(void *a, void *b, int16_t type); +} SMetaFltParam; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 18fa893fa4..60e36fae0e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -26,6 +26,7 @@ #include "tfs.h" #include "wal.h" +#include "filter.h" #include "tcommon.h" #include "tfs.h" #include "tgrant.h" @@ -125,35 +126,24 @@ int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); -int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); -int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, - int32_t payloadLen); +int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); +int32_t metaGetCachedTbGroup(SMeta *pMeta, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); +int32_t metaPutTbGroupToCache(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen); -int64_t metaGetTbNum(SMeta *pMeta); -int64_t metaGetNtbNum(SMeta *pMeta); +int64_t metaGetTbNum(SMeta *pMeta); +int64_t metaGetNtbNum(SMeta *pMeta); typedef struct { int64_t uid; int64_t ctbNum; } SMetaStbStats; int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); -typedef struct SMetaFltParam { - tb_uid_t suid; - int16_t cid; - int16_t type; - void *val; - bool reverse; - bool equal; - int (*filterFunc)(void *a, void *b, int16_t type); - -} SMetaFltParam; - // TODO, refactor later -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); +int32_t metaFilterTableIds(void *pMeta, void *param, SArray *results); +int32_t metaFilterCreateTime(void *pMeta, void *parm, SArray *pUids); +int32_t metaFilterTableName(void *pMeta, void *param, SArray *pUids); +int32_t metaFilterTtl(void *pMeta, void *param, SArray *pUids); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; @@ -183,7 +173,8 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST 0x8 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, + SHashObj **pIgnoreTables); int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); void tsdbReaderClose(STsdbReader *pReader); @@ -198,9 +189,9 @@ void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); -int64_t tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); +int64_t tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); +int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, @@ -265,13 +256,13 @@ int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); -bool tqNextBlockInWal(STqReader* pReader, const char* idstr); -bool tqNextBlockImpl(STqReader *pReader, const char* idstr); +bool tqNextBlockInWal(STqReader *pReader, const char *idstr); +bool tqNextBlockImpl(STqReader *pReader, const char *idstr); -int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id); +int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr); +int32_t tqRetrieveDataBlock(STqReader *pReader, const char *idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de3..8e914d6dab 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1083,8 +1083,10 @@ typedef struct { int32_t vLen; } SIdxCursor; -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; +int32_t metaFilterCreateTime(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; SIdxCursor *pCursor = NULL; pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); @@ -1141,9 +1143,11 @@ END: return ret; } -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTableName(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STagIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1206,9 +1210,11 @@ END: return ret; } -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTtl(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STtlIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1235,7 +1241,10 @@ END: // impl later return 0; } -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { +int32_t metaFilterTableIds(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + SMetaEntry oStbEntry = {0}; int32_t ret = -1; char *buf = NULL; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fa46715c22..82eeb4b1bd 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -435,7 +435,6 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { taosMemoryFree(payload); } - static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_CTX* pContext) { char* payload = NULL; int32_t len = 0; @@ -453,8 +452,8 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C taosMemoryFree(payload); } - -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) { +int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, + uint8_t* digest) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -492,18 +491,19 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis if (tsTagFilterCache) { SNodeListNode* listNode = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); listNode->pNodeList = group; - genTbGroupDigest((SNode *)listNode, digest, &context); + genTbGroupDigest((SNode*)listNode, digest, &context); nodesFree(listNode); - + metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); if (tableList) { taosArrayDestroy(pTableListInfo->pTableList); pTableListInfo->pTableList = tableList; - qDebug("retrieve tb group list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); + qDebug("retrieve tb group list from cache, numOfTables:%d", + (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); goto end; } } - + pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); @@ -632,9 +632,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); - metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); + metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, + taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); } - + // int64_t st2 = taosGetTimestampUs(); // qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); @@ -839,7 +840,6 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis return -1; } - static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle) { SSDataBlock* pResBlock = createDataBlock(); @@ -1096,8 +1096,11 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { void* pIndex = tsdbGetIvtIdx(metaHandle); - SIndexMetaArg metaArg = { - .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; + SIndexMetaArg metaArg = {.metaEx = metaHandle, + .idx = tsdbGetIdx(metaHandle), + .ivtIdx = pIndex, + .suid = pScanNode->uid, + .metaFilterFunc = metaFilterTableIds}; status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); @@ -1514,7 +1517,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_OUT_OF_MEMORY; } - SHashObj *pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) { @@ -1841,9 +1844,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { - return pTableList->idInfo.suid; -} +uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->idInfo.suid; } STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { @@ -2033,8 +2034,8 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TDB_CODE_SUCCESS; } -int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, - bool groupSort, uint8_t *digest) { +int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, + SNodeList* group, bool groupSort, uint8_t* digest) { int32_t code = TSDB_CODE_SUCCESS; bool groupByTbname = groupbyTbname(group); @@ -2086,7 +2087,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags } uint8_t digest[17] = {0}; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); + int32_t code = + getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); return code; diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 0c2ce37c40..6f3f48610c 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,7 +12,6 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common - PUBLIC vnode PUBLIC nodes PUBLIC scalar PUBLIC function diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 02ed0d2d05..fc064b762f 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "filter.h" #include "index.h" #include "indexComm.h" #include "indexInt.h" @@ -20,7 +21,6 @@ #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" -#include "vnode.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) @@ -659,7 +659,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP } else { if (sifSetFltParam(left, right, &typedata, ¶m) != 0) return -1; } - ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); + ret = arg->metaFilterFunc(arg->metaEx, ¶m, output->result); } return ret; } From e67b532d8a4f8ebaf47b34381e1279f1379a41f3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:29:23 +0800 Subject: [PATCH 03/25] refactor: refactor the module dependency. --- include/common/tcommon.h | 7 + include/libs/executor/executor.h | 7 +- include/libs/executor/storageapi.h | 576 ++++++++++++++++++ include/libs/index/index.h | 31 + include/libs/stream/streamState.h | 82 ++- include/libs/stream/tstreamFileState.h | 9 +- include/libs/stream/tstreamUpdate.h | 26 +- source/dnode/qnode/CMakeLists.txt | 2 + source/dnode/qnode/src/qnode.c | 1 - source/dnode/snode/CMakeLists.txt | 1 + source/dnode/vnode/inc/vnode.h | 186 +++--- source/dnode/vnode/src/inc/tsdb.h | 3 - source/dnode/vnode/src/meta/metaQuery.c | 12 +- source/dnode/vnode/src/meta/metaSnapshot.c | 30 +- source/dnode/vnode/src/tq/tqRead.c | 25 +- source/libs/executor/CMakeLists.txt | 2 +- source/libs/executor/inc/dataSinkInt.h | 4 +- source/libs/executor/inc/executil.h | 12 +- source/libs/executor/inc/executorInt.h | 51 +- source/libs/executor/inc/operator.h | 2 +- source/libs/executor/inc/querytask.h | 28 +- source/libs/executor/src/aggregateoperator.c | 3 +- source/libs/executor/src/cachescanoperator.c | 22 +- source/libs/executor/src/dataInserter.c | 4 +- source/libs/executor/src/executil.c | 124 ++-- source/libs/executor/src/executor.c | 67 +- source/libs/executor/src/executorInt.c | 141 +---- source/libs/executor/src/filloperator.c | 85 +-- source/libs/executor/src/groupoperator.c | 27 +- source/libs/executor/src/operator.c | 14 +- source/libs/executor/src/projectoperator.c | 1 + source/libs/executor/src/querytask.c | 18 +- source/libs/executor/src/scanoperator.c | 354 +++++------ source/libs/executor/src/sysscanoperator.c | 191 +++--- source/libs/executor/src/timesliceoperator.c | 1 + source/libs/executor/src/timewindowoperator.c | 154 +++-- source/libs/function/CMakeLists.txt | 1 - source/libs/function/src/builtinsimpl.c | 12 +- source/libs/index/CMakeLists.txt | 2 +- source/libs/index/src/indexFilter.c | 5 +- source/libs/scalar/CMakeLists.txt | 1 + source/libs/stream/inc/streamBackendRocksdb.h | 1 - source/libs/stream/src/streamBackendRocksdb.c | 18 +- source/libs/stream/src/tstreamFileState.c | 1 + 44 files changed, 1477 insertions(+), 867 deletions(-) create mode 100644 include/libs/executor/storageapi.h diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 520a8e9c2c..2f93f8c3e3 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -37,6 +37,13 @@ extern "C" { ) // clang-format on +typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); + +typedef struct STableKeyInfo { + uint64_t uid; + uint64_t groupId; +} STableKeyInfo; + typedef struct SWinKey { uint64_t groupId; TSKEY ts; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 1fb00e743f..7f86b8f5e2 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -23,6 +23,7 @@ extern "C" { #include "query.h" #include "tcommon.h" #include "tmsgcb.h" +#include "storageapi.h" typedef void* qTaskInfo_t; typedef void* DataSinkHandle; @@ -51,10 +52,10 @@ typedef struct { bool initTableReader; bool initTqReader; int32_t numOfVgroups; + void* sContext; // SSnapContext* - void* sContext; // SSnapContext* - - void* pStateBackend; + void* pStateBackend; + struct SStorageAPI api; } SReadHandle; // in queue mode, data streams are seperated by msg diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h new file mode 100644 index 0000000000..9b53f11ad9 --- /dev/null +++ b/include/libs/executor/storageapi.h @@ -0,0 +1,576 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_STORAGEAPI_H +#define TDENGINE_STORAGEAPI_H + +#include "tsimplehash.h" +#include "tscalablebf.h" +//#include "tdb.h" +#include "taosdef.h" +#include "tmsg.h" +#include "tcommon.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define TIMEWINDOW_RANGE_CONTAINED 1 +#define TIMEWINDOW_RANGE_EXTERNAL 2 + +#define CACHESCAN_RETRIEVE_TYPE_ALL 0x1 +#define CACHESCAN_RETRIEVE_TYPE_SINGLE 0x2 +#define CACHESCAN_RETRIEVE_LAST_ROW 0x4 +#define CACHESCAN_RETRIEVE_LAST 0x8 + +typedef struct SMeta SMeta; + +typedef struct SMetaEntry { + int64_t version; + int8_t type; + int8_t flags; // TODO: need refactor? + tb_uid_t uid; + char * name; + union { + struct { + SSchemaWrapper schemaRow; + SSchemaWrapper schemaTag; + SRSmaParam rsmaParam; + } stbEntry; + struct { + int64_t ctime; + int32_t ttlDays; + int32_t commentLen; + char * comment; + tb_uid_t suid; + uint8_t *pTags; + } ctbEntry; + struct { + int64_t ctime; + int32_t ttlDays; + int32_t commentLen; + char * comment; + int32_t ncid; // next column id + SSchemaWrapper schemaRow; + } ntbEntry; + struct { + STSma *tsma; + } smaEntry; + }; + + uint8_t *pBuf; +} SMetaEntry; + +// int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); +// int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, +// SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); +// int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, +// SArray *pTableUids); +// void *tsdbCacherowsReaderClose(void *pReader); +// int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); + +// int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, +// SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); +// int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); +// void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); +// void tsdbReaderClose(STsdbReader *pReader); +// int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); +// int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +// void tsdbReleaseDataBlock(STsdbReader *pReader); +// SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +// int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); +// int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +// int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); +// void *tsdbGetIdx(void *pMeta); +// void *tsdbGetIvtIdx(void *pMeta); +// uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); +// void tsdbReaderSetCloseFlag(STsdbReader *pReader); +// int64_t tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); + +// int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); +// int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); +// int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); +// void *vnodeGetIdx(void *pVnode); +// void *vnodeGetIvtIdx(void *pVnode); + +// void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +// void metaReaderReleaseLock(SMetaReader *pReader); +// void metaReaderClear(SMetaReader *pReader); +// int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +// int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +// int metaGetTableEntryByName(SMetaReader *pReader, const char *name); +// int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); +// int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); +// const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); +// int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); +// +// int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); +// int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +// int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); +// bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +// int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, +// bool *acquired); +// int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, +// int32_t payloadLen, double selectivityRatio); +// int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +// tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +// int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); +// int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, +// int32_t payloadLen); + +// tq +typedef struct SMetaTableInfo { + int64_t suid; + int64_t uid; + SSchemaWrapper *schema; + char tbName[TSDB_TABLE_NAME_LEN]; +} SMetaTableInfo; + +typedef struct SSnapContext { + SMeta * pMeta; + int64_t snapVersion; + struct TBC *pCur; + int64_t suid; + int8_t subType; + SHashObj * idVersion; + SHashObj * suidInfo; + SArray * idList; + int32_t index; + bool withMeta; + bool queryMeta; // true-get meta, false-get data +} SSnapContext; + +typedef struct { + int64_t uid; + int64_t ctbNum; +} SMetaStbStats; + +// void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +// int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +// int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); +// int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +// bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +// bool tqCurrentBlockConsumed(const STqReader* pReader); +// +// int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +// bool tqNextBlockInWal(STqReader* pReader, const char* idstr); +// bool tqNextBlockImpl(STqReader *pReader, const char* idstr); + +// int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); +// SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); +// int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); +// int32_t destroySnapContext(SSnapContext *ctx); + +// SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +// void metaCloseTbCursor(SMTbCursor *pTbCur); +// int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); +// int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); + +#define META_READER_NOLOCK 0x1 + +/*-------------------------------------------------new api format---------------------------------------------------*/ + +// typedef int32_t (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num); +// typedef void (*tsdbReaderSetId(STsdbReader *pReader, const char *idstr); +// typedef void (*tsdbReaderClose(STsdbReader *pReader); +// typedef int32_t (*tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); +// typedef int32_t (*tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +// typedef void (*tsdbReleaseDataBlock(STsdbReader *pReader); +// typedef SSDataBlock * (*tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +// typedef int32_t (*tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); +// typedef int32_t (*tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +// typedef int64_t (*tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); +// typedef void * (*tsdbGetIdx(void *pMeta); +// typedef void * (*tsdbGetIvtIdx(void *pMeta); +// typedef uint64_t (*tsdbGetReaderMaxVersion(STsdbReader *pReader); +// typedef void (*tsdbReaderSetCloseFlag(STsdbReader *pReader); +// typedef int64_t (*tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); + +typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, + int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, + const char *idstr, bool countOnly, SHashObj **pIgnoreTables); + +typedef struct SStoreDataReaderFn { + __store_reader_open_fn_t storeReaderOpen; + void (*storeReaderClose)(); + void (*setReaderId)(void *pReader, const char *pId); + void (*storeReaderSetTableList)(); + int32_t (*storeReaderNextDataBlock)(); + int32_t (*storeReaderRetrieveBlockSMA)(); + + SSDataBlock *(*storeReaderRetrieveDataBlock)(); + void (*storeReaderReleaseDataBlock)(); + + void (*storeReaderResetStatus)(); + void (*storeReaderGetDataBlockDistInfo)(); + void (*storeReaderGetNumOfInMemRows)(); + void (*storeReaderNotifyClosing)(); +} SStoreDataReaderFn; + +/** + * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); +int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, + SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); +int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, + SArray *pTableUids); +void *tsdbCacherowsReaderClose(void *pReader); + */ +typedef struct SStoreCachedDataReaderFn { + int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, + SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); + void *(*closeReader)(void *pReader); + int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, + SArray *pTableUidList); + void (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); +} SStoreCachedDataReaderFn; + +/*------------------------------------------------------------------------------------------------------------------*/ +/* + * +void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); +int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +bool tqCurrentBlockConsumed(const STqReader* pReader); + +int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +bool tqNextBlockInWal(STqReader* pReader, const char* idstr); +bool tqNextBlockImpl(STqReader *pReader, const char* idstr); + + int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr); +STqReader *tqReaderOpen(void *pVnode); +void tqCloseReader(STqReader *); + +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); +SWalReader* tqGetWalReader(STqReader* pReader); +int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); +*/ +// todo rename +typedef struct SStoreTqReaderFn { + void *(*tqReaderOpen)(); + void (*tqReaderClose)(); + + int32_t (*tqReaderSeek)(); + int32_t (*tqRetrieveBlock)(); + bool (*tqReaderNextBlockInWal)(); + bool (*tqNextBlockImpl)(); // todo remove it + + void (*tqReaderSetColIdList)(); + int32_t (*tqReaderSetTargetTableList)(); + + int32_t (*tqReaderAddTables)(); + int32_t (*tqReaderRemoveTables)(); + + bool (*tqReaderIsQueriedTable)(); + bool (*tqReaderCurrentBlockConsumed)(); + + struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it + void (*tqReaderRetrieveTaosXBlock)(); // todo remove it + + int32_t (*tqReaderSetSubmitMsg)(); // todo remove it + void (*tqReaderNextBlockFilterOut)(); +} SStoreTqReaderFn; + +typedef struct SStoreSnapshotFn { + /* + int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); + SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); + int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); + int32_t destroySnapContext(SSnapContext *ctx); + */ + int32_t (*storeCreateSnapshot)(); + void (*storeDestroySnapshot)(); + SMetaTableInfo (*storeSSGetTableInfo)(); + int32_t (*storeSSGetMetaInfo)(); +} SStoreSnapshotFn; + +/** +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderReleaseLock(SMetaReader *pReader); +void metaReaderClear(SMetaReader *pReader); +int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +int metaGetTableEntryByName(SMetaReader *pReader, const char *name); +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); +const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); +int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); + +int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); +int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, + bool *acquired); +int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen, double selectivityRatio); +int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); + */ +typedef struct SMetaReader { + int32_t flags; + void * pMeta; + SDecoder coder; + SMetaEntry me; + void * pBuf; + int32_t szBuf; + struct SStorageAPI *storageAPI; +} SMetaReader; + +typedef struct SStoreMetaReaderFn { + void (*initReader)(void *pReader, void *pMeta, int32_t flags); + void *(*clearReader)(); + + void (*readerReleaseLock)(); + + int32_t (*getTableEntryByUid)(); + int32_t (*getTableEntryByName)(); + int32_t (*readerGetEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); +} SStoreMetaReaderFn; + +typedef struct SStoreMetaFn { + /* +SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +void metaCloseTbCursor(SMTbCursor *pTbCur); +int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); + */ + void *(*openMetaCursor)(); + void (*closeMetaCursor)(); + int32_t (*cursorNext)(); + void (*cursorPrev)(); + + int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); + int32_t (*getTableTagsByUid)(); + const char *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it + + int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); + int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); + int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); + bool (*isTableExisted)(void *pVnode, uint64_t uid); + + /** + * int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen, double selectivityRatio); +int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); + */ + void (*getCachedTableList)(); + void (*putTableListIntoCache)(); + + /** + * + */ + void *(*storeGetIndexInfo)(); + void *(*storeGetInvertIndex)(); + void (*storeGetChildTableList)(); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + int32_t (*storeGetTableList)(); // vnodeGetStbIdList & vnodeGetAllTableList + void *storeGetVersionRange; + void *storeGetLastTimestamp; + + int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema + + // db name, vgId, numOfTables, numOfSTables + void (*storeGetNumOfChildTables)(); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void (*storeGetBasicInfo)(); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); + + int64_t (*getNumOfRowsInMem)(); + /** +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); +int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); + */ +} SStoreMetaFn; + +typedef struct STdbState { + void* rocksdb; + void** pHandle; + void* writeOpts; + void* readOpts; + void** cfOpts; + void* dbOpt; + struct SStreamTask* pOwner; + void* param; + void* env; + SListNode* pComparNode; + void* pBackendHandle; + char idstr[64]; + void* compactFactory; + + void* db; + void* pStateDb; + void* pFuncStateDb; + void* pFillStateDb; // todo refactor + void* pSessionStateDb; + void* pParNameDb; + void* pParTagDb; + void* txn; +} STdbState; + +// incremental state storage +typedef struct { + STdbState* pTdbState; + struct SStreamFileState* pFileState; + int32_t number; + SSHashObj* parNameMap; + int64_t checkPointId; + int32_t taskId; + int64_t streamId; +} SStreamState; + +typedef struct SUpdateInfo { + SArray *pTsBuckets; + uint64_t numBuckets; + SArray *pTsSBFs; + uint64_t numSBFs; + int64_t interval; + int64_t watermark; + TSKEY minTS; + SScalableBf *pCloseWinSBF; + SHashObj *pMap; + uint64_t maxDataVersion; +} SUpdateInfo; + +typedef struct { + void* iter; + void* snapshot; + void* readOpt; + void* db; +// rocksdb_iterator_t* iter; +// rocksdb_snapshot_t* snapshot; +// rocksdb_readoptions_t* readOpt; +// rocksdb_t* db; + + void* pCur; + int64_t number; +} SStreamStateCur; + +typedef TSKEY (*GetTsFun)(void*); + +typedef struct SStateStore { + int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); + int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal); + + int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal); + void (*streamStateFreeVal)(void* val); + + int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); + int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); + int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); + int32_t (*streamStateClear)(SStreamState* pState); + void (*streamStateSetNumber)(SStreamState* pState, int32_t number); + int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); + int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); + + int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); + + int32_t (*streamStateCurNext)(SStreamState* pState, void* pCur); + int32_t (*streamStateCurPrev)(SStreamState* pState, void* pCur); + + void* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); + void* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); + void* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); + void* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); + void (*streamStateFreeCur)(void* pCur); + + int32_t (*streamStateGetGroupKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + int32_t (*streamStateGetKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + + int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); + int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); + int32_t (*streamStateSessionClear)(SStreamState* pState); + int32_t (*streamStateSessionGetKVByCur)(void* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); + int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionGetKeyByRange)(void* pState, const SSessionKey* range, SSessionKey* curKey); + + void* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); + TSKEY (*updateInfoFillBlockData)(void *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(void *pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(void *pInfo, int64_t tbUid); + void (*updateInfoDestroy)(void *pInfo); + + SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark); + void (*updateInfoAddCloseWindowSBF)(void *pInfo); + void (*updateInfoDestoryColseWinSBF)(void *pInfo); + int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const void *pInfo); + int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, void *pInfo); + + void* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); + void* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); + void* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); + + void* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, + void* pFile, TSKEY delMark); + + void (*streamFileStateDestroy)(void* pFileState); + void (*streamFileStateClear)(void* pFileState); + bool (*needClearDiskBuff)(void* pFileState); + + SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); + void (*streamStateClose)(SStreamState* pState, bool remove); + int32_t (*streamStateBegin)(SStreamState* pState); + int32_t (*streamStateCommit)(SStreamState* pState); + void (*streamStateDestroy)(SStreamState* pState, bool remove); + int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); +} SStateStore; + +typedef struct SStorageAPI { + SStoreMetaFn metaFn; // todo: refactor + SStoreDataReaderFn storeReader; + SStoreMetaReaderFn metaReaderFn; + SStoreCachedDataReaderFn cacheFn; + SStoreSnapshotFn snapshotFn; + SStoreTqReaderFn tqReaderFn; + SStateStore stateStore; +} SStorageAPI; + +typedef struct SMTbCursor { + struct TBC *pDbc; + void * pKey; + void * pVal; + int32_t kLen; + int32_t vLen; + SMetaReader mr; +} SMTbCursor; + +typedef struct SRowBuffPos { + void* pRowBuff; + void* pKey; + bool beFlushed; + bool beUsed; +} SRowBuffPos; + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_STORAGEAPI_H diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 0d31ca2f68..93e4f18ae9 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -21,6 +21,7 @@ #include "taoserror.h" #include "tarray.h" #include "tglobal.h" +#include "tsdstorage.h" #ifdef __cplusplus extern "C" { @@ -231,6 +232,36 @@ void indexInit(int32_t threads); */ void indexCleanup(); +/** + * the underlying storage module must implement this API to employ the index functions. + * @param pMeta + * @param param + * @param results + * @return + */ + +typedef struct SMetaFltParam { + uint64_t suid; + int16_t cid; + int16_t type; + void *val; + bool reverse; + bool equal; + int (*filterFunc)(void *a, void *b, int16_t type); +} SMetaFltParam; + +typedef struct SStoreAPI { + int32_t (*metaFilterTableIds)(); + int32_t (*metaFilterCreateTime)(); + int32_t (*metaFilterTableName)(); + int32_t (*metaFilterTtl)(); +} SStoreAPI; + +//int32_t metaFilterTableIds(void *pMeta, SMetaFltParam *param, SArray *results); +//int32_t metaFilterCreateTime(void *pMeta, SMetaFltParam *parm, SArray *pUids); +//int32_t metaFilterTableName(void *pMeta, SMetaFltParam *param, SArray *pUids); +//int32_t metaFilterTtl(void *pMeta, SMetaFltParam *param, SArray *pUids); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 1cc61ec072..f9fffef7c2 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -27,47 +27,37 @@ extern "C" { #endif +#include "storageapi.h" + // void* streamBackendInit(const char* path); // void streamBackendCleanup(void* arg); // SListNode* streamBackendAddCompare(void* backend, void* arg); // void streamBackendDelCompare(void* backend, void* arg); -typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); -typedef struct STdbState { - rocksdb_t* rocksdb; - rocksdb_column_family_handle_t** pHandle; - rocksdb_writeoptions_t* writeOpts; - rocksdb_readoptions_t* readOpts; - rocksdb_options_t** cfOpts; - rocksdb_options_t* dbOpt; - struct SStreamTask* pOwner; - void* param; - void* env; - SListNode* pComparNode; - void* pBackendHandle; - char idstr[64]; - void* compactFactory; - - TDB* db; - TTB* pStateDb; - TTB* pFuncStateDb; - TTB* pFillStateDb; // todo refactor - TTB* pSessionStateDb; - TTB* pParNameDb; - TTB* pParTagDb; - TXN* txn; -} STdbState; - -// incremental state storage -typedef struct { - STdbState* pTdbState; - SStreamFileState* pFileState; - int32_t number; - SSHashObj* parNameMap; - int64_t checkPointId; - int32_t taskId; - int64_t streamId; -} SStreamState; +//typedef struct STdbState { +// rocksdb_t* rocksdb; +// rocksdb_column_family_handle_t** pHandle; +// rocksdb_writeoptions_t* writeOpts; +// rocksdb_readoptions_t* readOpts; +// rocksdb_options_t** cfOpts; +// rocksdb_options_t* dbOpt; +// struct SStreamTask* pOwner; +// void* param; +// void* env; +// SListNode* pComparNode; +// void* pBackendHandle; +// char idstr[64]; +// void* compactFactory; +// +// TDB* db; +// TTB* pStateDb; +// TTB* pFuncStateDb; +// TTB* pFillStateDb; // todo refactor +// TTB* pSessionStateDb; +// TTB* pParNameDb; +// TTB* pParTagDb; +// TXN* txn; +//} STdbState; SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState, bool remove); @@ -76,15 +66,15 @@ int32_t streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); -typedef struct { - rocksdb_iterator_t* iter; - rocksdb_snapshot_t* snapshot; - rocksdb_readoptions_t* readOpt; - rocksdb_t* db; - - TBC* pCur; - int64_t number; -} SStreamStateCur; +//typedef struct { +// rocksdb_iterator_t* iter; +// rocksdb_snapshot_t* snapshot; +// rocksdb_readoptions_t* readOpt; +// rocksdb_t* db; +// +// TBC* pCur; +// int64_t number; +//} SStreamStateCur; int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); @@ -119,7 +109,7 @@ int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); -void streamFreeVal(void* val); +void streamStateFreeVal(void* val); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 7124e2d251..0dbacf6c9f 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -21,23 +21,16 @@ #include "tarray.h" #include "tdef.h" #include "tlist.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { #endif typedef struct SStreamFileState SStreamFileState; -typedef struct SRowBuffPos { - void* pRowBuff; - void* pKey; - bool beFlushed; - bool beUsed; -} SRowBuffPos; typedef SList SStreamSnapshot; -typedef TSKEY (*GetTsFun)(void*); - SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark); void streamFileStateDestroy(SStreamFileState* pFileState); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 4678aa0bd9..bd5a3be8de 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -19,7 +19,7 @@ #include "tarray.h" #include "tcommon.h" #include "tmsg.h" -#include "tscalablebf.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -30,18 +30,18 @@ typedef struct SUpdateKey { TSKEY ts; } SUpdateKey; -typedef struct SUpdateInfo { - SArray *pTsBuckets; - uint64_t numBuckets; - SArray *pTsSBFs; - uint64_t numSBFs; - int64_t interval; - int64_t watermark; - TSKEY minTS; - SScalableBf *pCloseWinSBF; - SHashObj *pMap; - uint64_t maxDataVersion; -} SUpdateInfo; +//typedef struct SUpdateInfo { +// SArray *pTsBuckets; +// uint64_t numBuckets; +// SArray *pTsSBFs; +// uint64_t numSBFs; +// int64_t interval; +// int64_t watermark; +// TSKEY minTS; +// SScalableBf *pCloseWinSBF; +// SHashObj *pMap; +// uint64_t maxDataVersion; +//} SUpdateInfo; SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt index 5426cd55d3..13429d0cc1 100644 --- a/source/dnode/qnode/CMakeLists.txt +++ b/source/dnode/qnode/CMakeLists.txt @@ -14,4 +14,6 @@ target_link_libraries( PRIVATE qworker PRIVATE qcom PRIVATE executor + PRIVATE tdb + PRIVATE wal ) \ No newline at end of file diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 5efc714e95..3482355512 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -14,7 +14,6 @@ */ #include "executor.h" -#include "libs/function/function.h" #include "qndInt.h" #include "query.h" #include "qworker.h" diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 6f1e7f9593..1d8c80c422 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -14,4 +14,5 @@ target_link_libraries( PRIVATE util PRIVATE qcom PRIVATE stream + PRIVATE wal ) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 18fa893fa4..8afeb05716 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -33,6 +33,7 @@ #include "trow.h" #include "tdb.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -132,28 +133,28 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in int64_t metaGetTbNum(SMeta *pMeta); int64_t metaGetNtbNum(SMeta *pMeta); -typedef struct { - int64_t uid; - int64_t ctbNum; -} SMetaStbStats; +//typedef struct { +// int64_t uid; +// int64_t ctbNum; +//} SMetaStbStats; int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); -typedef struct SMetaFltParam { - tb_uid_t suid; - int16_t cid; - int16_t type; - void *val; - bool reverse; - bool equal; - int (*filterFunc)(void *a, void *b, int16_t type); - -} SMetaFltParam; +//typedef struct SMetaFltParam { +// tb_uid_t suid; +// int16_t cid; +// int16_t type; +// void *val; +// bool reverse; +// bool equal; +// int (*filterFunc)(void *a, void *b, int16_t type); +// +//} SMetaFltParam; // TODO, refactor later -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); +//int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); +//int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); +//int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); +//int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; @@ -213,32 +214,32 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); size_t tsdbCacheGetUsage(SVnode *pVnode); int32_t tsdbCacheGetElems(SVnode *pVnode); -// tq -typedef struct SMetaTableInfo { - int64_t suid; - int64_t uid; - SSchemaWrapper *schema; - char tbName[TSDB_TABLE_NAME_LEN]; -} SMetaTableInfo; +//// tq +//typedef struct SMetaTableInfo { +// int64_t suid; +// int64_t uid; +// SSchemaWrapper *schema; +// char tbName[TSDB_TABLE_NAME_LEN]; +//} SMetaTableInfo; typedef struct SIdInfo { int64_t version; int32_t index; } SIdInfo; -typedef struct SSnapContext { - SMeta *pMeta; - int64_t snapVersion; - TBC *pCur; - int64_t suid; - int8_t subType; - SHashObj *idVersion; - SHashObj *suidInfo; - SArray *idList; - int32_t index; - bool withMeta; - bool queryMeta; // true-get meta, false-get data -} SSnapContext; +//typedef struct SSnapContext { +// SMeta *pMeta; +// int64_t snapVersion; +// TBC *pCur; +// int64_t suid; +// int8_t subType; +// SHashObj *idVersion; +// SHashObj *suidInfo; +// SArray *idList; +// int32_t index; +// bool withMeta; +// bool queryMeta; // true-get meta, false-get data +//} SSnapContext; typedef struct STqReader { SPackedData msg; @@ -271,7 +272,7 @@ bool tqNextBlockImpl(STqReader *pReader, const char* idstr); int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr); +int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); @@ -350,67 +351,62 @@ struct SVnodeCfg { int32_t tsdbPageSize; }; -typedef struct { - uint64_t uid; - uint64_t groupId; -} STableKeyInfo; - #define TABLE_ROLLUP_ON ((int8_t)0x1) #define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) -struct SMetaEntry { - int64_t version; - int8_t type; - int8_t flags; // TODO: need refactor? - tb_uid_t uid; - char *name; - union { - struct { - SSchemaWrapper schemaRow; - SSchemaWrapper schemaTag; - SRSmaParam rsmaParam; - } stbEntry; - struct { - int64_t ctime; - int32_t ttlDays; - int32_t commentLen; - char *comment; - tb_uid_t suid; - uint8_t *pTags; - } ctbEntry; - struct { - int64_t ctime; - int32_t ttlDays; - int32_t commentLen; - char *comment; - int32_t ncid; // next column id - SSchemaWrapper schemaRow; - } ntbEntry; - struct { - STSma *tsma; - } smaEntry; - }; +//struct SMetaEntry { +// int64_t version; +// int8_t type; +// int8_t flags; // TODO: need refactor? +// tb_uid_t uid; +// char *name; +// union { +// struct { +// SSchemaWrapper schemaRow; +// SSchemaWrapper schemaTag; +// SRSmaParam rsmaParam; +// } stbEntry; +// struct { +// int64_t ctime; +// int32_t ttlDays; +// int32_t commentLen; +// char *comment; +// tb_uid_t suid; +// uint8_t *pTags; +// } ctbEntry; +// struct { +// int64_t ctime; +// int32_t ttlDays; +// int32_t commentLen; +// char *comment; +// int32_t ncid; // next column id +// SSchemaWrapper schemaRow; +// } ntbEntry; +// struct { +// STSma *tsma; +// } smaEntry; +// }; +// +// uint8_t *pBuf; +//}; - uint8_t *pBuf; -}; +//struct SMetaReader { +// int32_t flags; +// SMeta *pMeta; +// SDecoder coder; +// SMetaEntry me; +// void *pBuf; +// int32_t szBuf; +//}; -struct SMetaReader { - int32_t flags; - SMeta *pMeta; - SDecoder coder; - SMetaEntry me; - void *pBuf; - int32_t szBuf; -}; - -struct SMTbCursor { - TBC *pDbc; - void *pKey; - void *pVal; - int32_t kLen; - int32_t vLen; - SMetaReader mr; -}; +//struct SMTbCursor { +// TBC *pDbc; +// void *pKey; +// void *pVal; +// int32_t kLen; +// int32_t vLen; +// SMetaReader mr; +//}; #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b2bc9abf33..4bd16cf14f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -844,9 +844,6 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); -void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); -size_t tsdbCacheGetCapacity(SVnode *pVnode); - // int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); // ========== inline functions ========== diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de3..4cc4176fda 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -249,7 +249,7 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { SMetaReader *pReader = &mr; // query name.idx - if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(((SMeta*)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; metaReaderClear(&mr); return -1; @@ -293,9 +293,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { metaReaderInit(&pTbCur->mr, pMeta, 0); - tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); + tdbTbcOpen(pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); - tdbTbcMoveToFirst(pTbCur->pDbc); + tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); return pTbCur; } @@ -306,7 +306,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { tdbFree(pTbCur->pVal); metaReaderClear(&pTbCur->mr); if (pTbCur->pDbc) { - tdbTbcClose(pTbCur->pDbc); + tdbTbcClose((TBC *)pTbCur->pDbc); } taosMemoryFree(pTbCur); } @@ -318,7 +318,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { STbCfg tbCfg; for (;;) { - ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcNext((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -342,7 +342,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) { STbCfg tbCfg; for (;;) { - ret = tdbTbcPrev(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcPrev((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 707dd66e30..d3b6471282 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -222,29 +222,29 @@ static void destroySTableInfoForChildTable(void* data) { } static void MoveToSnapShotVersion(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); STbDbKey key = {.version = ctx->snapVersion, .uid = INT64_MAX}; int c = 0; - tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); + tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c); if (c < 0) { - tdbTbcMoveToPrev(ctx->pCur); + tdbTbcMoveToPrev((TBC*)ctx->pCur); } } static int32_t MoveToPosition(SSnapContext* ctx, int64_t ver, int64_t uid) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); STbDbKey key = {.version = ver, .uid = uid}; int c = 0; - tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); + tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c); return c; } static void MoveToFirst(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); - tdbTbcMoveToFirst(ctx->pCur); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); + tdbTbcMoveToFirst((TBC*)ctx->pCur); } static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) { @@ -291,7 +291,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t metaDebug("tmqsnap init snapVersion:%" PRIi64, ctx->snapVersion); MoveToFirst(ctx); while (1) { - int32_t ret = tdbTbcNext(ctx->pCur, &pKey, &kLen, &pVal, &vLen); + int32_t ret = tdbTbcNext((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; STbDbKey* tmp = (STbDbKey*)pKey; if (tmp->version > ctx->snapVersion) break; @@ -329,7 +329,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t MoveToSnapShotVersion(ctx); while (1) { - int32_t ret = tdbTbcPrev(ctx->pCur, &pKey, &kLen, &pVal, &vLen); + int32_t ret = tdbTbcPrev((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; STbDbKey* tmp = (STbDbKey*)pKey; @@ -378,7 +378,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t } int32_t destroySnapContext(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); + tdbTbcClose((TBC*)ctx->pCur); taosArrayDestroy(ctx->idList); taosHashCleanup(ctx->idVersion); taosHashCleanup(ctx->suidInfo); @@ -496,7 +496,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in metaDebug("tmqsnap get meta not exist uid:%" PRIi64 " version:%" PRIi64, *uid, idInfo->version); } - tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); + tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); SDecoder dc = {0}; SMetaEntry me = {0}; tDecoderInit(&dc, pVal, vLen); @@ -622,7 +622,7 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); continue; } - tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); + tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); SDecoder dc = {0}; SMetaEntry me = {0}; tDecoderInit(&dc, pVal, vLen); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index b756f99f32..84ead71fb5 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -394,8 +394,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) { SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) { - int32_t code = tqRetrieveDataBlock(pReader, NULL); - if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { return true; } } @@ -404,8 +405,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) { if (ret != NULL) { tqDebug("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver); - int32_t code = tqRetrieveDataBlock(pReader, NULL); - if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { return true; } } else { @@ -440,6 +442,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return 0; } +SWalReader* tqGetWalReader(STqReader* pReader) { + return pReader->pWalReader; +} + + bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { return false; @@ -592,7 +599,7 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol return code; } -int32_t tqRetrieveDataBlock(STqReader* pReader, const char* id) { +int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); @@ -1024,6 +1031,14 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { return 0; } +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) { + return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t)); +} + +bool tqCurrentBlockConsumed(const STqReader* pReader) { + return pReader->msg.msgStr == NULL; +} + int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 4adff71d85..d2c39aba74 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -3,7 +3,7 @@ aux_source_directory(src EXECUTOR_SRC) add_library(executor STATIC ${EXECUTOR_SRC}) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes index + PRIVATE os util common function parser planner qcom scalar nodes index wal tdb ) target_include_directories( diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 57a771b275..3255831476 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -22,13 +22,15 @@ extern "C" { #include "dataSinkMgt.h" #include "plannodes.h" +#include "storageapi.h" #include "tcommon.h" struct SDataSink; struct SDataSinkHandle; typedef struct SDataSinkManager { - SDataSinkMgtCfg cfg; + SDataSinkMgtCfg cfg; + SStorageAPI storeFn; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 896858ac7f..6dc359c502 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -12,17 +12,17 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_QUERYUTIL_H -#define TDENGINE_QUERYUTIL_H +#ifndef TDENGINE_EXECUTIL_H +#define TDENGINE_EXECUTIL_H #include "executor.h" #include "function.h" #include "nodes.h" #include "plannodes.h" +#include "storageapi.h" #include "tcommon.h" #include "tpagedbuf.h" #include "tsimplehash.h" -#include "vnode.h" #define T_LONG_JMP(_obj, _c) \ do { \ @@ -154,7 +154,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, SStorageAPI* pAPI); size_t getTableTagsBufLen(const SNodeList* pGroups); SArray* createSortInfo(SNodeList* pNodeList); @@ -178,8 +178,8 @@ void cleanupQueryTableDataCond(SQueryTableDataCond* pCond); int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); -int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI); void printDataBlock(SSDataBlock* pBlock, const char* flag); -#endif // TDENGINE_QUERYUTIL_H +#endif // TDENGINE_EXECUTIL_H diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 2daeb70260..57b3bc3292 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -38,13 +38,15 @@ extern "C" { #include "tlockfree.h" #include "tmsg.h" #include "tpagedbuf.h" -#include "tstream.h" -#include "tstreamUpdate.h" - -#include "vnode.h" +//#include "tstream.h" +//#include "tstreamUpdate.h" +#include "tlrucache.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct STsdbReader STsdbReader; +typedef struct STqReader STqReader; + #define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) #define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) #define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) @@ -206,6 +208,7 @@ typedef struct STableScanBase { SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. STableListInfo* pTableListInfo; + SStoreDataReaderFn readerAPI; } STableScanBase; typedef struct STableScanInfo { @@ -221,6 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; + SStoreDataReaderFn readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -280,6 +284,7 @@ typedef struct SStreamAggSupporter { int32_t stateKeySize; int16_t stateKeyType; SDiskbasedBuf* pResultBuf; + SStateStore stateStore; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -335,7 +340,7 @@ typedef struct SStreamScanInfo { STqReader* tqReader; uint64_t groupId; - SUpdateInfo* pUpdateInfo; + struct SUpdateInfo* pUpdateInfo; EStreamScanMode scanMode; struct SOperatorInfo* pStreamScanOp; @@ -366,15 +371,18 @@ typedef struct SStreamScanInfo { SSDataBlock* pCreateTbRes; int8_t igCheckUpdate; int8_t igExpired; - SStreamState* pState; + void* pState; //void + SStoreTqReaderFn readerFn; + SStateStore stateStore; } SStreamScanInfo; typedef struct { - SVnode* vnode; - SSDataBlock pRes; // result SSDataBlock - STsdbReader* dataReader; - SSnapContext* sContext; - STableListInfo* pTableListInfo; + struct SVnode* vnode; // todo remove this + SSDataBlock pRes; // result SSDataBlock + STsdbReader* dataReader; + struct SSnapContext* sContext; + SStorageAPI* pAPI; + STableListInfo* pTableListInfo; } SStreamRawScanInfo; typedef struct STableCountScanSupp { @@ -441,12 +449,13 @@ typedef struct SStreamIntervalOperatorInfo { bool isFinal; SArray* pChildren; int32_t numOfChild; - SStreamState* pState; + SStreamState* pState; // void SWinKey delKey; uint64_t numOfDatapack; SArray* pUpdated; SSHashObj* pUpdatedMap; int64_t dataVersion; + SStateStore statestore; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -543,6 +552,7 @@ typedef struct SStreamFillSupporter { int32_t rowSize; SSHashObj* pResMap; bool hasDelete; + SStorageAPI* pAPI; } SStreamFillSupporter; typedef struct SStreamFillOperatorInfo { @@ -636,7 +646,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -645,20 +655,17 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); bool groupbyTbname(SNodeList* pGroupList); -int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); -int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI); +int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, - SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); -int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); -int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); +void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 632b817a07..1d2685b8c6 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -156,7 +156,7 @@ void destroyOperator(SOperatorInfo* pOperator); SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); -int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI); int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); #ifdef __cplusplus diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 37c93fef5c..1c2ed6c076 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -56,20 +56,19 @@ typedef struct STaskStopInfo { } STaskStopInfo; typedef struct { - STqOffsetVal currentOffset; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int64_t snapshotVer; -// SPackedData submit; // todo remove it - SSchemaWrapper* schema; - char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor - int8_t recoverStep; - int8_t recoverScanFinished; - SQueryTableDataCond tableCond; - int64_t fillHistoryVer1; - int64_t fillHistoryVer2; - SStreamState* pState; - int64_t dataVersion; - int64_t checkPointId; + STqOffsetVal currentOffset; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int64_t snapshotVer; + SSchemaWrapper* schema; + char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor + int8_t recoverStep; + int8_t recoverScanFinished; + SQueryTableDataCond tableCond; + int64_t fillHistoryVer1; + int64_t fillHistoryVer2; + SStreamState* pState; + int64_t dataVersion; + int64_t checkPointId; } SStreamTaskInfo; struct SExecTaskInfo { @@ -92,6 +91,7 @@ struct SExecTaskInfo { SArray* pResultBlockList; // result block list STaskStopInfo stopInfo; SRWLatch lock; // secure the access of STableListInfo + SStorageAPI storageAPI; }; void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 5cd95d3311..918038635b 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -21,7 +21,7 @@ #include "tname.h" #include "executorInt.h" -#include "index.h" +#include "tsdstorage.h" #include "operator.h" #include "query.h" #include "querytask.h" @@ -30,6 +30,7 @@ #include "tglobal.h" #include "thash.h" #include "ttypes.h" +#include "index.h" typedef struct { bool hasAgg; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 96dac643a5..1b6c06b6c4 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "function.h" #include "os.h" +#include "function.h" #include "tname.h" #include "tdatablock.h" @@ -27,6 +27,8 @@ #include "thash.h" #include "ttypes.h" +#include "storageapi.h" + typedef struct SCacheRowsScanInfo { SSDataBlock* pRes; SReadHandle readHandle; @@ -102,9 +104,9 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); uint64_t suid = tableListGetSuid(pTableListInfo); - code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, - taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, suid, - &pInfo->pLastrowReader, pTaskInfo->id.str); + code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, + taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, + suid, &pInfo->pLastrowReader, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -172,7 +174,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pBufferredRes); taosArrayClear(pInfo->pUidList); - int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, + int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -239,7 +241,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } if (NULL == pInfo->pLastrowReader) { - code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, + code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -248,12 +250,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { continue; } } else { - tsdbReuseCacherowsReader(pInfo->pLastrowReader, pList, num); + pInfo->readHandle.api.cacheFn.reuseReader(pInfo->pLastrowReader, pList, num); } taosArrayClear(pInfo->pUidList); - code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds, + code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -287,7 +289,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); setOperatorCompleted(pOperator); return NULL; } @@ -305,7 +307,7 @@ void destroyCacheScanOperator(void* param) { tableListDestroy(pInfo->pTableList); if (pInfo->pLastrowReader != NULL) { - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); } cleanupExprSupp(&pInfo->pseudoExprSup); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index d31ac0bc51..aadbf1f1a9 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -17,6 +17,7 @@ #include "dataSinkMgt.h" #include "executorInt.h" #include "planner.h" +#include "storageapi.h" #include "tcompression.h" #include "tdatablock.h" #include "tglobal.h" @@ -428,8 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->explain = pInserterNode->explain; int64_t suid = 0; - int32_t code = - tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); + int32_t code = pManager->storeFn.metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fa46715c22..c742794498 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -27,6 +27,7 @@ #include "executorInt.h" #include "querytask.h" #include "tcompression.h" +#include "storageapi.h" typedef struct tagFilterAssist { SHashObj* colHash; @@ -41,13 +42,13 @@ typedef enum { } FilterCondType; static FilterCondType checkTagCond(SNode* cond); -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond); +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SStorageAPI* pAPI); +static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond, SStorageAPI* pStoreAPI); -static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, - SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr); +static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, + SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI); static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, - void* metaHandle); + void* pVnode, SStorageAPI* pStorageAPI); static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } @@ -262,7 +263,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { STagVal tagVal = {0}; tagVal.cid = pSColumnNode->colId; - const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); + const char* p = mr->storageAPI->metaFn.extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { @@ -301,14 +302,14 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI) { int32_t code = TSDB_CODE_SUCCESS; SMetaReader mr = {0}; - metaReaderInit(&mr, metaHandle, 0); - code = metaGetTableEntryByUidCache(&mr, info->uid); + pAPI->metaReaderFn.initReader(&mr, metaHandle, 0); + code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, info->uid); if (TSDB_CODE_SUCCESS != code) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); *pQualified = false; return TSDB_CODE_SUCCESS; @@ -317,7 +318,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, SNode* pTagCondTmp = nodesCloneNode(pTagCond); nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); SNode* pNew = NULL; code = scalarCalculateConstants(pTagCondTmp, &pNew); @@ -453,8 +454,8 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C taosMemoryFree(payload); } - -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) { +int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest, + SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -465,7 +466,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis int32_t rows = taosArrayGetSize(pTableListInfo->pTableList); if (rows == 0) { - return TDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } tagFilterAssist ctx = {0}; @@ -494,8 +495,8 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis listNode->pNodeList = group; genTbGroupDigest((SNode *)listNode, digest, &context); nodesFree(listNode); - - metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); + + pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); if (tableList) { taosArrayDestroy(pTableListInfo->pTableList); pTableListInfo->pTableList = tableList; @@ -511,14 +512,13 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis taosArrayPush(pUidTagList, &info); } - // int64_t stt = taosGetTimestampUs(); - code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTags(pVnode, pTableListInfo->idInfo.suid, pUidTagList); if (code != TSDB_CODE_SUCCESS) { goto end; } int32_t numOfTables = taosArrayGetSize(pUidTagList); - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI); if (pResBlock == NULL) { code = terrno; goto end; @@ -632,7 +632,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); - metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); + pAPI->metaFn.putTableListIntoCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); } // int64_t st2 = taosGetTimestampUs(); @@ -734,12 +734,12 @@ static FilterCondType checkTagCond(SNode* cond) { return FILTER_OTHER; } -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) { +static int32_t optimizeTbnameInCond(void* pVnode, int64_t suid, SArray* list, SNode* cond, SStorageAPI* pAPI) { int32_t ret = -1; int32_t ntype = nodeType(cond); if (ntype == QUERY_NODE_OPERATOR) { - ret = optimizeTbnameInCondImpl(metaHandle, list, cond); + ret = optimizeTbnameInCondImpl(pVnode, list, cond, pAPI); } if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { @@ -758,7 +758,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list SListCell* cell = pList->pHead; for (int i = 0; i < len; i++) { if (cell == NULL) break; - if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) { + if (optimizeTbnameInCondImpl(pVnode, list, cell->pNode, pAPI) == 0) { hasTbnameCond = true; break; } @@ -769,14 +769,14 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL); if (hasTbnameCond) { - ret = metaGetTableTagsByUids(metaHandle, suid, list); + ret = pAPI->metaFn.getTableTagsByUid(pVnode, suid, list); } return ret; } // only return uid that does not contained in pExistedUidList -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) { +static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, SNode* pTagCond, SStorageAPI* pStoreAPI) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { return -1; } @@ -813,9 +813,9 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis char* name = taosArrayGetP(pTbList, i); uint64_t uid = 0; - if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { + if (pStoreAPI->metaFn.getTableUidByName(pVnode, name, &uid) == 0) { ETableType tbType = TSDB_TABLE_MAX; - if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { + if (pStoreAPI->metaFn.getTableTypeByName(pVnode, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL}; taosArrayPush(pExistedUidList, &s); @@ -841,7 +841,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, - void* metaHandle) { + void* pVnode, SStorageAPI* pStorageAPI) { SSDataBlock* pResBlock = createDataBlock(); if (pResBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -876,7 +876,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa if (p1->name != NULL) { STR_TO_VARSTR(str, p1->name); } else { // name is not retrieved during filter - metaGetTableNameByUid(metaHandle, p1->uid, str); + pStorageAPI->metaFn.getTableNameByUid(pVnode, p1->uid, str); } colDataSetVal(pColInfo, i, str, false); @@ -889,7 +889,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa if (p1->pTagVal == NULL) { colDataSetNULL(pColInfo, i); } else { - const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); + const char* p = pStorageAPI->metaFn.extractTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { colDataSetNULL(pColInfo, i); @@ -949,13 +949,13 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) { } } -static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle, - SIdxFltStatus status) { +static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* pVnode, + SIdxFltStatus status, SStorageAPI* pAPI) { if (pTagCond == NULL) { return TSDB_CODE_SUCCESS; } - terrno = TDB_CODE_SUCCESS; + terrno = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; @@ -985,7 +985,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN FilterCondType condType = checkTagCond(pTagCond); - int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond); + int32_t filter = optimizeTbnameInCond(pVnode, pListInfo->idInfo.suid, pUidTagList, pTagCond, pAPI); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); @@ -998,9 +998,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; } else { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { - code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTagsByUid(pVnode, pListInfo->idInfo.suid, pUidTagList); } else { - code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTags(pVnode, pListInfo->idInfo.suid, pUidTagList); } if (code != TSDB_CODE_SUCCESS) { qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid); @@ -1014,7 +1014,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN goto end; } - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI); if (pResBlock == NULL) { code = terrno; goto end; @@ -1052,8 +1052,8 @@ end: return code; } -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, - STableListInfo* pListInfo, uint8_t* digest, const char* idstr) { +int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, + STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI) { int32_t code = TSDB_CODE_SUCCESS; size_t numOfTables = 0; @@ -1065,10 +1065,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { pListInfo->idInfo.uid = pScanNode->uid; - if (metaIsTableExist(metaHandle, pScanNode->uid)) { + if (pStorageAPI->metaFn.isTableExisted(pVnode, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1080,7 +1080,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, genTagFilterDigest(pTagCond, &context); bool acquired = false; - metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, + pStorageAPI->metaFn.getCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, &acquired); if (acquired) { digest[0] = 1; @@ -1091,26 +1091,28 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table - vnodeGetCtbIdList(pVnode, pScanNode->suid, pUidList); +// vnodeGetCtbIdList(); + pStorageAPI->metaFn.storeGetChildTableList(pVnode, pScanNode->suid, pUidList); } else { // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { - void* pIndex = tsdbGetIvtIdx(metaHandle); + void* pIndex = pStorageAPI->metaFn.storeGetInvertIndex(pVnode); + SIndexMetaArg metaArg = { - .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; + .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid}; status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); - code = TDB_CODE_SUCCESS; + code = TSDB_CODE_SUCCESS; } else { qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); } } } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1127,7 +1129,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t)); } - metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); +// metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); digest[0] = 1; memcpy(digest + 1, context.digest, tListLen(context.digest)); } @@ -1164,11 +1166,13 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) { return keyLen; } -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { +int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, + SStorageAPI* pAPI) { SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, uid) != 0) { // table not exist - metaReaderClear(&mr); + + pAPI->metaReaderFn.initReader(&mr, pVnode, 0); + if (pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, uid) != 0) { // table not exist + pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } @@ -1187,7 +1191,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, REPLACE_NODE(pNew); } else { nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return code; } @@ -1204,7 +1208,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, if (tTagIsJson(data)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } int32_t len = getJsonValueLen(data); @@ -1224,7 +1228,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, *pGroupId = calcGroupId(keyBuf, len); nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_SUCCESS; } @@ -2030,11 +2034,11 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups); taosArrayDestroy(pList); - return TDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, - bool groupSort, uint8_t *digest) { + bool groupSort, uint8_t *digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; bool groupByTbname = groupbyTbname(group); @@ -2054,7 +2058,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest); + code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest, pAPI); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2086,7 +2090,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags } uint8_t digest[17] = {0}; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); + int32_t code = getTableList(pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); return code; @@ -2104,7 +2108,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return TSDB_CODE_SUCCESS; } - code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest); + code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b6d4615997..e89db5b59c 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,8 @@ */ #include "executor.h" +#include +#include #include "executorInt.h" #include "operator.h" #include "planner.h" @@ -21,7 +23,8 @@ #include "tdatablock.h" #include "tref.h" #include "tudf.h" -#include "vnode.h" + +#include "storageapi.h" static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; int32_t exchangeObjRefPool = -1; @@ -156,18 +159,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -void doSetTaskId(SOperatorInfo* pOperator) { +void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pStreamScanInfo = pOperator->info; if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str); + pAPI->storeReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); } } } else { - doSetTaskId(pOperator->pDownstream[0]); + doSetTaskId(pOperator->pDownstream[0], pAPI); } } @@ -177,7 +180,7 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { buildTaskId(taskId, queryId, pTaskInfo->id.str); // set the idstr for tsdbReader - doSetTaskId(pTaskInfo->pRoot); + doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); } int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { @@ -320,7 +323,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v return pTaskInfo; } -static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { +static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr, + SStorageAPI* pAPI) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); int32_t numOfUids = taosArrayGetSize(tableIdList); if (numOfUids == 0) { @@ -336,11 +340,11 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; - metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.meta, 0); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); - int32_t code = metaGetTableEntryByUid(&mr, *id); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr); continue; @@ -368,7 +372,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S if (pScanInfo->pTagCond != NULL) { bool qualified = false; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; - code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified, pAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); continue; @@ -383,7 +387,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S taosArrayPush(qa, id); } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return qa; } @@ -401,10 +405,10 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI); int32_t numOfQualifiedTables = taosArrayGetSize(qa); qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); - code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); + code = pTaskInfo->storageAPI.tqReaderFn.tqReaderAddTables(pScanInfo->tqReader, qa); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(qa); return code; @@ -434,7 +438,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI keyInfo.groupId = keyInfo.uid; } else { code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, - &keyInfo.groupId); + &keyInfo.groupId, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(keyBuf); taosArrayDestroy(qa); @@ -456,7 +460,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI } else { // remove the table id in current list qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id); taosWLockLatch(&pTaskInfo->lock); - code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); + code = pTaskInfo->storageAPI.tqReaderFn.tqReaderRemoveTables(pScanInfo->tqReader, tableIdList); taosWUnLockLatch(&pTaskInfo->lock); } @@ -1060,6 +1064,8 @@ void qStreamSetOpen(qTaskInfo_t tinfo) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SOperatorInfo* pOperator = pTaskInfo->pRoot; const char* id = GET_TASKID(pTaskInfo); @@ -1081,14 +1087,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__LOG) { // todo refactor: move away - tsdbReaderClose(pScanBaseInfo->dataReader); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; - walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { - qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); - return -1; - } + ASSERT(0); +// walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); +// if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { +// qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); +// return -1; +// } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { // iterate all tables from tableInfoList, and retrieve rows from each table one-by-one // those data are from the snapshot in tsdb, besides the data in the wal file. @@ -1141,8 +1148,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, - pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false, NULL); + int32_t code = pTaskInfo->storageAPI.storeReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); terrno = code; @@ -1152,8 +1159,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { - tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + pTaskInfo->storageAPI.storeReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + pTaskInfo->storageAPI.storeReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } @@ -1175,14 +1182,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; - if (setForSnapShot(sContext, pOffset->uid) != 0) { + if (pAPI->snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } - SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); - tsdbReaderClose(pInfo->dataReader); + SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.storeSSGetTableInfo(sContext); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1200,7 +1207,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); int32_t size = tableListGetSize(pTableListInfo); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, + pTaskInfo->storageAPI.storeReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1212,7 +1219,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; - if (setForSnapShot(sContext, pOffset->uid) != 0) { + if (pTaskInfo->storageAPI.snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; @@ -1221,7 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; - tsdbReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index d229f7d0ee..c11c5be1ae 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -33,7 +33,7 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" +#include "storageapi.h" #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) @@ -844,6 +844,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSDataBlock* pBlock = pbInfo->pRes; // set output datablock version @@ -860,12 +862,13 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false); void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + + pAPI->stateStore.streamStateFreeVal(tbname); } void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, @@ -1066,122 +1069,22 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* return TSDB_CODE_SUCCESS; } -int32_t resultRowEncode(void* k, int32_t* size, char* buf) { - // SResultRow* key = k; - // int len = 0; - // int struLen = *size; - // len += taosEncodeFixedI32((void**)&buf, key->pageId); - - // uint32_t offset = key->offset; - // len += taosEncodeFixedU32((void**)&buf, offset); - - // len += taosEncodeFixedI8((void**)&buf, key->startInterp); - // len += taosEncodeFixedI8((void**)&buf, key->endInterp); - // len += taosEncodeFixedI8((void**)&buf, key->closed); - // len += taosEncodeFixedU32((void**)&buf, key->numOfRows); - - // len += taosEncodeFixedI64((void**)&buf, key->win.skey); - // len += taosEncodeFixedI64((void**)&buf, key->win.ekey); - - // int32_t numOfEntryInfo = (struLen - sizeof(SResultRow)) / sizeof(struct SResultRowEntryInfo); - // len += taosEncodeFixedI32((void**)&buf, numOfEntryInfo); - // for (int i = 0; i < numOfEntryInfo; i++) { - // SResultRowEntryInfo* p = &key->pEntryInfo[i]; - - // uint8_t value = p->initialized ? 1 : 0; - // len += taosEncodeFixedU8((void**)&buf, value); - - // value = p->complete ? 1 : 0; - // len += taosEncodeFixedU8((void**)&buf, value); - - // value = p->isNullRes; - // len += taosEncodeFixedU8((void**)&buf, value); - - // len += taosEncodeFixedU16((void**)&buf, p->numOfRes); - // } - // { - // char* strBuf = taosMemoryCalloc(1, *size * 100); - // resultRowToString(key, *size, strBuf); - // qWarn("encode result row:%s", strBuf); - // } - - // return len; - return 0; -} - -int32_t resultRowDecode(void** k, size_t size, char* buf) { - // char* p1 = buf; - // int32_t numOfEntryInfo = 0; - // uint32_t entryOffset = sizeof(int32_t) + sizeof(uint32_t) + sizeof(int8_t) + sizeof(int8_t) + sizeof(int8_t) + - // sizeof(uint32_t) + sizeof(int64_t) + sizeof(int64_t); - // taosDecodeFixedI32(p1 + entryOffset, &numOfEntryInfo); - - // char* p = buf; - // size = sizeof(SResultRow) + numOfEntryInfo * sizeof(SResultRowEntryInfo); - // SResultRow* key = taosMemoryCalloc(1, size); - - // p = taosDecodeFixedI32(p, (int32_t*)&key->pageId); - // uint32_t offset = 0; - // p = taosDecodeFixedU32(p, &offset); - // key->offset = offset; - - // p = taosDecodeFixedI8(p, (int8_t*)(&key->startInterp)); - // p = taosDecodeFixedI8(p, (int8_t*)(&key->endInterp)); - // p = taosDecodeFixedI8(p, (int8_t*)&key->closed); - // p = taosDecodeFixedU32(p, &key->numOfRows); - - // p = taosDecodeFixedI64(p, &key->win.skey); - // p = taosDecodeFixedI64(p, &key->win.ekey); - // p = taosDecodeFixedI32(p, &numOfEntryInfo); - // for (int i = 0; i < numOfEntryInfo; i++) { - // SResultRowEntryInfo* pInfo = &key->pEntryInfo[i]; - // uint8_t value = 0; - // p = taosDecodeFixedU8(p, &value); - // pInfo->initialized = (value == 1) ? true : false; - - // p = taosDecodeFixedU8(p, &value); - // pInfo->complete = (value == 1) ? true : false; - - // p = taosDecodeFixedU8(p, &value); - // pInfo->isNullRes = value; - - // p = taosDecodeFixedU16(p, &pInfo->numOfRes); - // } - // *k = key; - - // { - // char* strBuf = taosMemoryCalloc(1, size * 100); - // resultRowToString(key, size, strBuf); - // qWarn("decode result row:%s", strBuf); - // } - // return size; - return 0; -} - -int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { - // char* buf = taosMemoryCalloc(1, resSize * 10); - // int len = resultRowEncode((void*)pResult, &resSize, buf); - // char* buf = taosMemoryCalloc(1, resSize); - // memcpy(buf, pResult, resSize); - streamStatePut(pState, pKey, (char*)pResult, resSize); - // taosMemoryFree(buf); +int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI) { + pAPI->streamStateReleaseBuf(pState, pKey, pResult); return TSDB_CODE_SUCCESS; } -int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) { - streamStateReleaseBuf(pState, pKey, pResult); +int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI) { + pAPI->streamStateSessionPut(pState, key, (const void*)buf, size); + releaseOutputBuf(pState, NULL, (SResultRow*)buf, pAPI); return TSDB_CODE_SUCCESS; } -int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) { - streamStateSessionPut(pState, key, (const void*)buf, size); - releaseOutputBuf(pState, NULL, (SResultRow*)buf); - return TSDB_CODE_SUCCESS; -} - -int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -1193,7 +1096,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); int32_t size = 0; void* pVal = NULL; - int32_t code = streamStateSessionGet(pState, pKey, &pVal, &size); + int32_t code = pAPI->stateStore.streamStateSessionGet(pState, pKey, &pVal, &size); ASSERT(code == 0); if (code == -1) { // coverity scan @@ -1205,7 +1108,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); continue; } @@ -1213,23 +1116,23 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.id.groupId != pKey->groupId) { - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); break; } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { ASSERT(pBlock->info.rows > 0); - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); break; } @@ -1260,7 +1163,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta pBlock->info.dataLoad = 1; pBlock->info.rows += pRow->numOfRows; - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); } blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; @@ -1292,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) { qDebug("wait for the reader stopping"); } - tsdbReaderClose(pTSInfo->base.dataReader); + pTaskInfo->storageAPI.storeReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; // restore the status, todo refactor. diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 5101e36992..0b89dbc80b 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -443,7 +443,8 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); + ASSERT(0); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore); //????? pFillSup->cur.pRowVal = NULL; cleanupExprSupp(&pFillSup->notFillExprSup); @@ -490,74 +491,79 @@ static void resetFillWindow(SResultRowData* pRowData) { pRowData->pRowVal = NULL; } -void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { +void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { resetFillWindow(&pFillSup->prev); - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); + ASSERT(0); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore); //??? resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->nextNext); } void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); + + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); ASSERT(code == TSDB_CODE_SUCCESS); pFillSup->cur.key = key.ts; } void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; void* curVal = NULL; int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); ASSERT(code == TSDB_CODE_SUCCESS); pFillSup->cur.key = key.ts; pFillSup->cur.pRowVal = curVal; - SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key); + void* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; void* preVal = NULL; int32_t preVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->prev.key = preKey.ts; pFillSup->prev.pRowVal = preVal; - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); ASSERT(code == TSDB_CODE_SUCCESS); - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); if (code != TSDB_CODE_SUCCESS) { - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); pCur = NULL; } } else { - streamStateFreeCur(pCur); - pCur = streamStateFillSeekKeyNext(pState, &key); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); } SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; void* nextVal = NULL; int32_t nextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->next.key = nextKey.ts; pFillSup->next.pRowVal = nextVal; if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); if (code == TSDB_CODE_SUCCESS) { SWinKey nextNextKey = {.groupId = groupId}; void* nextNextVal = NULL; int32_t nextNextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->nextNext.key = nextNextKey.ts; pFillSup->nextNext.pRowVal = nextNextVal; @@ -565,7 +571,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, } } } - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); } static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } @@ -922,8 +928,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* } static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SWinKey key = {.groupId = groupId, .ts = pRow->key}; - int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); + int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); ASSERT(code == TSDB_CODE_SUCCESS); } @@ -1021,7 +1029,8 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { } static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStreamState* pState = pOp->pTaskInfo->streamInfo.pState; + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + void* pState = pOp->pTaskInfo->streamInfo.pState; SSDataBlock* pBlock = delRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); @@ -1041,14 +1050,14 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; - streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); if (tbname == NULL) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows++; @@ -1070,12 +1079,13 @@ static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY end } static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamFillOperatorInfo* pInfo = pOperator->info; getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); SWinKey key = {.ts = startTs, .groupId = groupId}; if (!pInfo->pFillInfo->needFill) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); } else { STimeRange tw = { @@ -1093,6 +1103,8 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE } static void doDeleteFillFinalize(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; int32_t size = taosArrayGetSize(pFillInfo->delRanges); @@ -1109,17 +1121,16 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) { pInfo->pRes->info.id.groupId = range->groupId; } SWinKey key = {.ts = range->skey, .groupId = range->groupId}; - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); } } static void doDeleteFillResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SSDataBlock* pBlock = pInfo->pSrcDelBlock; - SSDataBlock* pRes = pInfo->pRes; - SSDataBlock* pDelRes = pInfo->pDelRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsStarts = (TSKEY*)pStartCol->pData; @@ -1130,7 +1141,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { TSKEY endTs = ts; uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; SWinKey key = {.ts = ts, .groupId = groupId}; - SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + void* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); if (!pCur) { pInfo->srcDelRowIndex++; continue; @@ -1152,15 +1163,15 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; if (delTs == nextKey.ts) { - code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); if (code == TSDB_CODE_SUCCESS) { - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); } // ts will be deleted later if (delTs != ts) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); - streamStateFreeCur(pCur); - pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } endTs = TMAX(delTs, nextKey.ts - 1); if (code != TSDB_CODE_SUCCESS) { @@ -1169,9 +1180,11 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { } pInfo->srcDelRowIndex++; } - streamStateFreeCur(pCur); + + pAPI->stateStore.streamStateFreeCur(pCur); doDeleteFillResultImpl(pOperator, ts, endTs, groupId); } + pFillInfo->current = pFillInfo->end + 1; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f2eee82579..77d251bab9 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -950,6 +950,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; SSDataBlock* pDest = pInfo->binfo.pRes; ASSERT(hasRemainPartion(pInfo)); @@ -972,9 +974,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { pDest->info.parTbName[0] = 0; if (pInfo->tbnameCalSup.numOfExprs > 0) { void* tbname = NULL; - if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { + if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } } taosArrayDestroy(pParInfo->rowIds); @@ -990,10 +992,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { return pDest; } -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { +void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { void* pValue = NULL; - if (streamStateGetParName(pState, groupId, &pValue) != 0) { + if (pAPI->streamStateGetParName(pState, groupId, &pValue) != 0) { SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); pTmpBlock->info.id.groupId = groupId; @@ -1010,7 +1012,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); - streamStatePutParName(pState, groupId, tbName); + pAPI->streamStatePutParName(pState, groupId, tbName); } memcpy(pTmpBlock->info.parTbName, tbName, len); pDestBlock->info.rows--; @@ -1034,10 +1036,12 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* } else { memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); } - streamStateReleaseBuf(pState, NULL, pValue); + pAPI->streamStateReleaseBuf(pState, NULL, pValue); } static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { + SExecTaskInfo* pTask = pOperator->pTaskInfo; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || taosHashGetSize(pInfo->pPartitions) == 0) { @@ -1050,8 +1054,8 @@ static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { if (pInfo->pTbNameIte != NULL) { SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); - appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes); + appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); } return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; @@ -1164,14 +1168,17 @@ static void destroyStreamPartitionOperatorInfo(void* param) { } void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) { + SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { return; } + SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->partitionSup = *pParSup; pScanInfo->pPartScalarSup = pExpr; if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); + pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); } } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 729178dc60..1b85aedc1e 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -25,7 +25,8 @@ #include "operator.h" #include "query.h" #include "querytask.h" -#include "vnode.h" + +#include "storageapi.h" SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, @@ -233,11 +234,12 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) { + SStorageAPI* pAPI = pParam->pParam; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pInfo = pOperator->info; if (pInfo->base.dataReader != NULL) { - tsdbReaderSetCloseFlag(pInfo->base.dataReader); + pAPI->storeReader.storeReaderNotifyClosing(pInfo->base.dataReader); } return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -246,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, if (pInfo->pTableScanOp != NULL) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { - tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); } } @@ -256,8 +258,8 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, return OPTR_FN_RET_CONTINUE; } -int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) { - STraverParam p = {0}; +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI) { + STraverParam p = {.pParam = pAPI}; traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr); return p.code; } @@ -379,7 +381,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR if (pBlockNode->tableType == TSDB_SUPER_TABLE) { SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); - int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList); + int32_t code = pTaskInfo->storageAPI.metaFn.storeGetTableList(pHandle->vnode, pBlockNode->uid, pList); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = terrno; return NULL; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 02f504bef0..ae72627575 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -18,6 +18,7 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" +#include "tdatablock.h" typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 7716b5976b..a48de1d158 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -29,9 +29,9 @@ #include "operator.h" #include "query.h" #include "querytask.h" +#include "storageapi.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) @@ -63,7 +63,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); } void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; - stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str); + stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI); } void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { @@ -120,13 +120,15 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo return terrno; } - metaReaderInit(&mr, pHandle->meta, 0); - int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + + pAPI->metaReaderFn.initReader(&mr, pHandle->meta, 0); + int32_t code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } @@ -142,9 +144,9 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo tDecoderClear(&mr.coder); tb_uid_t suid = mr.me.ctbEntry.suid; - code = metaGetTableEntryByUidCache(&mr, suid); + code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, suid); if (code != TSDB_CODE_SUCCESS) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } @@ -154,7 +156,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow); } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0537702a75..ac8ca675ce 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -33,6 +33,9 @@ #include "operator.h" #include "querytask.h" +#include "storageapi.h" +#include "wal.h" + int32_t scanDebug = 0; #define MULTI_READER_MAX_TABLE_NUM 5000 @@ -261,8 +264,10 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA } static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + int32_t code = pAPI->storeReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -323,6 +328,8 @@ bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; @@ -343,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, @@ -351,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -361,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -383,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -398,10 +405,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); - STableScanInfo* pTableScanInfo = pOperator->info; - if (taosHashGetSize(pTableScanInfo->pIgnoreTables) == taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList)) { + STableScanInfo* p1 = pOperator->info; + if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) { *status = FUNC_DATA_REQUIRED_ALL_FILTEROUT; } else { *status = FUNC_DATA_REQUIRED_FILTEROUT; @@ -412,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + SSDataBlock* p = pAPI->storeReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (p == NULL) { return terrno; } @@ -524,8 +531,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { - metaReaderInit(&mr, pHandle->meta, 0); - code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -538,11 +545,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } - metaReaderReleaseLock(&mr); + pHandle->api.metaReaderFn.readerReleaseLock(&mr); val.pName = mr.me.name; val.pTags = (STag*)mr.me.ctbEntry.pTags; @@ -553,8 +560,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { - metaReaderInit(&mr, pHandle->meta, 0); - code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", @@ -565,11 +572,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } - metaReaderReleaseLock(&mr); + pHandle->api.metaReaderFn.readerReleaseLock(&mr); STableCachedVal* pVal = createTableCacheVal(&mr); @@ -609,7 +616,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } else { // these are tags STagVal tagVal = {0}; tagVal.cid = pExpr1->base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(val.pTags, pColInfoData->info.type, &tagVal); + const char* p = pHandle->api.metaFn.extractTagVal(val.pTags, pColInfoData->info.type, &tagVal); char* data = NULL; if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -628,7 +635,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } if (code) { if (freeReader) { - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); } return code; } @@ -643,7 +650,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // restore the rows pBlock->info.rows = backupRows; if (freeReader) { - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); } return TSDB_CODE_SUCCESS; @@ -677,6 +684,8 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSDataBlock* pBlock = pTableScanInfo->pResBlock; bool hasNext = false; int32_t code = TSDB_CODE_SUCCESS; @@ -684,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); } @@ -695,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -733,14 +742,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime; - - // todo refactor - /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ - /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ - // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; - return pBlock; } return NULL; @@ -749,7 +750,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; @@ -772,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } @@ -780,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -798,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -809,6 +811,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; // scan table one by one sequentially if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { @@ -836,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); - tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); + pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); - tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -855,8 +858,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); + int32_t code = pAPI->storeReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, + (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -884,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - tsdbSetTableList(pInfo->base.dataReader, pList, num); - tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); + pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; result = doGroupedTableScan(pOperator); @@ -907,10 +910,10 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanBase(STableScanBase* pBase) { +static void destroyTableScanBase(STableScanBase* pBase, SStoreDataReaderFn* pAPI) { cleanupQueryTableDataCond(&pBase->cond); - tsdbReaderClose(pBase->dataReader); + pAPI->storeReaderClose(pBase->dataReader); pBase->dataReader = NULL; if (pBase->matchInfo.pList != NULL) { @@ -926,7 +929,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - destroyTableScanBase(&pTableScanInfo->base); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->readerAPI); taosMemoryFreeClear(param); } @@ -1071,8 +1074,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.endVersion = version; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - tsdbReaderClose(pTableScanInfo->base.dataReader); - qDebug("1"); + pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -1088,11 +1090,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs}; SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false, NULL); + int32_t code = pAPI->storeReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1100,7 +1103,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = tsdbNextDataBlock(pReader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1108,12 +1111,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); + /*SSDataBlock* p = */ pAPI->storeReader.storeReaderRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } - tsdbReaderClose(pReader); + pAPI->storeReader.storeReaderClose(pReader); qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1187,7 +1190,8 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ break; } - resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); + ASSERT(0); +// resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -1245,8 +1249,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTableScanInfo->base.dataReader); - qDebug("2"); + pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -1291,8 +1294,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, pKey->win.ekey = endTs; pKey->groupId = groupId; - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); - int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); + void* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -1463,11 +1466,11 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS } if (pInfo->tbnameCalSup.pExprInfo) { void* parTbname = NULL; - streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname); + pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname); memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN); varDataSetLen(tbname, strlen(varDataVal(tbname))); - streamFreeVal(parTbname); + pInfo->stateStore.streamStateFreeVal(parTbname); } appendOneRowToStreamSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, tbname[0] == 0 ? NULL : tbname); @@ -1498,7 +1501,7 @@ static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { pBlock->info.parTbName[0] = 0; } else { appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes); + pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes, &pInfo->stateStore); } } @@ -1529,7 +1532,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; - bool tableInserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid); + bool tableInserted = pInfo->stateStore.updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid); for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -1545,10 +1548,9 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock isClosed = isCloseWindow(&win, &pInfo->twAggSup); } // must check update info first. - bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]); + bool update = pInfo->stateStore.updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]); bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && - isDeletedStreamWindow(&win, pBlock->info.id.groupId, - pInfo->pState, &pInfo->twAggSup); + isDeletedStreamWindow(&win, pBlock->info.id.groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore); if ((update || closedWin) && out) { qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); uint64_t gpId = 0; @@ -1636,58 +1638,30 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamScanInfo* pInfo = pOperator->info; const char* id = GET_TASKID(pTaskInfo); qDebug("start to exec queue scan, %s", id); -#if 0 - if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg.msgStr == NULL) { - SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - return NULL; - } - } - - blockDataCleanup(pInfo->pRes); - SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - - while (tqNextBlockImpl(pInfo->tqReader, NULL)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { - continue; - } - - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true); - - if (pBlockInfo->rows > 0) { - return pInfo->pRes; - } - } - - pInfo->tqReader->msg = (SPackedData){0}; - pTaskInfo->streamInfo.submit = (SPackedData){0}; - return NULL; - } -#endif - if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, - pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, - pInfo->tqReader->pWalReader->curVersion); + ASSERT(0); +// qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, +// pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, +// pInfo->tqReader->pWalReader->curVersion); tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; } STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); - if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { + if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { return NULL; } @@ -1697,11 +1671,14 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { - bool hasResult = tqNextBlockInWal(pInfo->tqReader, id); - SSDataBlock* pRes = pInfo->tqReader->pResBlock; + bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id); + ASSERT(0); + + SSDataBlock* pRes = NULL; + struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); // curVersion move to next, so currentOffset = curVersion - 1 - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion - 1); if (hasResult) { qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, @@ -1737,9 +1714,10 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX); - int32_t j = 0; + + int32_t j = 0; for (int32_t i = 0; i < rows; i++) { - if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) { + if (pInfo->readerFn.tqReaderIsQueriedTable(pReader, uidCol[i])) { colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false); colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false); colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false); @@ -1750,6 +1728,7 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S j++; } } + uint32_t cap = pDst->info.capacity; pDst->info = pSrc->info; pDst->info.rows = j; @@ -1777,7 +1756,8 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { - pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + ASSERT(0); +// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); checkUpdateData(pInfo, true, pBlock, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); if (pInfo->pUpdateDataRes->info.rows > 0) { @@ -1794,21 +1774,21 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl } } -int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { - int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - *pBuff = taosMemoryCalloc(1, len); - updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo); - return len; -} +//int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { +// int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); +// *pBuff = taosMemoryCalloc(1, len); +// updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo); +// return len; +//} // other properties are recovered from the execution plan -void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) { +void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) { if (!pBuff || len == 0) { return; } - SUpdateInfo* pUpInfo = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); - int32_t code = updateInfoDeserialize(pBuff, len, pUpInfo); + void* pUpInfo = pInfo->stateStore.updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); if (code == TSDB_CODE_SUCCESS) { pInfo->pUpdateInfo = pUpInfo; } @@ -1817,6 +1797,8 @@ void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamScanInfo* pInfo = pOperator->info; qDebug("stream scan started, %s", GET_TASKID(pTaskInfo)); @@ -1839,7 +1821,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1902,7 +1884,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { calBlockTbName(pInfo, pInfo->pRecoverRes); if (pInfo->pUpdateInfo) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { - TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); + TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } else { doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); @@ -1919,7 +1901,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; @@ -1943,7 +1925,7 @@ FETCH_NEXT_BLOCK: SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { - streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); + pAPI->stateStore.streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } // TODO move into scan @@ -1951,7 +1933,8 @@ FETCH_NEXT_BLOCK: pBlock->info.calWin.ekey = INT64_MAX; pBlock->info.dataLoad = 1; if (pInfo->pUpdateInfo) { - pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + ASSERT(0); +// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); } blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { @@ -1964,7 +1947,7 @@ FETCH_NEXT_BLOCK: copyDataBlock(pInfo->pUpdateRes, pBlock); pInfo->updateResIndex = 0; prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); + pAPI->stateStore.updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } break; case STREAM_DELETE_DATA: { printDataBlock(pBlock, "stream scan delete recv"); @@ -2081,9 +2064,9 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->msg.msgStr == NULL) { + if (pInfo->readerFn.tqReaderCurrentBlockConsumed(pInfo->tqReader)) { if (pInfo->validBlockIndex >= totalBlocks) { - updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); + pAPI->stateStore.updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); qDebug("stream scan return empty, all %d submit blocks consumed, %s", totalBlocks, id); @@ -2094,7 +2077,7 @@ FETCH_NEXT_BLOCK: SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); qDebug("set %d/%d as the input submit block, %s", current, totalBlocks, id); - if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { + if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id); continue; } @@ -2102,13 +2085,14 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pBlock); - while (tqNextBlockImpl(pInfo->tqReader, id)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, id); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { + while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { + SSDataBlock* pRes = NULL; + int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); + if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { continue; } - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, false); + setBlockIntoRes(pInfo, pRes, false); if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; @@ -2166,7 +2150,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableListInfo) { static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamRawScanInfo* pInfo = pOperator->info; int32_t code = TSDB_CODE_SUCCESS; pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta @@ -2176,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); if (code) { - tsdbReleaseDataBlock(pInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(pInfo->dataReader); + pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); + SSDataBlock* pBlock = pAPI->storeReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2199,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return pBlock; } - SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); + SMetaTableInfo mtInfo = pAPI->snapshotFn.storeSSGetTableInfo(pInfo->sContext); STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); @@ -2217,7 +2203,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { int32_t dataLen = 0; int16_t type = 0; int64_t uid = 0; - if (getMetafromSnapShot(sContext, &data, &dataLen, &type, &uid) < 0) { + if (pAPI->snapshotFn.storeSSGetMetaInfo(sContext, &data, &dataLen, &type, &uid) < 0) { qError("tmqsnap getMetafromSnapShot error"); taosMemoryFreeClear(data); return NULL; @@ -2241,8 +2227,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; - tsdbReaderClose(pRawScan->dataReader); - destroySnapContext(pRawScan->sContext); + pRawScan->pAPI->storeReader.storeReaderClose(pRawScan->dataReader); + pRawScan->pAPI->snapshotFn.storeDestroySnapshot(pRawScan->sContext); tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); } @@ -2290,7 +2276,7 @@ static void destroyStreamScanOperatorInfo(void* param) { } if (pStreamScan->tqReader) { - tqCloseReader(pStreamScan->tqReader); + pStreamScan->readerFn.tqReaderClose(pStreamScan->tqReader); } if (pStreamScan->matchInfo.pList) { taosArrayDestroy(pStreamScan->matchInfo.pList); @@ -2303,7 +2289,7 @@ static void destroyStreamScanOperatorInfo(void* param) { cleanupExprSupp(&pStreamScan->tbnameCalSup); cleanupExprSupp(&pStreamScan->tagCalSup); - updateInfoDestroy(pStreamScan->pUpdateInfo); + pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); blockDataDestroy(pStreamScan->pPullDataRes); @@ -2319,6 +2305,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2408,7 +2395,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTqReader) { ASSERT(pHandle->tqReader == NULL); - pInfo->tqReader = tqReaderOpen(pHandle->vnode); + pInfo->tqReader = pAPI->tqReaderFn.tqReaderOpen(pHandle->vnode); ASSERT(pInfo->tqReader); } else { ASSERT(pHandle->tqReader); @@ -2418,7 +2405,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) { - streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); + pAPI->stateStore.streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); } pInfo->readHandle = *pHandle; @@ -2427,9 +2414,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); // set the extract column id to streamHandle - tqReaderSetColIdList(pInfo->tqReader, pColIds); + pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); - code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); + code = pAPI->tqReaderFn.tqReaderSetTargetTableList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; @@ -2475,8 +2462,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pTaskInfo->streamInfo.pState) { void* buff = NULL; int32_t len = 0; - streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); - streamScanOperatorDeocde(buff, len, pInfo); + pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); + streamScanOperatorDecode(buff, len, pInfo); } setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, @@ -2502,18 +2489,18 @@ _error: return NULL; } -static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr) { +static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); - int32_t code = metaGetTableEntryByUid(mr, item->uid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(mr, item->uid); tDecoderClear(&(*mr).coder); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(mr); + pAPI->metaReaderFn.clearReader(mr); T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2528,7 +2515,7 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, } else { // it is a tag value STagVal val = {0}; val.cid = pExprInfo[j].base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); + const char* p = pAPI->metaFn.extractTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); char* data = NULL; if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -2553,6 +2540,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; @@ -2568,10 +2556,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { char str[512] = {0}; int32_t count = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - doTagScanOneTable(pOperator, pRes, count, &mr); + doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); ++count; if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); @@ -2589,7 +2577,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { @@ -2661,6 +2649,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SOperatorInfo* pOperator = source->pOperator; STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int32_t readIdx = source->readerIdx; SSDataBlock* pBlock = source->inputBlock; int32_t code = 0; @@ -2672,7 +2662,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false, NULL); + code = pAPI->storeReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2684,9 +2674,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = tsdbNextDataBlock(reader, &hasNext); + code = pAPI->storeReader.storeReaderNextDataBlock(reader, &hasNext); if (code != 0) { - tsdbReleaseDataBlock(reader); + pAPI->storeReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, code); } @@ -2696,7 +2686,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(reader); + pAPI->storeReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2736,7 +2726,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, close reader", reader); if (!source->multiReader) { - tsdbReaderClose(pInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2744,7 +2734,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (!source->multiReader) { - tsdbReaderClose(pInfo->base.dataReader); + pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2849,6 +2839,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; int32_t numOfTable = taosArrayGetSize(pInfo->queryConds); @@ -2862,7 +2853,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { for (int32_t i = 0; i < numOfTable; ++i) { STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); blockDataDestroy(param->inputBlock); - tsdbReaderClose(param->dataReader); + pAPI->storeReader.storeReaderClose(param->dataReader); param->dataReader = NULL; } taosArrayClear(pInfo->sortSourceParams); @@ -2974,11 +2965,11 @@ void destroyTableMergeScanOperatorInfo(void* param) { for (int32_t i = 0; i < numOfTable; i++) { STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); blockDataDestroy(p->inputBlock); - tsdbReaderClose(p->dataReader); + pTableScanInfo->base.readerAPI.storeReaderClose(p->dataReader); p->dataReader = NULL; } - tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; taosArrayDestroy(pTableScanInfo->sortSourceParams); @@ -2991,7 +2982,7 @@ void destroyTableMergeScanOperatorInfo(void* param) { } taosArrayDestroy(pTableScanInfo->queryConds); - destroyTableScanBase(&pTableScanInfo->base); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); @@ -3359,9 +3350,11 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount const char* db = NULL; int32_t vgId = 0; char dbName[TSDB_DB_NAME_LEN] = {0}; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; // get dbname - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, dbName); @@ -3376,10 +3369,13 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + if (pSupp->groupByStbName) { if (pInfo->stbUidList == NULL) { pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { + if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, 0, pInfo->stbUidList, TSDB_SUPER_TABLE) < 0) { qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); } } @@ -3398,7 +3394,9 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca } else { uint64_t groupId = calcGroupId(dbName, strlen(dbName)); pRes->info.id.groupId = groupId; - int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); + + int64_t dbTableCount = 0; + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.meta, &dbTableCount); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } @@ -3406,19 +3404,25 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) { - tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter); + tb_uid_t uid = 0; + pAPI->metaFn.getTableUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter, &uid); SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, uid, &stats); + ASSERT(0); +// metaGetStbStats(pInfo->readHandle.meta, uid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } setOperatorCompleted(pOperator); @@ -3433,7 +3437,8 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); + int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.meta); + ASSERT(0); if (ntbNum != 0) { fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); } @@ -3442,7 +3447,8 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { char stbName[TSDB_TABLE_NAME_LEN] = {0}; - metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + ASSERT(0); +// metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { @@ -3455,7 +3461,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S pRes->info.id.groupId = groupId; SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); +// metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 04c6c619aa..460373b537 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -21,19 +21,19 @@ #include "querynodes.h" #include "systable.h" #include "tname.h" -#include "ttime.h" #include "tdatablock.h" #include "tmsg.h" +#include "operator.h" #include "query.h" +#include "querytask.h" +#include "storageapi.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" -#include "operator.h" -#include "querytask.h" - +#include "tsdstorage.h" +#include "index.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); @@ -72,6 +72,7 @@ typedef struct SSysTableScanInfo { SLoadRemoteDataInfo loadInfo; SLimitInfo limitInfo; int32_t tbnameSlotId; + SStorageAPI* pAPI; } SSysTableScanInfo; typedef struct { @@ -156,7 +157,8 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { void* pVnode = ((SSTabFltArg*)arg)->pVnode; const char* db = NULL; - vnodeGetInfo(pVnode, &db, NULL); + ASSERT(0); +// pAPI->metaFn.storeGetBasicInfo(pVnode, &db, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -183,7 +185,8 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { void* pVnode = ((SSTabFltArg*)arg)->pVnode; int64_t vgId = 0; - vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); + ASSERT(0); +// pAPI->metaFn.storeGetBasicInfo(pVnode, NULL, (int32_t*)&vgId); SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -237,8 +240,10 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { .equal = equal, .filterFunc = func}; - int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); - return ret; + ASSERT(0); + return 0; +// int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); +// return ret; } int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { @@ -431,8 +436,9 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { } static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { - qDebug("sysTableScanUserCols get cols start"); - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -446,7 +452,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -461,18 +467,18 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; - metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } if (smrTable.me.type == TSDB_SUPER_TABLE) { - metaReaderClear(&smrTable); + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -480,12 +486,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (smrTable.me.type == TSDB_CHILD_TABLE) { int64_t suid = smrTable.me.ctbEntry.suid; - metaReaderClear(&smrTable); - metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); - code = metaGetTableEntryByUid(&smrTable, suid); + pAPI->metaReaderFn.clearReader(&smrTable); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -503,7 +509,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); - metaReaderClear(&smrTable); + pAPI->metaReaderFn.clearReader(&smrTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -517,7 +523,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); } if (pInfo->pSchema == NULL) { @@ -535,8 +541,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t restore = pInfo->restore; pInfo->restore = false; - while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { - if (restore) restore = false; + + while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { + if (restore) { + restore = false; + } + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -560,12 +570,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); - int code = metaGetTableEntryByUid(&smrSuperTable, suid); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); + int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -573,7 +583,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow); taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); schemaRow = schemaWrapper; - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); } } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { qDebug("sysTableScanUserCols cursor get normal table"); @@ -605,7 +615,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -618,6 +628,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -631,7 +643,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -647,37 +659,37 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, condTableName); SMetaReader smrChildTable = {0}; - metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.meta, 0); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrChildTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } if (smrChildTable.me.type != TSDB_CHILD_TABLE) { - metaReaderClear(&smrChildTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByUid - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); return NULL; } sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -691,11 +703,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); } bool blockFull = false; - while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { + while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { continue; } @@ -704,27 +716,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&smrSuperTable); - metaCloseTbCursor(pInfo->pCur); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { - metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX); + pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX); blockFull = true; } else { sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); } - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); if (blockFull || numOfRows >= pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -745,7 +757,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1090,6 +1102,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; SSysTableIndex* pIdx = pInfo->pIdx; @@ -1100,7 +1114,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1118,10 +1132,10 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - ret = metaGetTableEntryByUid(&mr, *uid); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); + ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); if (ret < 0) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); continue; } STR_TO_VARSTR(n, mr.me.name); @@ -1146,15 +1160,15 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; - metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); int64_t suid = mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr1, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr1); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr1); + pAPI->metaReaderFn.clearReader(&mr); T_LONG_JMP(pTaskInfo->env, terrno); } pColInfoData = taosArrayGet(p->pDataBlock, 3); @@ -1164,7 +1178,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { STR_TO_VARSTR(n, mr1.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); colDataSetVal(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr1); + pAPI->metaReaderFn.clearReader(&mr1); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); @@ -1229,7 +1243,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // impl later } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); pColInfoData = taosArrayGet(p->pDataBlock, 9); colDataSetVal(pColInfoData, numOfRows, n, false); @@ -1275,10 +1289,11 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); } blockDataCleanup(pInfo->pRes); @@ -1286,7 +1301,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1301,7 +1316,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { + while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); // table name @@ -1324,15 +1339,15 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); + pAPI->metaReaderFn.clearReader(&mr); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -1345,7 +1360,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { STR_TO_VARSTR(n, mr.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); colDataSetVal(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); @@ -1442,7 +1457,8 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1708,7 +1724,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo) { - int32_t code = TDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1798,7 +1814,7 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - metaCloseTbCursor(pInfo->pCur); + pInfo->pAPI->metaFn.closeMetaCursor(pInfo->pCur); pInfo->pCur = NULL; } if (pInfo->pIdx) { @@ -2124,15 +2140,15 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { return -1; } -static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { +static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) { *rowLen = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - int32_t code = metaGetTableEntryByUid(&mr, uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } @@ -2144,10 +2160,10 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con } else if (mr.me.type == TSDB_CHILD_TABLE) { uint64_t suid = mr.me.ctbEntry.suid; tDecoderClear(&mr.coder); - code = metaGetTableEntryByUid(&mr, suid); + code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } @@ -2163,7 +2179,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con } } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return TSDB_CODE_SUCCESS; } @@ -2174,16 +2190,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SBlockDistInfo* pBlockScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, + int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid, (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); + pAPI->storeReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2213,7 +2230,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - tsdbReaderClose(pDistInfo->pHandle); + pDistInfo->readHandle.api.storeReader.storeReaderClose(pDistInfo->pHandle); tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2268,7 +2285,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.storeReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 2698b0bfd9..209e5cb8fc 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -18,6 +18,7 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" +#include "storageapi.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7a9efde4f3..b4fd571e23 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1331,10 +1331,12 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, } static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); - streamStateDel(pInfo->pState, &key); + pAPI->stateStore.streamStateDel(pInfo->pState, &key); return true; } @@ -1471,7 +1473,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin for (int32_t i = *index; i < size; i++) { SWinKey* pWin = taosArrayGet(pWins, i); void* tbname = NULL; - streamStateGetParName(pInfo->pState, pWin->groupId, &tbname); + pInfo->statestore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname); if (tbname == NULL) { appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); } else { @@ -1479,7 +1481,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); } - streamFreeVal(tbname); + pInfo->statestore.streamStateFreeVal(tbname); (*index)++; } } @@ -1533,7 +1535,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pDelWins); blockDataDestroy(pInfo->pDelRes); - streamFileStateDestroy(pInfo->pState->pFileState); + pInfo->statestore.streamFileStateDestroy(pInfo->pState->pFileState); taosMemoryFreeClear(pInfo->pState); nodesDestroyNode((SNode*)pInfo->pPhyNode); @@ -1608,16 +1610,20 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) { + SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { initIntervalDownStream(downstream->pDownstream[0], type, pInfo); return; } + SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); + pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); } + pScanInfo->interval = pInfo->interval; pScanInfo->twAggSup = pInfo->twAggSup; pScanInfo->pState = pInfo->pState; @@ -2045,33 +2051,33 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } } -bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return streamStateCheck(pState, pKey); } +bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { return pStore->streamStateCheck(pState, pKey); } -int32_t setIntervalOutputBuf(SStreamState* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, +int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, - SAggSupporter* pAggSup) { - SWinKey key = { - .ts = win->skey, - .groupId = groupId, - }; + SAggSupporter* pAggSup, SStateStore* pStore) { + + SWinKey key = { .ts = win->skey, .groupId = groupId }; char* value = NULL; int32_t size = pAggSup->resultRowSize; - if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { + if (pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_OUT_OF_MEMORY; } + *pResult = (SRowBuffPos*)value; SResultRow* res = (SResultRow*)((*pResult)->pRowBuff); + // set time window for current result res->win = (*win); setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore) { if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; - if (!hasIntervalWindow(pState, &key)) { + if (!hasIntervalWindow(pState, &key, pStore)) { return true; } return false; @@ -2102,7 +2108,7 @@ static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { clearDiskbasedBuf(pInfo->aggSup.pResultBuf); initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->aggSup.currentPageId = -1; - streamStateClear(pInfo->pState); + pInfo->statestore.streamStateClear(pInfo->pState); } static void clearSpecialDataBlock(SSDataBlock* pBlock) { @@ -2198,13 +2204,15 @@ static void clearFunctionContext(SExprSupp* pSup) { } } -int32_t getOutputBuf(SStreamState* pState, SRowBuffPos* pPos, SResultRow** pResult) { - return streamStateGetByPos(pState, pPos, (void**)pResult); +int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) { + return pStore->streamStateGetByPos(pState, pPos, (void**)pResult); } -int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -2215,7 +2223,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i); SResultRow* pRow = NULL; - int32_t code = getOutputBuf(pState, pPos, &pRow); + int32_t code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore); uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; ASSERT(code == 0); doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); @@ -2227,12 +2235,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.id.groupId != groupId) { @@ -2279,7 +2287,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat return TSDB_CODE_SUCCESS; } -void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version @@ -2354,7 +2362,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p .groupId = groupId, }; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); - if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && isClosed && !chIds) { + if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->statestore) && isClosed && !chIds) { SPullWindowInfo pull = { .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request @@ -2385,7 +2393,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup); + pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->statestore); pResult = (SResultRow*)pResPos->pRowBuff; if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); @@ -2467,6 +2475,8 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) { static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SOperatorInfo* downstream = pOperator->pDownstream[0]; SExprSupp* pSup = &pOperator->exprSupp; @@ -2505,8 +2515,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } else { if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { - streamStateCommit(pInfo->pState); - streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); + pAPI->stateStore.streamStateCommit(pInfo->pState); + pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; } qDebug("===stream===interval final close"); @@ -2707,6 +2717,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } pOperator->pTaskInfo = pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = pIntervalPhyNode->intervalUnit, @@ -2743,9 +2755,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + pInfo->pState = taosMemoryCalloc(1, sizeof(void)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); - streamStateSetNumber(pInfo->pState, -1); + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState); if (code != TSDB_CODE_SUCCESS) { @@ -2753,10 +2765,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - initResultRowInfo(&pInfo->binfo.resultRowInfo); + pInfo->numOfChild = numOfChild; pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode); @@ -2773,6 +2784,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } + pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); pInfo->pullIndex = 0; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -2789,7 +2801,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols); - pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, + pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); pInfo->dataVersion = 0; @@ -2887,7 +2899,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); + pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); } pScanInfo->twAggSup = *pTwSup; } @@ -2905,9 +2917,9 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, } initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); - pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + pSup->pState = taosMemoryCalloc(1, sizeof(void)); *(pSup->pState) = *pState; - streamStateSetNumber(pSup->pState, -1); + pSup->stateStore.streamStateSetNumber(pSup->pState, -1); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pSup->pResultRows = tSimpleHashInit(32, hashFn); @@ -2950,7 +2962,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pKey->win.skey = startTs; pKey->win.ekey = endTs; pKey->groupId = groupId; - int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); + int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -2964,8 +2976,8 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = - streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, + pAggSup->gap, &pCurWin->pOutputBuf, &size); if (code == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; } else { @@ -2976,11 +2988,12 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { return code; } - streamStateCurNext(pAggSup->pState, pCur); + + pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); return TSDB_CODE_SUCCESS; } void saveDeleteInfo(SArray* pWins, SSessionKey key) { @@ -3065,7 +3078,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo } static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { - streamStateSessionDel(pAggSup->pState, pKey); + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); SSessionKey hashKey = {0}; getSessionHashKey(pKey, &hashKey); tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); @@ -3083,12 +3096,12 @@ static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin) { - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); pNextWin->isOutput = true; setSessionWinOutputInfo(pStUpdated, pNextWin); int32_t size = 0; pNextWin->sessionWin = pCurWin->sessionWin; - int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pNextWin->pOutputBuf); SET_SESSION_WIN_INVALID(*pNextWin); @@ -3100,6 +3113,8 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC SSHashObj* pStDeleted) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; @@ -3111,7 +3126,7 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo); if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) { taosMemoryFree(winInfo.pOutputBuf); - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); break; } SResultRow* pWinResult = NULL; @@ -3125,13 +3140,13 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC } removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin); doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); taosMemoryFree(winInfo.pOutputBuf); } } int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { - saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize); + saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize, &pAggSup->stateStore); return TSDB_CODE_SUCCESS; } @@ -3251,6 +3266,8 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { } void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + blockDataCleanup(pBlock); int32_t size = tSimpleHashGetSize(pStDeleted); if (size == 0) { @@ -3279,14 +3296,14 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; - streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); if (tbname == NULL) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows += 1; } @@ -3298,6 +3315,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t size = taosArrayGetSize(pWinArray); SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -3314,7 +3333,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; SSessionKey chWinKey = {0}; getSessionHashKey(pWinKey, &chWinKey); - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); SResultRow* pResult = NULL; SResultRow* pChResult = NULL; while (1) { @@ -3341,7 +3360,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS break; } } - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); } if (num > 0) { saveSessionOutputBuf(pAggSup, &parentWin); @@ -3406,7 +3425,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL pGroupResInfo->pBuf = NULL; } -void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, +void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version @@ -3636,7 +3655,7 @@ _error: static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { tSimpleHashClear(pInfo->streamAggSup.pResultRows); - streamStateSessionClear(pInfo->streamAggSup.pState); + pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState); } static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { @@ -3762,6 +3781,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream goto _error; } + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); @@ -3786,7 +3806,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream } SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - streamStateSetNumber(pChInfo->streamAggSup.pState, i); + pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i); taosArrayPush(pInfo->pChildren, &pChildOp); } } @@ -3849,7 +3869,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; int32_t code = - streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, compareStateKey, &pCurWin->winInfo.pOutputBuf, &size); pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); @@ -3870,12 +3890,12 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; pNextWin->winInfo.pOutputBuf = NULL; - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); + code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); } - streamStateFreeCur(pCur); + pAggSup->stateStore.streamStateFreeCur(pCur); } int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, @@ -3914,7 +3934,9 @@ int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNex static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, SSHashObj* pStDeleted) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamStateAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; @@ -3956,7 +3978,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); - releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf); + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf, &pAPI->stateStore); continue; } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, @@ -4736,6 +4758,8 @@ _error: static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { @@ -4757,8 +4781,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { - streamStateCommit(pInfo->pState); - streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); + pAPI->stateStore.streamStateCommit(pInfo->pState); + pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; } @@ -4900,6 +4924,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pOperator->pTaskInfo = pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredDataSaved = false; pInfo->isFinal = false; @@ -4912,9 +4938,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + pInfo->pState = taosMemoryCalloc(1, sizeof(void)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); - streamStateSetNumber(pInfo->pState, -1); + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, @@ -4952,7 +4978,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; int32_t funResSize= getMaxFunResSize(pSup, numOfCols); - pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, + pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index f23b4d3e87..39ebee4fb1 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -33,7 +33,6 @@ target_link_libraries( PRIVATE qcom PRIVATE scalar PRIVATE transport - PRIVATE stream ${LINK_JEMALLOC} PUBLIC uv_a ) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1a43802e6b..f83873848e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,7 +18,7 @@ #include "function.h" #include "query.h" #include "querynodes.h" -#include "streamState.h" +//#include "streamState.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" @@ -3155,9 +3155,9 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, releaseBufPage(pHandle->pBuf, pPage); } else { // other tuple save policy - if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { - p.streamTupleKey = *key; - } +// if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { +// p.streamTupleKey = *key; +// } } *pPos = p; @@ -3192,7 +3192,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { - streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); +// streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); } return TSDB_CODE_SUCCESS; @@ -3217,7 +3217,7 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo } else { void* value = NULL; int32_t vLen; - streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); +// streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); return (char*)value; } } diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 0c2ce37c40..8ef134444f 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common - PUBLIC vnode +# PUBLIC vnode PUBLIC nodes PUBLIC scalar PUBLIC function diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 02ed0d2d05..7b3013362d 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -20,7 +20,7 @@ #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" -#include "vnode.h" +#include "tsdstorage.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) @@ -79,6 +79,7 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; + SStoreAPI api; } SIFParam; typedef struct SIFCtx { @@ -659,7 +660,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP } else { if (sifSetFltParam(left, right, &typedata, ¶m) != 0) return -1; } - ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); + ret = left->api.metaFilterTableIds(arg->metaEx, ¶m, output->result); } return ret; } diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt index 193a6971e5..30c68cb512 100644 --- a/source/libs/scalar/CMakeLists.txt +++ b/source/libs/scalar/CMakeLists.txt @@ -14,6 +14,7 @@ target_link_libraries(scalar PRIVATE nodes PRIVATE function PRIVATE qcom + PRIVATE parser ) if(${BUILD_TEST}) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 0f39cf817b..1cbd7b042c 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,7 +17,6 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" -// #include "streamInc.h" #include "streamState.h" #include "tcoding.h" #include "tcommon.h" diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 16ba81c74a..0015fea10f 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -214,8 +214,7 @@ int streamGetInit(const char* funcName); // |key|-----value------| // |key|ttl|len|userData| -static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, - rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); +static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, void** snapshot, void** readOpt); int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int ret = memcmp(aBuf, bBuf, aLen); @@ -799,10 +798,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; pState->pTdbState->rocksdb = inst->db; - pState->pTdbState->pHandle = inst->pHandle; + pState->pTdbState->pHandle = (void**)inst->pHandle; pState->pTdbState->writeOpts = inst->wOpt; pState->pTdbState->readOpts = inst->rOpt; - pState->pTdbState->cfOpts = inst->cfOpt; + pState->pTdbState->cfOpts = (void**)inst->cfOpt; pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->param = inst->param; pState->pTdbState->pBackendHandle = handle; @@ -810,6 +809,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { taosThreadMutexUnlock(&handle->cfMutex); return 0; } + taosThreadMutexUnlock(&handle->cfMutex); char* err = NULL; @@ -850,10 +850,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { } } pState->pTdbState->rocksdb = handle->db; - pState->pTdbState->pHandle = cfHandle; + pState->pTdbState->pHandle = (void**)cfHandle; pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); pState->pTdbState->readOpts = rocksdb_readoptions_create(); - pState->pTdbState->cfOpts = (rocksdb_options_t**)cfOpt; + pState->pTdbState->cfOpts = (void**)(rocksdb_options_t**)cfOpt; pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->param = param; pState->pTdbState->pBackendHandle = handle; @@ -954,8 +954,7 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len } return true; } -rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, - rocksdb_readoptions_t** readOpt) { +rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, void** snapshot, void** readOpt) { int idx = streamGetInit(cfName); if (snapshot != NULL) { @@ -1896,7 +1895,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co rocksdb_snapshot_t* snapshot = NULL; rocksdb_readoptions_t* readopts = NULL; - rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); + rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", (void**)&snapshot, (void**)&readopts); if (pIter == NULL) { return -1; } @@ -1966,6 +1965,7 @@ char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) { } return dst; } + // batch func void* streamStateCreateBatch() { rocksdb_writebatch_t* pBatch = rocksdb_writebatch_create(); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index ad66bb5a27..f531f65565 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -21,6 +21,7 @@ #include "tcommon.h" #include "thash.h" #include "tsimplehash.h" +#include "storageapi.h" #define FLUSH_RATIO 0.5 #define FLUSH_NUM 4 From c66c0c4dfd415b0815dba3dce4fc7d44980be5ee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:42:52 +0800 Subject: [PATCH 04/25] refactor: do some internal refactor. --- include/libs/executor/storageapi.h | 7 +++---- include/libs/index/index.h | 1 - include/libs/scalar/filter.h | 10 ---------- source/libs/executor/src/aggregateoperator.c | 1 - source/libs/executor/src/sysscanoperator.c | 1 - source/libs/index/src/indexFilter.c | 1 - 6 files changed, 3 insertions(+), 18 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 9b53f11ad9..7bcf6f8fd4 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -18,7 +18,6 @@ #include "tsimplehash.h" #include "tscalablebf.h" -//#include "tdb.h" #include "taosdef.h" #include "tmsg.h" #include "tcommon.h" @@ -35,8 +34,6 @@ extern "C" { #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 -typedef struct SMeta SMeta; - typedef struct SMetaEntry { int64_t version; int8_t type; @@ -138,8 +135,10 @@ typedef struct SMetaTableInfo { char tbName[TSDB_TABLE_NAME_LEN]; } SMetaTableInfo; +typedef struct SMeta SMeta; + typedef struct SSnapContext { - SMeta * pMeta; + SMeta * pMeta; // todo remove it int64_t snapVersion; struct TBC *pCur; int64_t suid; diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 9260ec4b1e..fd71c531fa 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -21,7 +21,6 @@ #include "taoserror.h" #include "tarray.h" #include "tglobal.h" -#include "tsdstorage.h" #ifdef __cplusplus extern "C" { diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 7c5ca628cd..f20ba287de 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -55,16 +55,6 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, SNode **pOtherCond); -typedef struct SMetaFltParam { - tb_uid_t suid; - int16_t cid; - int16_t type; - void *val; - bool reverse; - bool equal; - int (*filterFunc)(void *a, void *b, int16_t type); -} SMetaFltParam; - #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 918038635b..00aa52b593 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -21,7 +21,6 @@ #include "tname.h" #include "executorInt.h" -#include "tsdstorage.h" #include "operator.h" #include "query.h" #include "querytask.h" diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 460373b537..5bbfb188ed 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -32,7 +32,6 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "tsdstorage.h" #include "index.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 2dce931ad4..b90959579d 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -21,7 +21,6 @@ #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" -#include "tsdstorage.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) From a63473b1cc5688c89e82542cd73a487d58bd01be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:54:32 +0800 Subject: [PATCH 05/25] refacotor: do some internal refactor. --- include/libs/executor/storageapi.h | 125 ++++++++++----------- source/libs/executor/inc/executorInt.h | 6 +- source/libs/executor/src/executor.c | 22 ++-- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/operator.c | 4 +- source/libs/executor/src/scanoperator.c | 84 +++++++------- source/libs/executor/src/sysscanoperator.c | 6 +- 7 files changed, 120 insertions(+), 129 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 7bcf6f8fd4..67c0eacd26 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -34,6 +34,11 @@ extern "C" { #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 +#define META_READER_NOLOCK 0x1 + +typedef struct SMeta SMeta; +typedef TSKEY (*GetTsFun)(void*); + typedef struct SMetaEntry { int64_t version; int8_t type; @@ -70,6 +75,32 @@ typedef struct SMetaEntry { uint8_t *pBuf; } SMetaEntry; +typedef struct SMetaReader { + int32_t flags; + void * pMeta; + SDecoder coder; + SMetaEntry me; + void * pBuf; + int32_t szBuf; + struct SStorageAPI *storageAPI; +} SMetaReader; + +typedef struct SMTbCursor { + struct TBC *pDbc; + void * pKey; + void * pVal; + int32_t kLen; + int32_t vLen; + SMetaReader mr; +} SMTbCursor; + +typedef struct SRowBuffPos { + void* pRowBuff; + void* pKey; + bool beFlushed; + bool beUsed; +} SRowBuffPos; + // int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); // int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, // SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); @@ -135,8 +166,6 @@ typedef struct SMetaTableInfo { char tbName[TSDB_TABLE_NAME_LEN]; } SMetaTableInfo; -typedef struct SMeta SMeta; - typedef struct SSnapContext { SMeta * pMeta; // todo remove it int64_t snapVersion; @@ -162,23 +191,18 @@ typedef struct { // int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); // bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); // bool tqCurrentBlockConsumed(const STqReader* pReader); -// // int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); - // int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); // SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); // int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); // int32_t destroySnapContext(SSnapContext *ctx); - // SMTbCursor *metaOpenTbCursor(SMeta *pMeta); // void metaCloseTbCursor(SMTbCursor *pTbCur); // int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); // int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); -#define META_READER_NOLOCK 0x1 - /*-------------------------------------------------new api format---------------------------------------------------*/ // typedef int32_t (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num); @@ -201,7 +225,7 @@ typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *p int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, SHashObj **pIgnoreTables); -typedef struct SStoreDataReaderFn { +typedef struct SStoreTSDReader { __store_reader_open_fn_t storeReaderOpen; void (*storeReaderClose)(); void (*setReaderId)(void *pReader, const char *pId); @@ -216,7 +240,7 @@ typedef struct SStoreDataReaderFn { void (*storeReaderGetDataBlockDistInfo)(); void (*storeReaderGetNumOfInMemRows)(); void (*storeReaderNotifyClosing)(); -} SStoreDataReaderFn; +} SStoreTSDReader; /** * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); @@ -226,14 +250,14 @@ int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32 SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); */ -typedef struct SStoreCachedDataReaderFn { +typedef struct SStoreCacheReader { int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); void *(*closeReader)(void *pReader); int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUidList); void (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); -} SStoreCachedDataReaderFn; +} SStoreCacheReader; /*------------------------------------------------------------------------------------------------------------------*/ /* @@ -259,7 +283,7 @@ SWalReader* tqGetWalReader(STqReader* pReader); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); */ // todo rename -typedef struct SStoreTqReaderFn { +typedef struct SStoreTqReader { void *(*tqReaderOpen)(); void (*tqReaderClose)(); @@ -282,7 +306,7 @@ typedef struct SStoreTqReaderFn { int32_t (*tqReaderSetSubmitMsg)(); // todo remove it void (*tqReaderNextBlockFilterOut)(); -} SStoreTqReaderFn; +} SStoreTqReader; typedef struct SStoreSnapshotFn { /* @@ -291,10 +315,10 @@ typedef struct SStoreSnapshotFn { int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); */ - int32_t (*storeCreateSnapshot)(); - void (*storeDestroySnapshot)(); - SMetaTableInfo (*storeSSGetTableInfo)(); - int32_t (*storeSSGetMetaInfo)(); + int32_t (*createSnapshot)(); + void (*destroySnapshot)(); + SMetaTableInfo (*getTableInfoFromSnapshot)(); + int32_t (*getMetaInfoFromSnapshot)(); } SStoreSnapshotFn; /** @@ -322,17 +346,9 @@ int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ -typedef struct SMetaReader { - int32_t flags; - void * pMeta; - SDecoder coder; - SMetaEntry me; - void * pBuf; - int32_t szBuf; - struct SStorageAPI *storageAPI; -} SMetaReader; -typedef struct SStoreMetaReaderFn { + +typedef struct SStoreMetaReader { void (*initReader)(void *pReader, void *pMeta, int32_t flags); void *(*clearReader)(); @@ -341,9 +357,9 @@ typedef struct SStoreMetaReaderFn { int32_t (*getTableEntryByUid)(); int32_t (*getTableEntryByName)(); int32_t (*readerGetEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); -} SStoreMetaReaderFn; +} SStoreMetaReader; -typedef struct SStoreMetaFn { +typedef struct SStoreMeta { /* SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); @@ -398,7 +414,7 @@ int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); */ -} SStoreMetaFn; +} SStoreMeta; typedef struct STdbState { void* rocksdb; @@ -450,21 +466,14 @@ typedef struct SUpdateInfo { } SUpdateInfo; typedef struct { - void* iter; - void* snapshot; - void* readOpt; - void* db; -// rocksdb_iterator_t* iter; -// rocksdb_snapshot_t* snapshot; -// rocksdb_readoptions_t* readOpt; -// rocksdb_t* db; - - void* pCur; + void* iter; // rocksdb_iterator_t* iter; + void* snapshot; // rocksdb_snapshot_t* snapshot; + void* readOpt; // rocksdb_readoptions_t* readOpt; + void* db; // rocksdb_t* db; + void* pCur; int64_t number; } SStreamStateCur; -typedef TSKEY (*GetTsFun)(void*); - typedef struct SStateStore { int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal); @@ -541,33 +550,15 @@ typedef struct SStateStore { } SStateStore; typedef struct SStorageAPI { - SStoreMetaFn metaFn; // todo: refactor - SStoreDataReaderFn storeReader; - SStoreMetaReaderFn metaReaderFn; - SStoreCachedDataReaderFn cacheFn; - SStoreSnapshotFn snapshotFn; - SStoreTqReaderFn tqReaderFn; - SStateStore stateStore; + SStoreMeta metaFn; // todo: refactor + SStoreTSDReader tsdReader; + SStoreMetaReader metaReaderFn; + SStoreCacheReader cacheFn; + SStoreSnapshotFn snapshotFn; + SStoreTqReader tqReaderFn; + SStateStore stateStore; } SStorageAPI; -typedef struct SMTbCursor { - struct TBC *pDbc; - void * pKey; - void * pVal; - int32_t kLen; - int32_t vLen; - SMetaReader mr; -} SMTbCursor; - -typedef struct SRowBuffPos { - void* pRowBuff; - void* pKey; - bool beFlushed; - bool beUsed; -} SRowBuffPos; - - - #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 57b3bc3292..4c5ea4c242 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -208,7 +208,7 @@ typedef struct STableScanBase { SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. STableListInfo* pTableListInfo; - SStoreDataReaderFn readerAPI; + SStoreTSDReader readerAPI; } STableScanBase; typedef struct STableScanInfo { @@ -224,7 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - SStoreDataReaderFn readerAPI; + SStoreTSDReader readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -372,7 +372,7 @@ typedef struct SStreamScanInfo { int8_t igCheckUpdate; int8_t igExpired; void* pState; //void - SStoreTqReaderFn readerFn; + SStoreTqReader readerFn; SStateStore stateStore; } SStreamScanInfo; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e89db5b59c..aa51b52c80 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -166,7 +166,7 @@ void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) { if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - pAPI->storeReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); + pAPI->tsdReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); } } } else { @@ -1087,7 +1087,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__LOG) { // todo refactor: move away - pTaskInfo->storageAPI.storeReader.storeReaderClose(pScanBaseInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; ASSERT(0); @@ -1148,7 +1148,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = pTaskInfo->storageAPI.storeReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + int32_t code = pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); @@ -1159,8 +1159,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { - pTaskInfo->storageAPI.storeReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - pTaskInfo->storageAPI.storeReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + pTaskInfo->storageAPI.tsdReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + pTaskInfo->storageAPI.tsdReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } @@ -1182,14 +1182,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; - if (pAPI->snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { + if (pAPI->snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } - SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.storeSSGetTableInfo(sContext); - pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); + SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getTableInfoFromSnapshot(sContext); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1207,7 +1207,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); int32_t size = tableListGetSize(pTableListInfo); - pTaskInfo->storageAPI.storeReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, + pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1219,7 +1219,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; - if (pTaskInfo->storageAPI.snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) { + if (pTaskInfo->storageAPI.snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; @@ -1228,7 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; - pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index c11c5be1ae..240803e243 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1195,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) { qDebug("wait for the reader stopping"); } - pTaskInfo->storageAPI.storeReader.storeReaderClose(pTSInfo->base.dataReader); + pTaskInfo->storageAPI.tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; // restore the status, todo refactor. diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 1b85aedc1e..8d62b7a65f 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -239,7 +239,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, STableScanInfo* pInfo = pOperator->info; if (pInfo->base.dataReader != NULL) { - pAPI->storeReader.storeReaderNotifyClosing(pInfo->base.dataReader); + pAPI->tsdReader.storeReaderNotifyClosing(pInfo->base.dataReader); } return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -248,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, if (pInfo->pTableScanOp != NULL) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { - pAPI->storeReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ac8ca675ce..4bed2d3a5d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -267,7 +267,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SStorageAPI* pAPI = &pTaskInfo->storageAPI; bool allColumnsHaveAgg = true; - int32_t code = pAPI->storeReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + int32_t code = pAPI->tsdReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -350,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, @@ -358,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -368,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -390,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -405,7 +405,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); STableScanInfo* p1 = pOperator->info; if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) { @@ -419,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = pAPI->storeReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + SSDataBlock* p = pAPI->tsdReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (p == NULL) { return terrno; } @@ -693,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = pAPI->storeReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); } @@ -704,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (isTaskKilled(pTaskInfo)) { - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { - pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -774,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } @@ -782,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); - pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -800,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -839,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); - pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); + pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); - pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -858,7 +858,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = pAPI->storeReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, + int32_t code = pAPI->tsdReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -887,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); - pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); + pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; result = doGroupedTableScan(pOperator); @@ -910,7 +910,7 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanBase(STableScanBase* pBase, SStoreDataReaderFn* pAPI) { +static void destroyTableScanBase(STableScanBase* pBase, SStoreTSDReader* pAPI) { cleanupQueryTableDataCond(&pBase->cond); pAPI->storeReaderClose(pBase->dataReader); @@ -1094,7 +1094,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; - int32_t code = pAPI->storeReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + int32_t code = pAPI->tsdReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -1103,7 +1103,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = pAPI->storeReader.storeReaderNextDataBlock(pReader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1111,12 +1111,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ pAPI->storeReader.storeReaderRetrieveDataBlock(pReader, NULL); + /*SSDataBlock* p = */ pAPI->tsdReader.storeReaderRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } - pAPI->storeReader.storeReaderClose(pReader); + pAPI->tsdReader.storeReaderClose(pReader); qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1657,7 +1657,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { return pResult; } STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); @@ -1821,7 +1821,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } - pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1901,7 +1901,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; @@ -2162,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = pAPI->storeReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); if (code) { - pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { - pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = pAPI->storeReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); + SSDataBlock* pBlock = pAPI->tsdReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2185,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return pBlock; } - SMetaTableInfo mtInfo = pAPI->snapshotFn.storeSSGetTableInfo(pInfo->sContext); + SMetaTableInfo mtInfo = pAPI->snapshotFn.getTableInfoFromSnapshot(pInfo->sContext); STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); @@ -2203,7 +2203,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { int32_t dataLen = 0; int16_t type = 0; int64_t uid = 0; - if (pAPI->snapshotFn.storeSSGetMetaInfo(sContext, &data, &dataLen, &type, &uid) < 0) { + if (pAPI->snapshotFn.getMetaInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { qError("tmqsnap getMetafromSnapShot error"); taosMemoryFreeClear(data); return NULL; @@ -2227,8 +2227,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; - pRawScan->pAPI->storeReader.storeReaderClose(pRawScan->dataReader); - pRawScan->pAPI->snapshotFn.storeDestroySnapshot(pRawScan->sContext); + pRawScan->pAPI->tsdReader.storeReaderClose(pRawScan->dataReader); + pRawScan->pAPI->snapshotFn.destroySnapshot(pRawScan->sContext); tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); } @@ -2662,7 +2662,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = pAPI->storeReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); + code = pAPI->tsdReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2674,9 +2674,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = pAPI->storeReader.storeReaderNextDataBlock(reader, &hasNext); + code = pAPI->tsdReader.storeReaderNextDataBlock(reader, &hasNext); if (code != 0) { - pAPI->storeReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, code); } @@ -2686,7 +2686,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { - pAPI->storeReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.storeReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2726,7 +2726,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, close reader", reader); if (!source->multiReader) { - pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2734,7 +2734,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (!source->multiReader) { - pAPI->storeReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2853,7 +2853,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { for (int32_t i = 0; i < numOfTable; ++i) { STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); blockDataDestroy(param->inputBlock); - pAPI->storeReader.storeReaderClose(param->dataReader); + pAPI->tsdReader.storeReaderClose(param->dataReader); param->dataReader = NULL; } taosArrayClear(pInfo->sortSourceParams); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 5bbfb188ed..0e2ea77b86 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2198,7 +2198,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - pAPI->storeReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + pAPI->tsdReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2229,7 +2229,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - pDistInfo->readHandle.api.storeReader.storeReaderClose(pDistInfo->pHandle); + pDistInfo->readHandle.api.tsdReader.storeReaderClose(pDistInfo->pHandle); tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2284,7 +2284,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = readHandle->api.storeReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.tsdReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; From 85afbf0d353ed460e502b067c7bda282922418e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 18:58:54 +0800 Subject: [PATCH 06/25] refactor: do some internal refactor. --- include/libs/executor/storageapi.h | 31 ++++---- source/libs/executor/inc/executorInt.h | 4 +- source/libs/executor/src/executor.c | 16 ++-- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/operator.c | 4 +- source/libs/executor/src/scanoperator.c | 88 +++++++++++----------- source/libs/executor/src/sysscanoperator.c | 6 +- 7 files changed, 76 insertions(+), 75 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 67c0eacd26..8c79ba0500 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -225,22 +225,23 @@ typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *p int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, SHashObj **pIgnoreTables); -typedef struct SStoreTSDReader { - __store_reader_open_fn_t storeReaderOpen; - void (*storeReaderClose)(); - void (*setReaderId)(void *pReader, const char *pId); - void (*storeReaderSetTableList)(); - int32_t (*storeReaderNextDataBlock)(); - int32_t (*storeReaderRetrieveBlockSMA)(); +typedef struct TsdReader { + __store_reader_open_fn_t tsdReaderOpen; + void (*tsdReaderClose)(); + void (*tsdSetReaderTaskId)(void *pReader, const char *pId); + void (*tsdSetQueryTableList)(); + int32_t (*tsdReaderNextDataBlock)(); - SSDataBlock *(*storeReaderRetrieveDataBlock)(); - void (*storeReaderReleaseDataBlock)(); + int32_t (*tsdReaderRetrieveBlockSMAInfo)(); + SSDataBlock *(*tsdReaderRetrieveDataBlock)(); - void (*storeReaderResetStatus)(); - void (*storeReaderGetDataBlockDistInfo)(); - void (*storeReaderGetNumOfInMemRows)(); - void (*storeReaderNotifyClosing)(); -} SStoreTSDReader; + void (*tsdReaderReleaseDataBlock)(); + + void (*tsdReaderResetStatus)(); + void (*tsdReaderGetDataBlockDistInfo)(); + void (*tsdReaderGetNumOfInMemRows)(); + void (*tsdReaderNotifyClosing)(); +} TsdReader; /** * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); @@ -551,7 +552,7 @@ typedef struct SStateStore { typedef struct SStorageAPI { SStoreMeta metaFn; // todo: refactor - SStoreTSDReader tsdReader; + TsdReader tsdReader; SStoreMetaReader metaReaderFn; SStoreCacheReader cacheFn; SStoreSnapshotFn snapshotFn; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 4c5ea4c242..eb585e7046 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -208,7 +208,7 @@ typedef struct STableScanBase { SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. STableListInfo* pTableListInfo; - SStoreTSDReader readerAPI; + TsdReader readerAPI; } STableScanBase; typedef struct STableScanInfo { @@ -224,7 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - SStoreTSDReader readerAPI; + TsdReader readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index aa51b52c80..73a6532f73 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -166,7 +166,7 @@ void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) { if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - pAPI->tsdReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str); + pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str); } } } else { @@ -1087,7 +1087,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__LOG) { // todo refactor: move away - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pScanBaseInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; ASSERT(0); @@ -1148,7 +1148,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + int32_t code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); @@ -1159,8 +1159,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { - pTaskInfo->storageAPI.tsdReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - pTaskInfo->storageAPI.tsdReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } @@ -1189,7 +1189,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getTableInfoFromSnapshot(sContext); - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1207,7 +1207,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); int32_t size = tableListGetSize(pTableListInfo); - pTaskInfo->storageAPI.tsdReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, + pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1228,7 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 240803e243..45478cf837 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1195,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) { qDebug("wait for the reader stopping"); } - pTaskInfo->storageAPI.tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; // restore the status, todo refactor. diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 8d62b7a65f..8521fb623e 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -239,7 +239,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, STableScanInfo* pInfo = pOperator->info; if (pInfo->base.dataReader != NULL) { - pAPI->tsdReader.storeReaderNotifyClosing(pInfo->base.dataReader); + pAPI->tsdReader.tsdReaderNotifyClosing(pInfo->base.dataReader); } return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -248,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, if (pInfo->pTableScanOp != NULL) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { - pAPI->tsdReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderNotifyClosing(pTableScanInfo->base.dataReader); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4bed2d3a5d..8c880bb9b1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -267,7 +267,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SStorageAPI* pAPI = &pTaskInfo->storageAPI; bool allColumnsHaveAgg = true; - int32_t code = pAPI->tsdReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -350,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, @@ -358,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -368,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -390,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -405,7 +405,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); STableScanInfo* p1 = pOperator->info; if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) { @@ -419,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = pAPI->tsdReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (p == NULL) { return terrno; } @@ -693,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = pAPI->tsdReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); } @@ -704,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (isTaskKilled(pTaskInfo)) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -774,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } @@ -782,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); - pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -800,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - pAPI->tsdReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -839,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); - pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1); + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); - pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -858,7 +858,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = pAPI->tsdReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, + int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -887,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pAPI->tsdReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num); - pAPI->tsdReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); + pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; result = doGroupedTableScan(pOperator); @@ -910,10 +910,10 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanBase(STableScanBase* pBase, SStoreTSDReader* pAPI) { +static void destroyTableScanBase(STableScanBase* pBase, TsdReader* pAPI) { cleanupQueryTableDataCond(&pBase->cond); - pAPI->storeReaderClose(pBase->dataReader); + pAPI->tsdReaderClose(pBase->dataReader); pBase->dataReader = NULL; if (pBase->matchInfo.pList != NULL) { @@ -1074,7 +1074,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.endVersion = version; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -1094,7 +1094,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; - int32_t code = pAPI->tsdReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + int32_t code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -1103,7 +1103,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = pAPI->tsdReader.storeReaderNextDataBlock(pReader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1111,12 +1111,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ pAPI->tsdReader.storeReaderRetrieveDataBlock(pReader, NULL); + /*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } - pAPI->tsdReader.storeReaderClose(pReader); + pAPI->tsdReader.tsdReaderClose(pReader); qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1249,7 +1249,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -1657,7 +1657,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { return pResult; } STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); @@ -1821,7 +1821,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } - pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1901,7 +1901,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - pAPI->tsdReader.storeReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; @@ -2162,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = pAPI->tsdReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(pInfo->dataReader, &hasNext); if (code) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { - pAPI->tsdReader.storeReaderReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = pAPI->tsdReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL); + SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2227,7 +2227,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; - pRawScan->pAPI->tsdReader.storeReaderClose(pRawScan->dataReader); + pRawScan->pAPI->tsdReader.tsdReaderClose(pRawScan->dataReader); pRawScan->pAPI->snapshotFn.destroySnapshot(pRawScan->sContext); tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); @@ -2662,7 +2662,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = pAPI->tsdReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); + code = pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2674,9 +2674,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = pAPI->tsdReader.storeReaderNextDataBlock(reader, &hasNext); + code = pAPI->tsdReader.tsdReaderNextDataBlock(reader, &hasNext); if (code != 0) { - pAPI->tsdReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, code); } @@ -2686,7 +2686,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { - pAPI->tsdReader.storeReaderReleaseDataBlock(reader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2726,7 +2726,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, close reader", reader); if (!source->multiReader) { - pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2734,7 +2734,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (!source->multiReader) { - pAPI->tsdReader.storeReaderClose(pInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); source->dataReader = NULL; } pInfo->base.dataReader = NULL; @@ -2853,7 +2853,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { for (int32_t i = 0; i < numOfTable; ++i) { STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); blockDataDestroy(param->inputBlock); - pAPI->tsdReader.storeReaderClose(param->dataReader); + pAPI->tsdReader.tsdReaderClose(param->dataReader); param->dataReader = NULL; } taosArrayClear(pInfo->sortSourceParams); @@ -2965,11 +2965,11 @@ void destroyTableMergeScanOperatorInfo(void* param) { for (int32_t i = 0; i < numOfTable; i++) { STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); blockDataDestroy(p->inputBlock); - pTableScanInfo->base.readerAPI.storeReaderClose(p->dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(p->dataReader); p->dataReader = NULL; } - pTableScanInfo->base.readerAPI.storeReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; taosArrayDestroy(pTableScanInfo->sortSourceParams); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 0e2ea77b86..76302a4eb4 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2198,7 +2198,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - pAPI->tsdReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2229,7 +2229,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - pDistInfo->readHandle.api.tsdReader.storeReaderClose(pDistInfo->pHandle); + pDistInfo->readHandle.api.tsdReader.tsdReaderClose(pDistInfo->pHandle); tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2284,7 +2284,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = readHandle->api.tsdReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; From 66477a28ca8552fae482a527bf631a7aec2d738d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 19:10:50 +0800 Subject: [PATCH 07/25] refactor: do some internal refactor. --- include/libs/executor/executor.h | 4 +- include/libs/executor/storageapi.h | 4 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 206 ++++++++++----------- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/executor.c | 6 +- source/libs/executor/src/querytask.c | 2 +- source/libs/executor/src/scanoperator.c | 22 +-- source/libs/executor/src/sysscanoperator.c | 26 +-- source/libs/qworker/src/qwMsg.c | 6 +- 9 files changed, 137 insertions(+), 141 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 7f86b8f5e2..c4c2c89fdb 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -42,7 +42,7 @@ typedef struct { typedef struct { void* tqReader; - void* meta; +// void* meta; void* config; void* vnode; void* mnd; @@ -54,7 +54,7 @@ typedef struct { int32_t numOfVgroups; void* sContext; // SSnapContext* - void* pStateBackend; + void* pStateBackend; struct SStorageAPI api; } SReadHandle; diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 8c79ba0500..c0fd6c34fc 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -243,6 +243,7 @@ typedef struct TsdReader { void (*tsdReaderNotifyClosing)(); } TsdReader; + /** * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, @@ -262,7 +263,6 @@ typedef struct SStoreCacheReader { /*------------------------------------------------------------------------------------------------------------------*/ /* - * void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); @@ -552,7 +552,7 @@ typedef struct SStateStore { typedef struct SStorageAPI { SStoreMeta metaFn; // todo: refactor - TsdReader tsdReader; + TsdReader tsdReader; SStoreMetaReader metaReaderFn; SStoreCacheReader cacheFn; SStoreSnapshotFn snapshotFn; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fe1ccf90c8..0f08b63294 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -19,23 +19,23 @@ #include "vnode.h" #include "vnodeInt.h" -static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) { int32_t code = 0; @@ -301,32 +301,32 @@ _exit: return code; } -int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { +int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg *pRsp) { void *ptr = NULL; void *pReq; int32_t len; int32_t ret; - if (version <= pVnode->state.applied) { - vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, + if (ver <= pVnode->state.applied) { + vError("vgId:%d, duplicate write request. ver: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), ver, pVnode->state.applied); terrno = TSDB_CODE_VND_DUP_REQUEST; return -1; } vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - version); + ver); ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); - ASSERT(pVnode->state.applied + 1 == version); + ASSERT(pVnode->state.applied + 1 == ver); - atomic_store_64(&pVnode->state.applied, version); + atomic_store_64(&pVnode->state.applied, ver); atomic_store_64(&pVnode->state.applyTerm, pMsg->info.conn.applyTerm); if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) { - if (tqCheckLogInWal(pVnode->pTq, version)) return 0; + if (tqCheckLogInWal(pVnode->pTq, ver)) return 0; } // skip header @@ -337,123 +337,123 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp switch (pMsg->msgType) { /* META */ case TDMT_VND_CREATE_STB: - if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_STB: - if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: - if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_TABLE: - if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_TABLE: - if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TABLE: - if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TTL_TABLE: - if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTtlTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_TRIM: - if (vnodeProcessTrimReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_SMA: - if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; break; case TDMT_VND_DELETE: - if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_BATCH_DEL: - if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; /* TQ */ case TDMT_VND_TMQ_SUBSCRIBE: - if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessSubscribeReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_DELETE_SUB: - if (tqProcessDeleteSubReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessDeleteSubReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } break; case TDMT_VND_TMQ_COMMIT_OFFSET: - if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessOffsetCommitReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; case TDMT_VND_TMQ_SEEK_TO_OFFSET: - if (tqProcessSeekReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessSeekReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; case TDMT_VND_TMQ_ADD_CHECKINFO: - if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessAddCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_DEL_CHECKINFO: - if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessDelCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_STREAM_TASK_DEPLOY: { - if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { + if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_DROP: { - if (tqProcessTaskDropReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessTaskDropReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_PAUSE: { - if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_RESUME: { - if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: { - if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessTaskRecover2Req(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_CHECK_RSP: { - if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessStreamTaskCheckRsp(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } } break; case TDMT_VND_ALTER_CONFIRM: needCommit = pVnode->config.hashChange; - if (vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp) < 0) { + if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) { goto _err; } break; case TDMT_VND_ALTER_CONFIG: - vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); + vnodeProcessAlterConfigReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_COMMIT: needCommit = true; break; case TDMT_VND_CREATE_INDEX: - vnodeProcessCreateIndexReq(pVnode, version, pReq, len, pRsp); + vnodeProcessCreateIndexReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_DROP_INDEX: - vnodeProcessDropIndexReq(pVnode, version, pReq, len, pRsp); + vnodeProcessDropIndexReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_COMPACT: - vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp); + vnodeProcessCompactVnodeReq(pVnode, ver, pReq, len, pRsp); goto _exit; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); @@ -461,18 +461,18 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } vTrace("vgId:%d, process %s request, code:0x%x index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), pRsp->code, - version); + ver); - walApplyVer(pVnode->pWal, version); + walApplyVer(pVnode->pWal, ver); - if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) { vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // commit if need if (needCommit) { - vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); + vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), ver); if (vnodeAsyncCommit(pVnode) < 0) { vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -489,8 +489,8 @@ _exit: return 0; _err: - vError("vgId:%d, process %s request failed since %s, version:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - tstrerror(terrno), version); + vError("vgId:%d, process %s request failed since %s, ver:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + tstrerror(terrno), ver); return -1; } @@ -514,7 +514,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return 0; } - SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { case TDMT_SCH_QUERY: case TDMT_SCH_MERGE_QUERY: @@ -603,7 +603,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { } extern int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now); -static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -624,7 +624,7 @@ _exit: return code; } -static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -650,7 +650,7 @@ end: return ret; } -static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder coder; @@ -667,7 +667,7 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p goto _err; } - if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) { + if (metaCreateSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; goto _err; } @@ -685,7 +685,7 @@ _err: return -1; } -static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SDecoder decoder = {0}; SEncoder encoder = {0}; int32_t rcode = 0; @@ -742,7 +742,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR } // do create table - if (metaCreateTable(pVnode->pMeta, version, pCreateReq, &cRsp.pMeta) < 0) { + if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) { if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { cRsp.code = TSDB_CODE_SUCCESS; } else { @@ -790,7 +790,7 @@ _exit: return rcode; } -static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -808,7 +808,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR return -1; } - if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) { + if (metaAlterSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; tDecoderClear(&dc); return -1; @@ -819,7 +819,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR return 0; } -static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropStbReq req = {0}; int32_t rcode = TSDB_CODE_SUCCESS; SDecoder decoder = {0}; @@ -839,7 +839,7 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pRe // process request tbUidList = taosArrayInit(8, sizeof(int64_t)); if (tbUidList == NULL) goto _exit; - if (metaDropSTable(pVnode->pMeta, version, &req, tbUidList) < 0) { + if (metaDropSTable(pVnode->pMeta, ver, &req, tbUidList) < 0) { rcode = terrno; goto _exit; } @@ -862,7 +862,7 @@ _exit: return 0; } -static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; @@ -887,7 +887,7 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pRe } // process - if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq, &vMetaRsp) < 0) { + if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) { vAlterTbRsp.code = terrno; tDecoderClear(&dc); rcode = -1; @@ -912,7 +912,7 @@ _exit: return 0; } -static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropTbBatchReq req = {0}; SVDropTbBatchRsp rsp = {0}; SDecoder decoder = {0}; @@ -946,7 +946,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq tb_uid_t tbUid = 0; /* code */ - ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid); + ret = metaDropTable(pVnode->pMeta, ver, pDropTbReq, tbUids, &tbUid); if (ret < 0) { if (pDropTbReq->igNotExists && terrno == TSDB_CODE_TDB_TABLE_NOT_EXIST) { dropTbRsp.code = TSDB_CODE_SUCCESS; @@ -1189,7 +1189,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { return code; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; terrno = 0; @@ -1203,7 +1203,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq void *pAllocMsg = NULL; SSubmitReq2Msg *pMsg = (SSubmitReq2Msg *)pReq; - if (0 == pMsg->version) { + if (0 == pMsg->ver) { code = vnodeSubmitReqConvertToSubmitReq2(pVnode, (SSubmitReq *)pMsg, pSubmitReq); if (TSDB_CODE_SUCCESS == code) { code = vnodeRebuildSubmitReqMsg(pSubmitReq, &pReq); @@ -1251,7 +1251,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) { if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) { code = TSDB_CODE_INVALID_MSG; - vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version); + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); goto _exit; } } @@ -1263,7 +1263,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq for (int32_t iRow = 0; iRow < nRow; ++iRow) { if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) { code = TSDB_CODE_INVALID_MSG; - vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version); + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); goto _exit; } } @@ -1350,7 +1350,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); // create table - if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { + if (metaCreateTable(pVnode->pMeta, ver, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { // create table success if (newTbUids == NULL && @@ -1376,7 +1376,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq // insert data int32_t affectedRows; - code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows); + code = tsdbInsertTableData(pVnode->pTsdb, ver, pSubmitTbData, &affectedRows); if (code) goto _exit; pSubmitRsp->affectedRows += affectedRows; @@ -1404,7 +1404,7 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); - tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); + tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } // clear @@ -1419,7 +1419,7 @@ _exit: return code; } -static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateTSmaReq req = {0}; SDecoder coder = {0}; @@ -1439,20 +1439,20 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void * goto _err; } - if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) { + if (tdProcessTSmaCreate(pVnode->pSma, ver, (const char *)&req) < 0) { if (pRsp) pRsp->code = terrno; goto _err; } tDecoderClear(&coder); vDebug("vgId:%d, success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode), - req.indexName, req.indexUid, version, req.tableUid); + req.indexName, req.indexUid, ver, req.tableUid); return 0; _err: tDecoderClear(&coder); vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", - TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr()); + TD_VID(pVnode), req.indexName, req.indexUid, ver, req.tableUid, terrstr()); return -1; } @@ -1468,28 +1468,28 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL); } -static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t version) { +static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) { int32_t code = TSDB_CODE_SUCCESS; vInfo("vgId:%d, trim meta of tables per hash range [%" PRIu32 ", %" PRIu32 "]. apply-index:%" PRId64, TD_VID(pVnode), - pVnode->config.hashBegin, pVnode->config.hashEnd, version); + pVnode->config.hashBegin, pVnode->config.hashEnd, ver); // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd] return code; } -static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { vInfo("vgId:%d, vnode handle msgType:alter-confirm, alter confim msg is processed", TD_VID(pVnode)); int32_t code = TSDB_CODE_SUCCESS; if (!pVnode->config.hashChange) { goto _exit; } - code = vnodeConsolidateAlterHashRange(pVnode, version); + code = vnodeConsolidateAlterHashRange(pVnode, ver); if (code < 0) { vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), - version); + ver); goto _exit; } pVnode->config.hashChange = false; @@ -1503,7 +1503,7 @@ _exit: return code; } -static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { bool walChanged = false; bool tsdbChanged = false; @@ -1606,7 +1606,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SBatchDeleteReq deleteReq; SDecoder decoder; tDecoderInit(&decoder, pReq, len); @@ -1626,7 +1626,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void int64_t uid = mr.me.uid; - int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, ver, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); if (code < 0) { terrno = code; vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64, @@ -1640,7 +1640,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; @@ -1661,7 +1661,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0)); for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) { - code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), + code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), pRes->skey, pRes->ekey); if (code) goto _err; } @@ -1682,7 +1682,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq _err: return code; } -static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -1698,7 +1698,7 @@ static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void tDecoderClear(&dc); return -1; } - if (metaAddIndexToSTable(pVnode->pMeta, version, &req) < 0) { + if (metaAddIndexToSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; goto _err; } @@ -1708,7 +1708,7 @@ _err: tDecoderClear(&dc); return -1; } -static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SDropIndexReq req = {0}; pRsp->msgType = TDMT_VND_DROP_INDEX_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -1720,21 +1720,21 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p return -1; } - if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) { + if (metaDropIndexFromSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; return -1; } return TSDB_CODE_SUCCESS; } -extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp); +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp); } #ifndef TD_ENTERPRISE -int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } #endif diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ec23f7b724..b24fcade79 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2055,7 +2055,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest, pAPI); + code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 73a6532f73..cad98348d0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -340,7 +340,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); @@ -372,7 +372,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S if (pScanInfo->pTagCond != NULL) { bool qualified = false; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; - code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified, pAPI); + code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.vnode, &qualified, pAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); continue; @@ -437,7 +437,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI if (assignUid) { keyInfo.groupId = keyInfo.uid; } else { - code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, + code = getGroupIdFromTagsVal(pScanInfo->readHandle.vnode, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, &keyInfo.groupId, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(keyBuf); diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index a48de1d158..a684a6bf94 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -122,7 +122,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo SStorageAPI* pAPI = &pTaskInfo->storageAPI; - pAPI->metaReaderFn.initReader(&mr, pHandle->meta, 0); + pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0); int32_t code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8c880bb9b1..23e029e780 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -531,7 +531,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. @@ -560,7 +560,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -2556,7 +2556,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { char str[512] = {0}; int32_t count = 0; SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); @@ -3396,7 +3396,7 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca pRes->info.id.groupId = groupId; int64_t dbTableCount = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.meta, &dbTableCount); + pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &dbTableCount); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } @@ -3410,18 +3410,18 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) { tb_uid_t uid = 0; - pAPI->metaFn.getTableUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter, &uid); + pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); SMetaStbStats stats = {0}; ASSERT(0); -// metaGetStbStats(pInfo->readHandle.meta, uid, &stats); +// metaGetStbStats(pInfo->readHandle.vnode, uid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } @@ -3437,7 +3437,7 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.meta); + int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.vnode); ASSERT(0); if (ntbNum != 0) { fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); @@ -3448,7 +3448,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { char stbName[TSDB_TABLE_NAME_LEN] = {0}; ASSERT(0); -// metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); +// metaGetTableSzNameByUid(pInfo->readHandle.vnode, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { @@ -3461,7 +3461,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S pRes->info.id.groupId = groupId; SMetaStbStats stats = {0}; -// metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); +// metaGetStbStats(pInfo->readHandle.vnode, stbUid, &stats); int64_t ctbNum = stats.ctbNum; fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 76302a4eb4..a9450fb183 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -466,7 +466,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -486,7 +486,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (smrTable.me.type == TSDB_CHILD_TABLE) { int64_t suid = smrTable.me.ctbEntry.suid; pAPI->metaReaderFn.clearReader(&smrTable); - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -522,7 +522,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); } if (pInfo->pSchema == NULL) { @@ -569,7 +569,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -658,7 +658,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, condTableName); SMetaReader smrChildTable = {0}; - pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -676,7 +676,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK); code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid @@ -702,7 +702,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); } bool blockFull = false; @@ -715,7 +715,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { @@ -1131,7 +1131,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); if (ret < 0) { pAPI->metaReaderFn.clearReader(&mr); @@ -1159,7 +1159,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; - pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK); int64_t suid = mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); @@ -1292,7 +1292,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); } blockDataCleanup(pInfo->pRes); @@ -1338,7 +1338,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); @@ -1486,7 +1486,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } else { if (pInfo->showRewrite == false) { if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; + SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode}; SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); idx->init = 0; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 1a3a740b34..231e597724 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -440,11 +440,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int int64_t rId = msg.refId; int32_t eId = msg.execId; - SQWMsg qwMsg = {.node = node, - .msg = msg.msg, - .msgLen = msg.msgLen, - .connInfo = pMsg->info, - .msgType = pMsg->msgType}; + SQWMsg qwMsg = {.node = node, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType}; qwMsg.msgInfo.explain = msg.explain; qwMsg.msgInfo.taskType = msg.taskType; qwMsg.msgInfo.needFetch = msg.needFetch; From 39a1fa8f782c8be0c4c00d6fc5f0ef4353f046f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 May 2023 19:20:25 +0800 Subject: [PATCH 08/25] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/meta/metaSnapshot.c | 6 +++--- source/dnode/vnode/src/sma/smaRollup.c | 14 ++------------ source/dnode/vnode/src/tq/tq.c | 10 ++++------ source/dnode/vnode/src/tq/tqMeta.c | 7 +++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 6 files changed, 15 insertions(+), 28 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ddaf0d9bbc..f6d731f2b4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -291,7 +291,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot); int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); -int32_t buildSnapContext(SMeta *pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, SSnapContext **ctxRet); int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index d3b6471282..ff4355fdf8 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -260,12 +260,12 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) taosHashPut(suidInfo, &me->uid, sizeof(tb_uid_t), &dataTmp, sizeof(STableInfoForChildTable)); } -int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, SSnapContext** ctxRet) { SSnapContext* ctx = taosMemoryCalloc(1, sizeof(SSnapContext)); if (ctx == NULL) return -1; *ctxRet = ctx; - ctx->pMeta = pMeta; + ctx->pMeta = pVnode->pMeta; ctx->snapVersion = snapVersion; ctx->suid = suid; ctx->subType = subType; @@ -301,7 +301,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t continue; } - if (tdbTbGet(pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) < + if (tdbTbGet(ctx->pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) < 0) { // check if table exist for now, need optimize later continue; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ccc00ce25e..f66f85cc88 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -276,12 +276,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat return TSDB_CODE_FAILED; } - SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTqReader = 1, - .pStateBackend = pStreamState, - }; + SReadHandle handle = { .vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState }; pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; @@ -853,12 +848,7 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len); TSDB_CHECK_CODE(code, lino, _exit); - SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTqReader = 1, - }; - + SReadHandle handle = { .vnode = pVnode, .initTqReader = 1 }; if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { code = TSDB_CODE_APP_ERROR; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 74361ec319..6befccdab2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -670,8 +670,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int64_t ver = pRef->refVer; pHandle->pRef = pRef; - SReadHandle handle = { - .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; + SReadHandle handle = {.vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -689,7 +688,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, + buildSnapContext(handle.vnode, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); @@ -708,7 +707,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, + buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } @@ -787,8 +786,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { return -1; } - SReadHandle handle = { - .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + SReadHandle handle = {.vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); if (pTask->exec.pExecutor == NULL) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 5654147b6d..d5cfa86e4c 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -298,11 +298,10 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { walSetRefVer(handle.pRef, handle.snapshotVer); SReadHandle reader = { - .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTableReader = true, .initTqReader = true, - .version = handle.snapshotVer, + .version = handle.snapshotVer }; if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -330,7 +329,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, + buildSnapContext(reader.vnode, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { @@ -347,7 +346,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, + buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0f08b63294..268d51cd16 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -250,7 +250,7 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { uint8_t *pCont; SEncoder *pCoder = &(SEncoder){0}; SDeleteRes res = {0}; - SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); if (code) goto _exit; @@ -1203,7 +1203,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in void *pAllocMsg = NULL; SSubmitReq2Msg *pMsg = (SSubmitReq2Msg *)pReq; - if (0 == pMsg->ver) { + if (0 == pMsg->version) { code = vnodeSubmitReqConvertToSubmitReq2(pVnode, (SSubmitReq *)pMsg, pSubmitReq); if (TSDB_CODE_SUCCESS == code) { code = vnodeRebuildSubmitReqMsg(pSubmitReq, &pReq); From 1f7f326bedeee288a89c23ecade532d2ce3736d7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 13:22:05 +0800 Subject: [PATCH 09/25] refactor: do some internal refactor and set the api function ptr. --- include/libs/executor/storageapi.h | 134 ++++++-------- include/libs/stream/streamState.h | 2 +- source/dnode/vnode/inc/vnode.h | 30 +-- source/dnode/vnode/src/inc/vnodeInt.h | 4 + source/dnode/vnode/src/meta/metaQuery.c | 144 ++++---------- source/dnode/vnode/src/meta/metaSma.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 4 +- source/dnode/vnode/src/sma/smaRollup.c | 4 +- source/dnode/vnode/src/tq/tq.c | 5 +- source/dnode/vnode/src/tq/tqRead.c | 8 +- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 175 ++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 42 +++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 + source/libs/executor/inc/querytask.h | 2 +- source/libs/executor/src/executil.c | 9 +- source/libs/executor/src/executor.c | 7 +- source/libs/executor/src/querytask.c | 10 +- source/libs/executor/src/scanoperator.c | 36 ++-- source/libs/executor/src/sysscanoperator.c | 33 ++-- source/libs/executor/src/timewindowoperator.c | 4 +- source/libs/stream/src/streamState.c | 29 +-- 24 files changed, 413 insertions(+), 281 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index c0fd6c34fc..8790c302ce 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -136,8 +136,8 @@ typedef struct SRowBuffPos { // void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); // void metaReaderReleaseLock(SMetaReader *pReader); // void metaReaderClear(SMetaReader *pReader); -// int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -// int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +// int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +// int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); // int metaGetTableEntryByName(SMetaReader *pReader, const char *name); // int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); // int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); @@ -191,17 +191,13 @@ typedef struct { // int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); // bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); // bool tqCurrentBlockConsumed(const STqReader* pReader); -// int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +// int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); // int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); // SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); // int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); // int32_t destroySnapContext(SSnapContext *ctx); -// SMTbCursor *metaOpenTbCursor(SMeta *pMeta); -// void metaCloseTbCursor(SMTbCursor *pTbCur); -// int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -// int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); /*-------------------------------------------------new api format---------------------------------------------------*/ @@ -229,17 +225,17 @@ typedef struct TsdReader { __store_reader_open_fn_t tsdReaderOpen; void (*tsdReaderClose)(); void (*tsdSetReaderTaskId)(void *pReader, const char *pId); - void (*tsdSetQueryTableList)(); - int32_t (*tsdReaderNextDataBlock)(); + int32_t (*tsdSetQueryTableList)(); + int32_t (*tsdNextDataBlock)(); int32_t (*tsdReaderRetrieveBlockSMAInfo)(); SSDataBlock *(*tsdReaderRetrieveDataBlock)(); void (*tsdReaderReleaseDataBlock)(); - void (*tsdReaderResetStatus)(); - void (*tsdReaderGetDataBlockDistInfo)(); - void (*tsdReaderGetNumOfInMemRows)(); + int32_t (*tsdReaderResetStatus)(); + int32_t (*tsdReaderGetDataBlockDistInfo)(); + int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); } TsdReader; @@ -270,13 +266,13 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); bool tqCurrentBlockConsumed(const STqReader* pReader); -int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); bool tqNextBlockInWal(STqReader* pReader, const char* idstr); bool tqNextBlockImpl(STqReader *pReader, const char* idstr); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr); STqReader *tqReaderOpen(void *pVnode); -void tqCloseReader(STqReader *); +void tqReaderClose(STqReader *); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); @@ -285,7 +281,7 @@ int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas */ // todo rename typedef struct SStoreTqReader { - void *(*tqReaderOpen)(); + struct STqReader* (*tqReaderOpen)(); void (*tqReaderClose)(); int32_t (*tqReaderSeek)(); @@ -294,7 +290,7 @@ typedef struct SStoreTqReader { bool (*tqNextBlockImpl)(); // todo remove it void (*tqReaderSetColIdList)(); - int32_t (*tqReaderSetTargetTableList)(); + int32_t (*tqReaderSetQueryTableList)(); int32_t (*tqReaderAddTables)(); int32_t (*tqReaderRemoveTables)(); @@ -303,10 +299,10 @@ typedef struct SStoreTqReader { bool (*tqReaderCurrentBlockConsumed)(); struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it - void (*tqReaderRetrieveTaosXBlock)(); // todo remove it + int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it - void (*tqReaderNextBlockFilterOut)(); + bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { @@ -326,8 +322,8 @@ typedef struct SStoreSnapshotFn { void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); -int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); @@ -347,39 +343,29 @@ int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ - - typedef struct SStoreMetaReader { - void (*initReader)(void *pReader, void *pMeta, int32_t flags); - void *(*clearReader)(); - - void (*readerReleaseLock)(); - - int32_t (*getTableEntryByUid)(); - int32_t (*getTableEntryByName)(); - int32_t (*readerGetEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); + void (*initReader)(SMetaReader *pReader, void *pMeta, int32_t flags); + void (*clearReader)(SMetaReader *pReader); + void (*readerReleaseLock)(SMetaReader *pReader); + int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); + int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); + int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); } SStoreMetaReader; typedef struct SStoreMeta { - /* -SMTbCursor *metaOpenTbCursor(SMeta *pMeta); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); - */ - void *(*openMetaCursor)(); - void (*closeMetaCursor)(); - int32_t (*cursorNext)(); - void (*cursorPrev)(); + SMTbCursor *(*openTableMetaCursor)(); // metaOpenTbCursor + void (*closeTableMetaCursor)(); // metaCloseTbCursor + int32_t (*cursorNext)(); // metaTbCursorNext + int32_t (*cursorPrev)(); // metaTbCursorPrev int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); int32_t (*getTableTagsByUid)(); - const char *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it + const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); - bool (*isTableExisted)(void *pVnode, uint64_t uid); + bool (*isTableExisted)(void *pVnode, tb_uid_t uid); /** * int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, @@ -397,8 +383,8 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in * */ void *(*storeGetIndexInfo)(); - void *(*storeGetInvertIndex)(); - void (*storeGetChildTableList)(); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + void *(*getInvertIndex)(void* pVnode); + int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] int32_t (*storeGetTableList)(); // vnodeGetStbIdList & vnodeGetAllTableList void *storeGetVersionRange; void *storeGetLastTimestamp; @@ -406,8 +392,8 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema // db name, vgId, numOfTables, numOfSTables - void (*storeGetNumOfChildTables)(); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); - void (*storeGetBasicInfo)(); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); + int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); int64_t (*getNumOfRowsInMem)(); /** @@ -497,50 +483,50 @@ typedef struct SStateStore { int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); - int32_t (*streamStateCurNext)(SStreamState* pState, void* pCur); - int32_t (*streamStateCurPrev)(SStreamState* pState, void* pCur); + int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); + int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); - void* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); - void* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); - void* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); - void* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); - void (*streamStateFreeCur)(void* pCur); + SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); + SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); + void (*streamStateFreeCur)(SStreamStateCur* pCur); - int32_t (*streamStateGetGroupKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); - int32_t (*streamStateGetKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + int32_t (*streamStateGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); int32_t (*streamStateSessionClear)(SStreamState* pState); - int32_t (*streamStateSessionGetKVByCur)(void* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); - int32_t (*streamStateSessionGetKeyByRange)(void* pState, const SSessionKey* range, SSessionKey* curKey); + int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); - void* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); - TSKEY (*updateInfoFillBlockData)(void *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); - bool (*updateInfoIsUpdated)(void *pInfo, uint64_t tableId, TSKEY ts); - bool (*updateInfoIsTableInserted)(void *pInfo, int64_t tbUid); - void (*updateInfoDestroy)(void *pInfo); + SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); + TSKEY (*updateInfoFillBlockData)(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(SUpdateInfo *pInfo, int64_t tbUid); + void (*updateInfoDestroy)(SUpdateInfo *pInfo); SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark); - void (*updateInfoAddCloseWindowSBF)(void *pInfo); - void (*updateInfoDestoryColseWinSBF)(void *pInfo); - int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const void *pInfo); - int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, void *pInfo); + void (*updateInfoAddCloseWindowSBF)(SUpdateInfo *pInfo); + void (*updateInfoDestoryColseWinSBF)(SUpdateInfo *pInfo); + int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); + int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, SUpdateInfo *pInfo); - void* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); - void* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); - void* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); - void* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, - void* pFile, TSKEY delMark); + struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, + uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark); - void (*streamFileStateDestroy)(void* pFileState); - void (*streamFileStateClear)(void* pFileState); - bool (*needClearDiskBuff)(void* pFileState); + void (*streamFileStateDestroy)(struct SStreamFileState* pFileState); + void (*streamFileStateClear)(struct SStreamFileState* pFileState); + bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); void (*streamStateClose)(SStreamState* pState, bool remove); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index f9fffef7c2..08357b8251 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -59,7 +59,7 @@ extern "C" { // TXN* txn; //} STdbState; -SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); +SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f6d731f2b4..95fe150443 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -67,17 +67,17 @@ int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); -void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeIsCatchUp(SVnode *pVnode); ESyncRole vnodeGetRole(SVnode *pVnode); -int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list); -void *vnodeGetIdx(SVnode *pVnode); -void *vnodeGetIvtIdx(SVnode *pVnode); +void *vnodeGetIdx(void *pVnode); +void *vnodeGetIvtIdx(void *pVnode); int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); @@ -105,22 +105,22 @@ typedef struct SMetaEntry SMetaEntry; #define META_READER_NOLOCK 0x1 -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); -int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); int32_t metaReadNext(SMetaReader *pReader); -const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); +const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +bool metaIsTableExist(void* pVnode, tb_uid_t uid); int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, bool *acquired); int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, @@ -138,7 +138,7 @@ int64_t metaGetNtbNum(SMeta *pMeta); // int64_t uid; // int64_t ctbNum; //} SMetaStbStats; -int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); //typedef struct SMetaFltParam { // tb_uid_t suid; @@ -163,7 +163,7 @@ typedef SVCreateTSmaReq SSmaCfg; typedef struct SMTbCursor SMTbCursor; -SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +SMTbCursor *metaOpenTbCursor(void *pVnode); void metaCloseTbCursor(SMTbCursor *pTbCur); int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); @@ -260,16 +260,20 @@ typedef struct STqReader { } STqReader; STqReader *tqReaderOpen(SVnode *pVnode); -void tqCloseReader(STqReader *); +void tqReaderClose(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); -int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +bool tqCurrentBlockConsumed(const STqReader* pReader); + +int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); bool tqNextBlockInWal(STqReader *pReader, const char *idstr); bool tqNextBlockImpl(STqReader *pReader, const char *idstr); +SWalReader* tqGetWalReader(STqReader* pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 230e8b8f88..735ff1700c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -161,6 +161,8 @@ void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); + int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -469,6 +471,8 @@ struct SCompactInfo { STimeWindow tw; }; +void initStorageAPI(SStorageAPI* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 48bfc5d86f..40f4b6fa67 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,6 +17,11 @@ #include "osMemory.h" #include "tencode.h" +void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + metaReaderInit(pReader, pMeta, flags); +} + void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->flags = flags; @@ -64,96 +69,20 @@ _err: return -1; } -// int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { -// -// SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); -// SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); -// SMeta *pMeta = meta; -// int64_t version; -// SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); -// -// int64_t stt1 = taosGetTimestampUs(); -// for(int i = 0; i < taosArrayGetSize(uidList); i++) { -// void* ppVal = NULL; -// int vlen = 0; -// uint64_t * uid = taosArrayGet(uidList, i); -// // query uid.idx -// if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { -// continue; -// } -// version = *(int64_t *)ppVal; -// -// STbDbKey tbDbKey = {.version = version, .uid = *uid}; -// taosArrayPush(uidVersion, &tbDbKey); -// taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); -// } -// int64_t stt2 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); -// -// TBC *pCur = NULL; -// tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); -// tdbTbcMoveToFirst(pCur); -// void *pKey = NULL; -// int kLen = 0; -// -// while(1){ -// SMetaReader pReader = {0}; -// int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); -// if (ret < 0) break; -// STbDbKey *tmp = (STbDbKey*)pKey; -// int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); -// if(ver == NULL || *ver != tmp->version) continue; -// taosArrayPush(readerList, &pReader); -// } -// tdbTbcClose(pCur); -// -// taosArrayClear(readerList); -// int64_t stt3 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); -// for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { -// SMetaReader pReader = {0}; -// -// STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); -// // query table.db -// if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { -// continue; -// } -// taosArrayPush(readerList, &pReader); -// } -// int64_t stt4 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); -// -// for(int i = 0; i < taosArrayGetSize(readerList); i++){ -// SMetaReader* pReader = taosArrayGet(readerList, i); -// metaReaderInit(pReader, meta, 0); -// // decode the entry -// tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); -// -// if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { -// } -// metaReaderClear(pReader); -// } -// int64_t stt5 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); -// return 0; -// } - -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { - // query uid.idx - metaRLock(pMeta); - - if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { - metaULock(pMeta); +bool metaIsTableExist(void *pVnode, tb_uid_t uid) { + SVnode* pVnodeObj = pVnode; + metaRLock(pVnodeObj->pMeta); // query uid.idx + if (tdbTbGet(pVnodeObj->pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { + metaULock(pVnodeObj->pMeta); return false; } - metaULock(pMeta); - + metaULock(pVnodeObj->pMeta); return true; } -int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { +int metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; int64_t version1; @@ -167,7 +96,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { return metaGetTableEntryByVersion(pReader, version1, uid); } -int metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) { +int metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; SMetaInfo info; @@ -190,7 +119,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { } uid = *(tb_uid_t *)pReader->pBuf; - return metaGetTableEntryByUid(pReader, uid); + return metaReaderGetTableEntryByUid(pReader, uid); } tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { @@ -214,7 +143,7 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - code = metaGetTableEntryByUid(&mr, uid); + code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); return -1; @@ -230,7 +159,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - code = metaGetTableEntryByUid(&mr, uid); + code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); return -1; @@ -283,7 +212,7 @@ int metaReadNext(SMetaReader *pReader) { } #if 1 // =================================================== -SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { +SMTbCursor *metaOpenTbCursor(void *pVnode) { SMTbCursor *pTbCur = NULL; pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); @@ -291,12 +220,12 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { return NULL; } - metaReaderInit(&pTbCur->mr, pMeta, 0); + SVnode* pVnodeObj = pVnode; + metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0); - tdbTbcOpen(pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); + tdbTbcOpen(pVnodeObj->pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); - return pTbCur; } @@ -876,7 +805,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { STSma *pTSma = NULL; for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); - if (metaGetTableEntryByUid(&mr, smaId) < 0) { + if (metaReaderGetTableEntryByUid(&mr, smaId) < 0) { tDecoderClear(&mr.coder); metaWarn("vgId:%d, no entry for tbId:%" PRIi64 ", smaId:%" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); continue; @@ -926,7 +855,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { STSma *pTSma = NULL; SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUid(&mr, indexUid) < 0) { + if (metaReaderGetTableEntryByUid(&mr, indexUid) < 0) { metaWarn("vgId:%d, failed to get table entry for smaId:%" PRIi64, TD_VID(pMeta->pVnode), indexUid); metaReaderClear(&mr); return NULL; @@ -1027,7 +956,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif -const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) { +const void *metaGetTableTagVal(const void *pTag, int16_t type, STagVal *val) { STag *tag = (STag *)pTag; if (type == TSDB_DATA_TYPE_JSON) { return tag; @@ -1565,30 +1494,35 @@ _exit: return code; } -int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo) { +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t* numOfTables) { int32_t code = 0; + *numOfTables = 0; - metaRLock(pMeta); + SVnode* pVnodeObj = pVnode; + metaRLock(pVnodeObj->pMeta); // fast path: search cache - if (metaStatsCacheGet(pMeta, uid, pInfo) == TSDB_CODE_SUCCESS) { - metaULock(pMeta); + SMetaStbStats state = {0}; + if (metaStatsCacheGet(pVnodeObj->pMeta, uid, &state) == TSDB_CODE_SUCCESS) { + metaULock(pVnodeObj->pMeta); + *numOfTables = state.ctbNum; goto _exit; } // slow path: search TDB int64_t ctbNum = 0; - vnodeGetCtbNum(pMeta->pVnode, uid, &ctbNum); + vnodeGetCtbNum(pVnode, uid, &ctbNum); - metaULock(pMeta); + metaULock(pVnodeObj->pMeta); + *numOfTables = ctbNum; - pInfo->uid = uid; - pInfo->ctbNum = ctbNum; + state.uid = uid; + state.ctbNum = ctbNum; // upsert the cache - metaWLock(pMeta); - metaStatsCacheUpsert(pMeta, pInfo); - metaULock(pMeta); + metaWLock(pVnodeObj->pMeta); + metaStatsCacheUpsert(pVnodeObj->pMeta, &state); + metaULock(pVnodeObj->pMeta); _exit: return code; diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 8d5821e28b..3fb491848a 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -36,7 +36,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { // validate req // save smaIndex metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) { + if (metaReaderGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) { #if 1 terrno = TSDB_CODE_TSMA_ALREADY_EXIST; metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0164a82c69..df16304e28 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -690,7 +690,7 @@ _err: return -1; } -int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { +int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { SMetaEntry me = {0}; SMetaReader mr = {0}; @@ -729,7 +729,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe metaReaderClear(&mr); // build SMetaEntry - me.version = version; + me.version = ver; me.type = pReq->type; me.uid = pReq->uid; me.name = pReq->name; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index f66f85cc88..558cfdac2b 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -894,7 +894,7 @@ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { metaReaderInit(&mr, SMA_META(pSma), 0); smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); - if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } @@ -1121,7 +1121,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { for (int64_t i = 0; i < arrSize; ++i) { suid = *(tb_uid_t *)taosArrayGet(suidList, i); smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid); - if (metaGetTableEntryByUidCache(&mr, suid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, suid) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6befccdab2..4adbf02302 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -69,12 +69,12 @@ static void destroyTqHandle(void* data) { if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { taosMemoryFreeClear(pData->execHandle.execCol.qmsg); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { - tqCloseReader(pData->execHandle.pTqReader); + tqReaderClose(pData->execHandle.pTqReader); walCloseReader(pData->pWalReader); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); - tqCloseReader(pData->execHandle.pTqReader); + tqReaderClose(pData->execHandle.pTqReader); } if(pData->msg != NULL) { rpcFreeCont(pData->msg->pCont); @@ -787,6 +787,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } SReadHandle handle = {.vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + initStorageAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); if (pTask->exec.pExecutor == NULL) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 84ead71fb5..2521d956a3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -273,7 +273,7 @@ STqReader* tqReaderOpen(SVnode* pVnode) { return pReader; } -void tqCloseReader(STqReader* pReader) { +void tqReaderClose(STqReader* pReader) { // close wal reader if (pReader->pWalReader) { walCloseReader(pReader->pWalReader); @@ -294,7 +294,7 @@ void tqCloseReader(STqReader* pReader) { taosMemoryFree(pReader); } -int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { +int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { if (walReaderSeekVer(pReader->pWalReader, ver) < 0) { return -1; } @@ -605,6 +605,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); SSDataBlock* pBlock = pReader->pResBlock; + *pRes = pBlock; + blockDataCleanup(pBlock); int32_t sversion = pSubmitTbData->sver; @@ -1084,7 +1086,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i); - int32_t code = metaGetTableEntryByUidCache(&mr, *id); + int32_t code = metaReaderGetTableEntryByUidCache(&mr, *id); if (code != TSDB_CODE_SUCCESS) { tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); continue; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index e268199e16..98304f9c18 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -51,7 +51,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in metaReaderInit(&mr, pTq->pVnode->pMeta, 0); // TODO add reference to gurantee success - if (metaGetTableEntryByUidCache(&mr, uid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index c0a8de5743..2d8b7f9ecf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -980,7 +980,7 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) { SMetaReader mr = {0}; metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, uid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); // table not esist return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cde2672541..414ccfc8b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5376,7 +5376,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); - int32_t code = metaGetTableEntryByUidCache(&mr, uid); + int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; metaReaderClear(&mr); @@ -5389,7 +5389,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 if (mr.me.type == TSDB_CHILD_TABLE) { tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; - code = metaGetTableEntryByUidCache(&mr, *suid); + code = metaReaderGetTableEntryByUidCache(&mr, *suid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b5e7c6875b..ebfecc4c15 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -14,6 +14,7 @@ */ #include "vnd.h" +#include "tstreamUpdate.h" int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { SVnodeInfo info = {0}; @@ -444,3 +445,177 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyTerm = pVnode->state.commitTerm; pSnapshot->lastConfigIndex = -1; } + +static void initTsdbReaderAPI(TsdReader* pReader); +static void initMetadataAPI(SStoreMeta* pMeta); +static void initTqAPI(SStoreTqReader* pTq); +static void initStateStoreAPI(SStateStore* pStore); +static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); + +void initStorageAPI(SStorageAPI* pAPI) { + initTsdbReaderAPI(&pAPI->tsdReader); + initMetadataAPI(&pAPI->metaFn); + initTqAPI(&pAPI->tqReaderFn); + initStateStoreAPI(&pAPI->stateStore); + initMetaReaderAPI(&pAPI->metaReaderFn); +} + +void initTsdbReaderAPI(TsdReader* pReader) { + pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen; + pReader->tsdReaderClose = tsdbReaderClose; + + pReader->tsdNextDataBlock = tsdbNextDataBlock; + + pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock; + pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock; + + pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA; + + pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; + pReader->tsdReaderResetStatus = tsdbReaderReset; + + pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away + + pReader->tsdSetQueryTableList = tsdbSetTableList; + pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; +} + +void initMetadataAPI(SStoreMeta* pMeta) { + pMeta->isTableExisted = metaIsTableExist; + + pMeta->openTableMetaCursor = metaOpenTbCursor; + pMeta->closeTableMetaCursor = metaCloseTbCursor; + pMeta->cursorNext = metaTbCursorNext; + pMeta->cursorPrev = metaTbCursorPrev; + + pMeta->getBasicInfo = vnodeGetInfo; + pMeta->getNumOfChildTables = metaGetStbStats; + + pMeta->getChildTableList = vnodeGetCtbIdList; + + pMeta->storeGetIndexInfo = vnodeGetIdx; + pMeta->getInvertIndex = vnodeGetIvtIdx; + + pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + +} + +void initTqAPI(SStoreTqReader* pTq) { + pTq->tqReaderOpen = tqReaderOpen; + pTq->tqReaderSetColIdList = tqReaderSetColIdList; + + pTq->tqReaderClose = tqReaderClose; + pTq->tqReaderSeek = tqReaderSeek; + pTq->tqRetrieveBlock = tqRetrieveDataBlock; + + pTq->tqReaderNextBlockInWal = tqNextBlockInWal; + + pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it + + pTq->tqReaderAddTables = tqReaderAddTbUidList; + pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; + + pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList; + + pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; + pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; + + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + + pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; +} + +void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { + pMetaReader->initReader = _metaReaderInit; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; + pMetaReader->getTableEntryByName = metaGetTableEntryByName; + + pMetaReader->readerReleaseLock = metaReaderReleaseLock; +} + diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 303d2a9ca4..ac3e632172 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -80,7 +80,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { metaReaderInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); - if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(metaRsp.stbName, mer2.me.name); metaRsp.suid = mer2.me.uid; @@ -189,7 +189,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { goto _exit; } else if (mer1.me.type == TSDB_CHILD_TABLE) { metaReaderInit(&mer2, pVnode->pMeta, 0); - if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(cfgRsp.stbName, mer2.me.name); schema = mer2.me.stbEntry.schemaRow; @@ -410,13 +410,24 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { "nBatchInsertSuccess"); } -void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) { + SVnode* pVnodeObj = pVnode; + SVnodeCfg* pConf = &pVnodeObj->config; + if (dbname) { - *dbname = pVnode->config.dbname; + *dbname = pConf->dbname; } if (vgId) { - *vgId = TD_VID(pVnode); + *vgId = TD_VID(pVnodeObj); + } + + if (numOfTables) { + *numOfTables = pConf->vndStats.numOfNTables + pConf->vndStats.numOfCTables; + } + + if (numOfNormalTables) { + *numOfNormalTables = pConf->vndStats.numOfNTables; } } @@ -440,8 +451,10 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) { return 0; } -int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) { - SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 1); + +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { + SVnode *pVnodeObj = pVnode; + SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1); while (1) { tb_uid_t id = metaCtbCursorNext(pCur); @@ -529,10 +542,8 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { for (int64_t i = 0; i < arrSize; ++i) { tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); - SMetaStbStats stats = {0}; - metaGetStbStats(pVnode->pMeta, suid, &stats); - int64_t ctbNum = stats.ctbNum; - // vnodeGetCtbNum(pVnode, id, &ctbNum); + int64_t ctbNum = 0; + metaGetStbStats(pVnode, suid, &ctbNum); int numOfCols = 0; vnodeGetStbColumnNum(pVnode, suid, &numOfCols); @@ -567,16 +578,17 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { return TSDB_CODE_SUCCESS; } -void *vnodeGetIdx(SVnode *pVnode) { +void *vnodeGetIdx(void *pVnode) { if (pVnode == NULL) { return NULL; } - return metaGetIdx(pVnode->pMeta); + + return metaGetIdx(((SVnode*)pVnode)->pMeta); } -void *vnodeGetIvtIdx(SVnode *pVnode) { +void *vnodeGetIvtIdx(void *pVnode) { if (pVnode == NULL) { return NULL; } - return metaGetIvtIdx(pVnode->pMeta); + return metaGetIvtIdx(((SVnode*)pVnode)->pMeta); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 268d51cd16..d16102037d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -515,6 +515,8 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { } SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + initStorageAPI(&handle.api); + switch (pMsg->msgType) { case TDMT_SCH_QUERY: case TDMT_SCH_MERGE_QUERY: diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 1c2ed6c076..6497bd90b4 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -95,7 +95,7 @@ struct SExecTaskInfo { }; void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model); +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI); void doDestroyTask(SExecTaskInfo* pTaskInfo); bool isTaskKilled(SExecTaskInfo* pTaskInfo); void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b24fcade79..a52e73eb49 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -307,7 +307,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, metaHandle, 0); - code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, info->uid); + code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, info->uid); if (TSDB_CODE_SUCCESS != code) { pAPI->metaReaderFn.clearReader(&mr); *pQualified = false; @@ -1090,12 +1090,11 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S } if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table -// vnodeGetCtbIdList(); - pStorageAPI->metaFn.storeGetChildTableList(pVnode, pScanNode->suid, pUidList); + pStorageAPI->metaFn.getChildTableList(pVnode, pScanNode->suid, pUidList); } else { // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { - void* pIndex = pStorageAPI->metaFn.storeGetInvertIndex(pVnode); + void* pIndex = pStorageAPI->metaFn.getInvertIndex(pVnode); SIndexMetaArg metaArg = { .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid}; @@ -1170,7 +1169,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pVnode, 0); - if (pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, uid) != 0) { // table not exist + if (pAPI->metaReaderFn.getEntryGetUidCache(&mr, uid) != 0) { // table not exist pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cad98348d0..78a015269a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -252,7 +252,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id) { if (msg == NULL) { // create raw scan - SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE); + SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, &pReaderHandle->api); if (NULL == pTaskInfo) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -264,6 +264,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 return NULL; } + pTaskInfo->storageAPI = pReaderHandle->api; qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo)); return pTaskInfo; } @@ -1092,8 +1093,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT ASSERT(0); // walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); -// if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { -// qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); +// if (tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) { +// qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id); // return -1; // } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index a684a6bf94..f79966a021 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -35,7 +35,7 @@ #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model) { +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); if (pTaskInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -48,6 +48,7 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP pTaskInfo->execModel = model; pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); + pTaskInfo->storageAPI = *pAPI; taosInitRWLatch(&pTaskInfo->lock); @@ -55,7 +56,6 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP pTaskInfo->id.queryId = queryId; pTaskInfo->id.str = taosMemoryMalloc(64); buildTaskId(taskId, queryId, pTaskInfo->id.str); - return pTaskInfo; } @@ -78,7 +78,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { - *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model); + *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api); if (*pTaskInfo == NULL) { taosMemoryFree(sql); return terrno; @@ -123,7 +123,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo SStorageAPI* pAPI = &pTaskInfo->storageAPI; pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0); - int32_t code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, pScanNode->uid); + int32_t code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, GET_TASKID(pTaskInfo)); @@ -144,7 +144,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo tDecoderClear(&mr.coder); tb_uid_t suid = mr.me.ctbEntry.suid; - code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, suid); + code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, suid); if (code != TSDB_CODE_SUCCESS) { pAPI->metaReaderFn.clearReader(&mr); return terrno; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 23e029e780..83e62faf90 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -532,7 +532,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); - code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); + code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -561,7 +561,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); - code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid); + code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", @@ -693,7 +693,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = pAPI->tsdReader.tsdReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); @@ -975,6 +975,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); + pInfo->readerAPI = pTaskInfo->storageAPI.tsdReader; initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); // blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); @@ -1103,7 +1104,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = pAPI->tsdReader.tsdReaderNextDataBlock(pReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -2162,7 +2163,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { - code = pAPI->tsdReader.tsdReaderNextDataBlock(pInfo->dataReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pInfo->dataReader, &hasNext); if (code) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); @@ -2416,7 +2417,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys // set the extract column id to streamHandle pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); - code = pAPI->tqReaderFn.tqReaderSetTargetTableList(pInfo->tqReader, tableIdList); + code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; @@ -2457,6 +2458,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->igExpired = pTableScanNode->igExpired; pInfo->twAggSup.maxTs = INT64_MIN; pInfo->pState = NULL; + pInfo->stateStore = pTaskInfo->storageAPI.stateStore; + pInfo->readerFn = pTaskInfo->storageAPI.tqReaderFn; // for stream if (pTaskInfo->streamInfo.pState) { @@ -2674,7 +2677,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (true) { - code = pAPI->tsdReader.tsdReaderNextDataBlock(reader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); if (code != 0) { pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); pInfo->base.dataReader = NULL; @@ -3354,7 +3357,7 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount SStorageAPI* pAPI = &pTaskInfo->storageAPI; // get dbname - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, dbName); @@ -3396,7 +3399,7 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca pRes->info.id.groupId = groupId; int64_t dbTableCount = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &dbTableCount); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &dbTableCount, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } @@ -3411,20 +3414,21 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO if (strlen(pSupp->stbNameFilter) != 0) { tb_uid_t uid = 0; pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); - SMetaStbStats stats = {0}; - ASSERT(0); -// metaGetStbStats(pInfo->readHandle.vnode, uid, &stats); - int64_t ctbNum = stats.ctbNum; - fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); + + int64_t numOfChildTables = 0; + pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables); + + fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes); } else { int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode); + int64_t tbNumVnode = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } + setOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index a9450fb183..16f2b64c07 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -157,7 +157,7 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { const char* db = NULL; ASSERT(0); -// pAPI->metaFn.storeGetBasicInfo(pVnode, &db, NULL); +// pAPI->metaFn.getBasicInfo(pVnode, &db, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -185,7 +185,7 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { int64_t vgId = 0; ASSERT(0); -// pAPI->metaFn.storeGetBasicInfo(pVnode, NULL, (int32_t*)&vgId); +// pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId); SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -451,7 +451,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -522,7 +522,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } if (pInfo->pSchema == NULL) { @@ -614,7 +614,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -642,7 +642,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -702,7 +702,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } bool blockFull = false; @@ -722,7 +722,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&smrSuperTable); - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -756,7 +756,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1113,7 +1113,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1292,7 +1292,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { - pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.vnode); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } blockDataCleanup(pInfo->pRes); @@ -1300,7 +1300,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1346,7 +1346,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&mr); - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -1456,8 +1456,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { - pAPI->metaFn.closeMetaCursor(pInfo->pCur); - pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1741,6 +1740,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan extractTbnameSlotId(pInfo, pScanNode); + pInfo->pAPI = &pTaskInfo->storageAPI; + pInfo->accountId = pScanPhyNode->accountId; pInfo->pUser = taosStrdup((void*)pUser); pInfo->sysInfo = pScanPhyNode->sysInfo; @@ -1813,7 +1814,7 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - pInfo->pAPI->metaFn.closeMetaCursor(pInfo->pCur); + pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; } if (pInfo->pIdx) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b4fd571e23..2d366b6110 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2804,6 +2804,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); pInfo->dataVersion = 0; + pInfo->statestore = pTaskInfo->storageAPI.stateStore; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -4938,7 +4939,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pState = taosMemoryCalloc(1, sizeof(void)); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); @@ -4986,6 +4987,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); + pInfo->statestore = pTaskInfo->storageAPI.stateStore; initIntervalDownStream(downstream, pPhyNode->type, pInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 98b685d8b9..24339337b7 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -91,13 +91,14 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { return winKeyCmprImpl(&pWin1->key, &pWin2->key); } -SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages) { - qWarn("open stream state, %s", path); +SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) { + qDebug("open stream state, %s", path); SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); if (pState == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState)); if (pState->pTdbState == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -105,29 +106,33 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int return NULL; } + SStreamTask* pStreamTask = pTask; char statePath[1024]; if (!specPath) { - sprintf(statePath, "%s/%d", path, pTask->id.taskId); + sprintf(statePath, "%s/%d", path, pStreamTask->id.taskId); } else { memset(statePath, 0, 1024); tstrncpy(statePath, path, 1024); } - pState->taskId = pTask->id.taskId; - pState->streamId = pTask->id.streamId; + + pState->taskId = pStreamTask->id.taskId; + pState->streamId = pStreamTask->id.streamId; + #ifdef USE_ROCKSDB - // qWarn("open stream state1"); - taosAcquireRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); - int code = streamStateOpenBackend(pTask->pMeta->streamBackend, pState); + SStreamMeta* pMeta = pStreamTask->pMeta; + taosAcquireRef(pMeta->streamBackendId, pMeta->streamBackendRid); + int code = streamStateOpenBackend(pMeta->streamBackend, pState); if (code == -1) { - taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); + taosReleaseRef(pMeta->streamBackendId, pMeta->streamBackendRid); taosMemoryFree(pState); pState = NULL; } + pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT); - pState->parNameMap = tSimpleHashInit(1024, hashFn); + pState->parNameMap = tSimpleHashInit(1024, hashFn); return pState; #else @@ -449,7 +454,7 @@ int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pV #ifdef USE_ROCKSDB taosMemoryFree(pVal); #else - streamFreeVal(pVal); + streamStateFreeVal(pVal); #endif return 0; } @@ -700,7 +705,7 @@ void streamStateFreeCur(SStreamStateCur* pCur) { taosMemoryFree(pCur); } -void streamFreeVal(void* val) { +void streamStateFreeVal(void* val) { #ifdef USE_ROCKSDB taosMemoryFree(val); #else From a2f29aa0f52decad5510024718184ffbe909fb2e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 13:34:46 +0800 Subject: [PATCH 10/25] refactor: do some internal refactor. --- source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/src/vnd/vnodeOpen.c | 175 ------------------------- 2 files changed, 1 insertion(+), 175 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e8660cd6ad..04fad2d78e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -15,6 +15,7 @@ target_sources( "src/vnd/vnodeSync.c" "src/vnd/vnodeSnapshot.c" "src/vnd/vnodeRetention.c" + "src/vnd/vnodeInitApi.c" # meta "src/meta/metaOpen.c" diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ebfecc4c15..b5e7c6875b 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -14,7 +14,6 @@ */ #include "vnd.h" -#include "tstreamUpdate.h" int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { SVnodeInfo info = {0}; @@ -445,177 +444,3 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyTerm = pVnode->state.commitTerm; pSnapshot->lastConfigIndex = -1; } - -static void initTsdbReaderAPI(TsdReader* pReader); -static void initMetadataAPI(SStoreMeta* pMeta); -static void initTqAPI(SStoreTqReader* pTq); -static void initStateStoreAPI(SStateStore* pStore); -static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); - -void initStorageAPI(SStorageAPI* pAPI) { - initTsdbReaderAPI(&pAPI->tsdReader); - initMetadataAPI(&pAPI->metaFn); - initTqAPI(&pAPI->tqReaderFn); - initStateStoreAPI(&pAPI->stateStore); - initMetaReaderAPI(&pAPI->metaReaderFn); -} - -void initTsdbReaderAPI(TsdReader* pReader) { - pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen; - pReader->tsdReaderClose = tsdbReaderClose; - - pReader->tsdNextDataBlock = tsdbNextDataBlock; - - pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock; - pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock; - - pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA; - - pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; - pReader->tsdReaderResetStatus = tsdbReaderReset; - - pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; - pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away - - pReader->tsdSetQueryTableList = tsdbSetTableList; - pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; -} - -void initMetadataAPI(SStoreMeta* pMeta) { - pMeta->isTableExisted = metaIsTableExist; - - pMeta->openTableMetaCursor = metaOpenTbCursor; - pMeta->closeTableMetaCursor = metaCloseTbCursor; - pMeta->cursorNext = metaTbCursorNext; - pMeta->cursorPrev = metaTbCursorPrev; - - pMeta->getBasicInfo = vnodeGetInfo; - pMeta->getNumOfChildTables = metaGetStbStats; - - pMeta->getChildTableList = vnodeGetCtbIdList; - - pMeta->storeGetIndexInfo = vnodeGetIdx; - pMeta->getInvertIndex = vnodeGetIvtIdx; - - pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; - -} - -void initTqAPI(SStoreTqReader* pTq) { - pTq->tqReaderOpen = tqReaderOpen; - pTq->tqReaderSetColIdList = tqReaderSetColIdList; - - pTq->tqReaderClose = tqReaderClose; - pTq->tqReaderSeek = tqReaderSeek; - pTq->tqRetrieveBlock = tqRetrieveDataBlock; - - pTq->tqReaderNextBlockInWal = tqNextBlockInWal; - - pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it - - pTq->tqReaderAddTables = tqReaderAddTbUidList; - pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; - - pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList; - - pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; - pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; - - pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it - pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it - - pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it - pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; -} - -void initStateStoreAPI(SStateStore* pStore) { - pStore->streamFileStateInit = streamFileStateInit; - pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; - - pStore->streamStateGetByPos = streamStateGetByPos; - - pStore->streamStatePutParName = streamStatePutParName; - pStore->streamStateGetParName = streamStateGetParName; - - pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; - pStore->streamStateReleaseBuf = streamStateReleaseBuf; - pStore->streamStateFreeVal = streamStateFreeVal; - - pStore->streamStatePut = streamStatePut; - pStore->streamStateGet = streamStateGet; - pStore->streamStateCheck = streamStateCheck; - pStore->streamStateGetByPos = streamStateGetByPos; - pStore->streamStateDel = streamStateDel; - pStore->streamStateClear = streamStateClear; - pStore->streamStateSaveInfo = streamStateSaveInfo; - pStore->streamStateGetInfo = streamStateGetInfo; - pStore->streamStateSetNumber = streamStateSetNumber; - - pStore->streamStateFillPut = streamStateFillPut; - pStore->streamStateFillGet = streamStateFillGet; - pStore->streamStateFillDel = streamStateFillDel; - - pStore->streamStateCurNext = streamStateCurNext; - pStore->streamStateCurPrev = streamStateCurPrev; - - pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; - pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; - pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; - pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; - pStore->streamStateFreeCur = streamStateFreeCur; - - pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; - pStore->streamStateGetKVByCur = streamStateGetKVByCur; - - pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; - pStore->streamStateSessionPut = streamStateSessionPut; - pStore->streamStateSessionGet = streamStateSessionGet; - pStore->streamStateSessionDel = streamStateSessionDel; - pStore->streamStateSessionClear = streamStateSessionClear; - pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; - pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; - pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; - - pStore->updateInfoInit = updateInfoInit; - pStore->updateInfoFillBlockData = updateInfoFillBlockData; - pStore->updateInfoIsUpdated = updateInfoIsUpdated; - pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; - pStore->updateInfoDestroy = updateInfoDestroy; - - pStore->updateInfoInitP = updateInfoInitP; - pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; - pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; - pStore->updateInfoSerialize = updateInfoSerialize; - pStore->updateInfoDeserialize = updateInfoDeserialize; - - pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; - pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; - pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; - - pStore->streamFileStateInit = streamFileStateInit; - - pStore->streamFileStateDestroy = streamFileStateDestroy; - pStore->streamFileStateClear = streamFileStateClear; - pStore->needClearDiskBuff = needClearDiskBuff; - - pStore->streamStateOpen = streamStateOpen; - pStore->streamStateClose = streamStateClose; - pStore->streamStateBegin = streamStateBegin; - pStore->streamStateCommit = streamStateCommit; - pStore->streamStateDestroy= streamStateDestroy; - pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; -} - -void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { - pMetaReader->initReader = _metaReaderInit; - pMetaReader->clearReader = metaReaderClear; - - pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; - pMetaReader->clearReader = metaReaderClear; - - pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; - pMetaReader->getTableEntryByName = metaGetTableEntryByName; - - pMetaReader->readerReleaseLock = metaReaderReleaseLock; -} - From b2ce45b07bb4094acfe483e9f65d788d04fbe2f3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 13:39:20 +0800 Subject: [PATCH 11/25] refactor: do some internal refactor. --- source/dnode/vnode/src/vnd/vnodeInitApi.c | 191 ++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 source/dnode/vnode/src/vnd/vnodeInitApi.c diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c new file mode 100644 index 0000000000..3f427d9965 --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "storageapi.h" +#include "vnodeInt.h" +#include "tstreamUpdate.h" + +static void initTsdbReaderAPI(TsdReader* pReader); +static void initMetadataAPI(SStoreMeta* pMeta); +static void initTqAPI(SStoreTqReader* pTq); +static void initStateStoreAPI(SStateStore* pStore); +static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); + +void initStorageAPI(SStorageAPI* pAPI) { + initTsdbReaderAPI(&pAPI->tsdReader); + initMetadataAPI(&pAPI->metaFn); + initTqAPI(&pAPI->tqReaderFn); + initStateStoreAPI(&pAPI->stateStore); + initMetaReaderAPI(&pAPI->metaReaderFn); +} + +void initTsdbReaderAPI(TsdReader* pReader) { + pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen; + pReader->tsdReaderClose = tsdbReaderClose; + + pReader->tsdNextDataBlock = tsdbNextDataBlock; + + pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock; + pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock; + + pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA; + + pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; + pReader->tsdReaderResetStatus = tsdbReaderReset; + + pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away + + pReader->tsdSetQueryTableList = tsdbSetTableList; + pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; +} + +void initMetadataAPI(SStoreMeta* pMeta) { + pMeta->isTableExisted = metaIsTableExist; + + pMeta->openTableMetaCursor = metaOpenTbCursor; + pMeta->closeTableMetaCursor = metaCloseTbCursor; + pMeta->cursorNext = metaTbCursorNext; + pMeta->cursorPrev = metaTbCursorPrev; + + pMeta->getBasicInfo = vnodeGetInfo; + pMeta->getNumOfChildTables = metaGetStbStats; + + pMeta->getChildTableList = vnodeGetCtbIdList; + + pMeta->storeGetIndexInfo = vnodeGetIdx; + pMeta->getInvertIndex = vnodeGetIvtIdx; + + pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + +} + +void initTqAPI(SStoreTqReader* pTq) { + pTq->tqReaderOpen = tqReaderOpen; + pTq->tqReaderSetColIdList = tqReaderSetColIdList; + + pTq->tqReaderClose = tqReaderClose; + pTq->tqReaderSeek = tqReaderSeek; + pTq->tqRetrieveBlock = tqRetrieveDataBlock; + + pTq->tqReaderNextBlockInWal = tqNextBlockInWal; + + pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it + + pTq->tqReaderAddTables = tqReaderAddTbUidList; + pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; + + pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList; + + pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; + pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; + + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + + pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; +} + +void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { + pMetaReader->initReader = _metaReaderInit; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; + pMetaReader->getTableEntryByName = metaGetTableEntryByName; + + pMetaReader->readerReleaseLock = metaReaderReleaseLock; +} \ No newline at end of file From f203c7806de763444c10f8ea08438ba8ea57300c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 14:52:50 +0800 Subject: [PATCH 12/25] fix(tools): fix build failure on enterprise edition. --- source/libs/qworker/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index 92ccde3163..a947a2dd19 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -7,9 +7,15 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(qworker - PRIVATE os util transport nodes planner qcom executor - ) +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(qworker + PRIVATE os util transport nodes planner qcom executor grant + ) +ELSE () + TARGET_LINK_LIBRARIES(qworker + PRIVATE os util transport nodes planner qcom executor + ) +ENDIF() if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) From a56618077049d97bf369f10f9f492a259e9f8dde Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 15:16:50 +0800 Subject: [PATCH 13/25] fix: fix syntax error. --- include/libs/executor/storageapi.h | 4 ++-- source/libs/stream/test/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 8790c302ce..300507c132 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -86,7 +86,7 @@ typedef struct SMetaReader { } SMetaReader; typedef struct SMTbCursor { - struct TBC *pDbc; + void * pDbc; void * pKey; void * pVal; int32_t kLen; @@ -169,7 +169,7 @@ typedef struct SMetaTableInfo { typedef struct SSnapContext { SMeta * pMeta; // todo remove it int64_t snapVersion; - struct TBC *pCur; + void * pCur; int64_t suid; int8_t subType; SHashObj * idVersion; diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index a0c1717690..75acec7f01 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -10,7 +10,7 @@ ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") TARGET_LINK_LIBRARIES( streamUpdateTest - PUBLIC os util common gtest gtest_main stream + PUBLIC os util common gtest gtest_main stream executor ) TARGET_INCLUDE_DIRECTORIES( From fe09705855ecd5f695ce91809cdbf0c596b26e29 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 17:13:20 +0800 Subject: [PATCH 14/25] fix: set the init function pointer. --- include/libs/executor/storageapi.h | 2 ++ include/libs/index/index.h | 18 +++++--------- source/dnode/qnode/CMakeLists.txt | 1 + source/dnode/snode/CMakeLists.txt | 1 + source/dnode/vnode/inc/vnode.h | 5 ---- source/dnode/vnode/src/inc/meta.h | 6 +++++ source/dnode/vnode/src/meta/metaQuery.c | 16 ++++++------- source/dnode/vnode/src/vnd/vnodeInitApi.c | 11 ++++++++- source/libs/executor/src/sysscanoperator.c | 28 +++++++++++----------- source/libs/index/src/indexFilter.c | 7 +++--- source/libs/qworker/CMakeLists.txt | 4 ++-- source/libs/scheduler/CMakeLists.txt | 2 +- source/libs/stream/CMakeLists.txt | 2 +- 13 files changed, 55 insertions(+), 48 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 300507c132..97900de7ba 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -21,6 +21,7 @@ #include "taosdef.h" #include "tmsg.h" #include "tcommon.h" +#include "index.h" #ifdef __cplusplus extern "C" { @@ -544,6 +545,7 @@ typedef struct SStorageAPI { SStoreSnapshotFn snapshotFn; SStoreTqReader tqReaderFn; SStateStore stateStore; + SMetaDataFilterAPI metaFilter; } SStorageAPI; #ifdef __cplusplus diff --git a/include/libs/index/index.h b/include/libs/index/index.h index fd71c531fa..162b99e553 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -239,7 +239,6 @@ void indexCleanup(); * @param results * @return */ - typedef struct SMetaFltParam { uint64_t suid; int16_t cid; @@ -250,17 +249,12 @@ typedef struct SMetaFltParam { int (*filterFunc)(void *a, void *b, int16_t type); } SMetaFltParam; -typedef struct SStoreAPI { - int32_t (*metaFilterTableIds)(); - int32_t (*metaFilterCreateTime)(); - int32_t (*metaFilterTableName)(); - int32_t (*metaFilterTtl)(); -} SStoreAPI; - -//int32_t metaFilterTableIds(void *pMeta, SMetaFltParam *param, SArray *results); -//int32_t metaFilterCreateTime(void *pMeta, SMetaFltParam *parm, SArray *pUids); -//int32_t metaFilterTableName(void *pMeta, SMetaFltParam *param, SArray *pUids); -//int32_t metaFilterTtl(void *pMeta, SMetaFltParam *param, SArray *pUids); +typedef struct SMetaDataFilterAPI { + int32_t (*metaFilterTableIds)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterCreateTime)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterTableName)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterTtl)(void *pVnode, SMetaFltParam *arg, SArray *pUids); +} SMetaDataFilterAPI; #ifdef __cplusplus } diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt index 13429d0cc1..10bbbc1b26 100644 --- a/source/dnode/qnode/CMakeLists.txt +++ b/source/dnode/qnode/CMakeLists.txt @@ -16,4 +16,5 @@ target_link_libraries( PRIVATE executor PRIVATE tdb PRIVATE wal + PRIVATE index ) \ No newline at end of file diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 1d8c80c422..ebfe80ecab 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -15,4 +15,5 @@ target_link_libraries( PRIVATE qcom PRIVATE stream PRIVATE wal + PRIVATE index ) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 95fe150443..84153f3867 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -151,11 +151,6 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); // //} SMetaFltParam; -// TODO, refactor later -//int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); -//int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); -//int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); -//int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 3999aa0b7f..b39008147b 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -162,6 +162,12 @@ typedef struct { int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); +// TODO, refactor later +int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *param, SArray *results); +int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *parm, SArray *pUids); +int32_t metaFilterTableName(void *pVnode, SMetaFltParam *param, SArray *pUids); +int32_t metaFilterTtl(void *pVnode, SMetaFltParam *param, SArray *pUids); + #ifndef META_REFACT // SMetaDB int metaOpenDB(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 40f4b6fa67..3836b37002 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1012,8 +1012,8 @@ typedef struct { int32_t vLen; } SIdxCursor; -int32_t metaFilterCreateTime(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; @@ -1072,8 +1072,8 @@ END: return ret; } -int32_t metaFilterTableName(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1139,8 +1139,8 @@ END: return ret; } -int32_t metaFilterTtl(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1170,8 +1170,8 @@ END: // impl later return 0; } -int32_t metaFilterTableIds(void *meta, void *arg, SArray *pUids) { - SMeta *pMeta = meta; +int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode*)pVnode)->pMeta; SMetaFltParam *param = arg; SMetaEntry oStbEntry = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 3f427d9965..7a26d7b837 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -16,12 +16,14 @@ #include "storageapi.h" #include "vnodeInt.h" #include "tstreamUpdate.h" +#include "meta.h" static void initTsdbReaderAPI(TsdReader* pReader); static void initMetadataAPI(SStoreMeta* pMeta); static void initTqAPI(SStoreTqReader* pTq); static void initStateStoreAPI(SStateStore* pStore); static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); +static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -29,6 +31,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initTqAPI(&pAPI->tqReaderFn); initStateStoreAPI(&pAPI->stateStore); initMetaReaderAPI(&pAPI->metaReaderFn); + initMetaFilterAPI(&pAPI->metaFilter); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -69,7 +72,6 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getInvertIndex = vnodeGetIvtIdx; pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; - } void initTqAPI(SStoreTqReader* pTq) { @@ -188,4 +190,11 @@ void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { pMetaReader->getTableEntryByName = metaGetTableEntryByName; pMetaReader->readerReleaseLock = metaReaderReleaseLock; +} + +void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { + pFilter->metaFilterCreateTime = metaFilterCreateTime; + pFilter->metaFilterTableIds = metaFilterTableIds; + pFilter->metaFilterTableName = metaFilterTableName; + pFilter->metaFilterTtl = metaFilterTtl; } \ No newline at end of file diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 16f2b64c07..0d1926e21d 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -39,8 +39,9 @@ typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); typedef int32_t (*__sys_check)(SNode* cond); typedef struct SSTabFltArg { - void* pMeta; - void* pVnode; + void* pMeta; + void* pVnode; + SStorageAPI* pAPI; } SSTabFltArg; typedef struct SSysTableIndex { @@ -153,11 +154,11 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t SFilterInfo* pFilterInfo); int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; + SSTabFltArg* pArg = arg; + void* pVnode = pArg->pVnode; const char* db = NULL; - ASSERT(0); -// pAPI->metaFn.getBasicInfo(pVnode, &db, NULL); + pArg->pAPI->metaFn.getBasicInfo(pVnode, &db, NULL, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -181,11 +182,11 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; + SSTabFltArg* pArg = arg; + void* pVnode = ((SSTabFltArg*)arg)->pVnode; int64_t vgId = 0; - ASSERT(0); -// pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId); + pArg->pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId, NULL, NULL); SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -202,7 +203,7 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; + SSTabFltArg* pArg = arg; SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -222,7 +223,8 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; + SSTabFltArg* pArg = arg; + SStorageAPI* pAPI = pArg->pAPI; SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -239,10 +241,8 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { .equal = equal, .filterFunc = func}; - ASSERT(0); - return 0; -// int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); -// return ret; + int32_t ret = pAPI->metaFilter.metaFilterCreateTime(pArg->pVnode, ¶m, result); + return ret; } int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index b90959579d..6be8b67264 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -67,9 +67,8 @@ typedef union { typedef struct SIFParam { SHashObj *pFilter; - - SArray *result; - char *condValue; + SArray *result; + char *condValue; SIdxFltStatus status; uint8_t colValType; @@ -79,7 +78,7 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; - SStoreAPI api; + SMetaDataFilterAPI api; } SIFParam; typedef struct SIFCtx { diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index a947a2dd19..8ba8b79ab8 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -9,11 +9,11 @@ target_include_directories( IF (TD_GRANT) TARGET_LINK_LIBRARIES(qworker - PRIVATE os util transport nodes planner qcom executor grant + PRIVATE os util transport nodes planner qcom executor index grant ) ELSE () TARGET_LINK_LIBRARIES(qworker - PRIVATE os util transport nodes planner qcom executor + PRIVATE os util transport nodes planner qcom executor index ) ENDIF() diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 3288120b67..fafc2a27e0 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PUBLIC os util nodes planner qcom common catalog transport command qworker executor + PUBLIC os util nodes planner qcom common catalog transport command qworker executor index ) if(${BUILD_TEST}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 2edbc44aae..89b82ef9ba 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -10,7 +10,7 @@ if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal + PRIVATE os util transport qcom executor wal index ) target_include_directories( From ab907e3344f74fa2bc66d0525b7defa1f7ceec21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 21:53:40 +0800 Subject: [PATCH 15/25] fix(query): fix syntax error. --- include/libs/executor/storageapi.h | 6 ++-- include/libs/index/index.h | 34 +++++++++++----------- source/dnode/vnode/inc/vnode.h | 16 ++-------- source/dnode/vnode/src/meta/metaQuery.c | 7 +++-- source/dnode/vnode/src/vnd/vnodeInitApi.c | 6 ++++ source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/index/src/indexFilter.c | 21 ++++++++----- source/libs/qworker/test/CMakeLists.txt | 2 +- 9 files changed, 48 insertions(+), 48 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 97900de7ba..77576cf1af 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -354,13 +354,13 @@ typedef struct SStoreMetaReader { } SStoreMetaReader; typedef struct SStoreMeta { - SMTbCursor *(*openTableMetaCursor)(); // metaOpenTbCursor - void (*closeTableMetaCursor)(); // metaCloseTbCursor + SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor int32_t (*cursorNext)(); // metaTbCursorNext int32_t (*cursorPrev)(); // metaTbCursorPrev int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); - int32_t (*getTableTagsByUid)(); + int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList); const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 162b99e553..cfcc9993cf 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -215,23 +215,6 @@ typedef struct SIndexMetaArg { int (*metaFilterFunc)(void* metaEx, void* param, SArray* result); } SIndexMetaArg; -typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; - -SIdxFltStatus idxGetFltStatus(SNode* pFilterNode); - -int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status); - -/* - * init index env - * - */ -void indexInit(int32_t threads); -/* - * destory index env - * - */ -void indexCleanup(); - /** * the underlying storage module must implement this API to employ the index functions. * @param pMeta @@ -256,6 +239,23 @@ typedef struct SMetaDataFilterAPI { int32_t (*metaFilterTtl)(void *pVnode, SMetaFltParam *arg, SArray *pUids); } SMetaDataFilterAPI; +typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; + +SIdxFltStatus idxGetFltStatus(SNode* pFilterNode, SMetaDataFilterAPI* pAPI); + +int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status, SMetaDataFilterAPI* pAPI); + +/* + * init index env + * + */ +void indexInit(int32_t threads); +/* + * destory index env + * + */ +void indexCleanup(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 84153f3867..4e3e8a6a8d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -111,8 +111,8 @@ void metaReaderClear(SMetaReader *pReader); int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); -int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); +int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *uidList); +int32_t metaGetTableTagsByUids(void* pVnode, int64_t suid, SArray *uidList); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); @@ -140,18 +140,6 @@ int64_t metaGetNtbNum(SMeta *pMeta); //} SMetaStbStats; int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); -//typedef struct SMetaFltParam { -// tb_uid_t suid; -// int16_t cid; -// int16_t type; -// void *val; -// bool reverse; -// bool equal; -// int (*filterFunc)(void *a, void *b, int16_t type); -// -//} SMetaFltParam; - - #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 3836b37002..617a3a2c08 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1356,7 +1356,8 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi return ret; } -int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { +int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) { + SMeta* pMeta = ((SVnode*) pVnode)->pMeta; const int32_t LIMIT = 128; int32_t isLock = false; @@ -1388,8 +1389,8 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { return 0; } -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) { - SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1); +int32_t metaGetTableTags(void* pVnode, uint64_t suid, SArray *pUidTagInfo) { + SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode*)pVnode)->pMeta, suid, 1); // If len > 0 means there already have uids, and we only want the // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 7a26d7b837..34c6da1398 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -72,6 +72,12 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getInvertIndex = vnodeGetIvtIdx; pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + pMeta->getTableTags = metaGetTableTags; + pMeta->getTableTagsByUid = metaGetTableTagsByUids; + + pMeta->getTableUidByName = metaGetTableUidByName; + pMeta->getTableTypeByName = metaGetTableTypeByName; + pMeta->getTableNameByUid = metaGetTableNameByUid; } void initTqAPI(SStoreTqReader* pTq) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a52e73eb49..47b08b3323 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1100,7 +1100,7 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid}; status = SFLT_NOT_INDEX; - code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); + code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status, &pStorageAPI->metaFilter); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); code = TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 0d1926e21d..8f705a7eeb 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1485,7 +1485,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } else { if (pInfo->showRewrite == false) { if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode}; + SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); idx->init = 0; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6be8b67264..2c12c84081 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -86,6 +86,7 @@ typedef struct SIFCtx { SHashObj *pRes; /* element is SIFParam */ bool noExec; // true: just iterate condition tree, and add hint to executor plan SIndexMetaArg arg; + SMetaDataFilterAPI *pAPI; } SIFCtx; static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { @@ -288,6 +289,8 @@ static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx } static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { param->status = SFLT_COARSE_INDEX; + param->api = *ctx->pAPI; + switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; @@ -364,6 +367,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } } + SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { @@ -972,8 +976,9 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (pNode == NULL || pDst == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; } + int32_t code = 0; - SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg}; + SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg, .pAPI = &pDst->api}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { @@ -1006,13 +1011,13 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { return code; } -static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { +static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; if (pNode == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; } - SIFCtx ctx = {.code = 0, .noExec = true}; + SIFCtx ctx = {.code = 0, .noExec = true, .pAPI = pAPI}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); @@ -1044,8 +1049,8 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { return code; } -int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status) { - SIdxFltStatus st = idxGetFltStatus(pFilterNode); +int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) { + SIdxFltStatus st = idxGetFltStatus(pFilterNode, pAPI); if (st == SFLT_NOT_INDEX) { *status = st; return 0; @@ -1054,7 +1059,7 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SFilterInfo *filter = NULL; SArray *output = taosArrayInit(8, sizeof(uint64_t)); - SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX}; + SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX, .api = *pAPI}; int32_t code = sifCalculate((SNode *)pFilterNode, ¶m); if (code != 0) { sifFreeParam(¶m); @@ -1071,13 +1076,13 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, return TSDB_CODE_SUCCESS; } -SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { +SIdxFltStatus idxGetFltStatus(SNode *pFilterNode, SMetaDataFilterAPI* pAPI) { SIdxFltStatus st = SFLT_NOT_INDEX; if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } - if (sifGetFltHint((SNode *)pFilterNode, &st) != TSDB_CODE_SUCCESS) { + if (sifGetFltHint((SNode *)pFilterNode, &st, pAPI) != TSDB_CODE_SUCCESS) { st = SFLT_NOT_INDEX; } return st; diff --git a/source/libs/qworker/test/CMakeLists.txt b/source/libs/qworker/test/CMakeLists.txt index 780f5ae84b..22870ea94d 100644 --- a/source/libs/qworker/test/CMakeLists.txt +++ b/source/libs/qworker/test/CMakeLists.txt @@ -8,7 +8,7 @@ IF(NOT TD_DARWIN) ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( qworkerTest - PUBLIC os util common transport gtest qcom nodes planner qworker executor + PUBLIC os util common transport gtest qcom nodes planner qworker executor index ) TARGET_INCLUDE_DIRECTORIES( From 01f6f74fb929429565bbadca51d49d2ffa9c3f50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 22:27:45 +0800 Subject: [PATCH 16/25] fix: update the cmake file. --- source/libs/stream/test/CMakeLists.txt | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index 75acec7f01..160cddbd4c 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -8,10 +8,20 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) # bloomFilterTest ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") -TARGET_LINK_LIBRARIES( - streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor -) +#TARGET_LINK_LIBRARIES( +# streamUpdateTest +# PUBLIC os util common gtest gtest_main stream executor +#) + +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(streamUpdateTest + PUBLIC os util common gtest gtest_main stream executor grant + ) +ELSE () + TARGET_LINK_LIBRARIES(streamUpdateTest + PUBLIC os util common gtest gtest_main stream executor + ) +ENDIF() TARGET_INCLUDE_DIRECTORIES( streamUpdateTest From a9b18d203daa2299aaa546723d86606f9f117ccc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 24 May 2023 23:49:26 +0800 Subject: [PATCH 17/25] fix:fix error. --- include/libs/executor/dataSinkMgt.h | 2 +- source/dnode/vnode/inc/vnode.h | 4 ++-- source/dnode/vnode/src/meta/metaQuery.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 ++ source/libs/executor/inc/dataSinkInt.h | 4 ++-- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/dataInserter.c | 2 +- source/libs/executor/src/dataSinkMgt.c | 3 ++- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 9 +++++---- 11 files changed, 26 insertions(+), 22 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index ce7d038d42..0a9037d21c 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -59,7 +59,7 @@ typedef struct SDataSinkMgtCfg { uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; -int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg); +int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI); typedef struct SInputData { const struct SSDataBlock* pData; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 4e3e8a6a8d..2723e869ff 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -118,7 +118,7 @@ const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(void* pVnode, tb_uid_t uid); int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, @@ -192,7 +192,7 @@ int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); -int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); +int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 617a3a2c08..255fa10f57 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -139,10 +139,10 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { return uid; } -int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { +int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); @@ -170,10 +170,10 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { +int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); SMetaReader *pReader = &mr; @@ -191,10 +191,10 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { return 0; } -int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) { +int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); code = metaGetTableEntryByName(&mr, tbName); if (code == 0) *tbType = mr.me.type; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 414ccfc8b2..62b121e4d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5373,9 +5373,9 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { return rows; } -int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { +int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { SMetaReader mr = {0}; - metaReaderInit(&mr, pVnode->pMeta, 0); + metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; @@ -5405,7 +5405,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 metaReaderClear(&mr); // get the newest table schema version - code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema); + code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 34c6da1398..70cff5e68a 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -78,6 +78,8 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableUidByName = metaGetTableUidByName; pMeta->getTableTypeByName = metaGetTableTypeByName; pMeta->getTableNameByUid = metaGetTableNameByUid; + + pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor } void initTqAPI(SStoreTqReader* pTq) { diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 3255831476..9893b4eb76 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -29,8 +29,8 @@ struct SDataSink; struct SDataSinkHandle; typedef struct SDataSinkManager { - SDataSinkMgtCfg cfg; - SStorageAPI storeFn; + SDataSinkMgtCfg cfg; + SStorageAPI* pAPI; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index eb585e7046..8188ecd305 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -224,7 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - TsdReader readerAPI; +// TsdReader readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index aadbf1f1a9..646964ebf4 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -429,7 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->explain = pInserterNode->explain; int64_t suid = 0; - int32_t code = pManager->storeFn.metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); + int32_t code = pManager->pAPI->metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index b3cb57325b..3a972c1c20 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -21,8 +21,9 @@ static SDataSinkManager gDataSinkManager = {0}; SDataSinkStat gDataSinkStat = {0}; -int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) { +int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI) { gDataSinkManager.cfg = *cfg; + gDataSinkManager.pAPI = pAPI; return 0; // to avoid compiler eror } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 78a015269a..53fad6503e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -507,7 +507,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50}; - code = dsDataSinkMgtInit(&cfg); + code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str); goto _error; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 83e62faf90..4bd8faf8e9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -929,7 +929,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->readerAPI); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -975,7 +975,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); - pInfo->readerAPI = pTaskInfo->storageAPI.tsdReader; + pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); // blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); @@ -1075,7 +1075,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.endVersion = version; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -1250,7 +1250,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pTableScanInfo->readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -3045,6 +3045,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN goto _error; } + pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.readHandle = *readHandle; From 4a42f6b983aeb26883ab863f0b8b19e53b2bcbd9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 00:50:03 +0800 Subject: [PATCH 18/25] fix: fix error in api call. --- include/libs/executor/storageapi.h | 2 +- source/dnode/mnode/impl/src/mndQuery.c | 1 + source/dnode/vnode/src/sma/smaRollup.c | 4 ++++ source/dnode/vnode/src/tq/tq.c | 2 ++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 2 ++ source/libs/executor/src/sysscanoperator.c | 8 ++++++-- source/libs/executor/src/timewindowoperator.c | 8 +++++--- 8 files changed, 22 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 77576cf1af..0fd8a74127 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -396,7 +396,7 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); - int64_t (*getNumOfRowsInMem)(); + int64_t (*getNumOfRowsInMem)(void* pVnode); /** int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 5278fc7761..8e95fa3d6d 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -33,6 +33,7 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) { int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { int32_t code = -1; SMnode *pMnode = pMsg->info.node; + SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb}; mTrace("msg:%p, in query queue is processing", pMsg); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 558cfdac2b..39aa5c3043 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -277,6 +277,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } SReadHandle handle = { .vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState }; + initStorageAPI(&handle.api); + pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; @@ -849,6 +851,8 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t TSDB_CHECK_CODE(code, lino, _exit); SReadHandle handle = { .vnode = pVnode, .initTqReader = 1 }; + initStorageAPI(&handle.api); + if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { code = TSDB_CODE_APP_ERROR; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4adbf02302..362b8204fc 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -671,6 +671,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->pRef = pRef; SReadHandle handle = {.vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; + initStorageAPI(&handle.api); + pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 70cff5e68a..d9bb4270d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -65,6 +65,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getBasicInfo = vnodeGetInfo; pMeta->getNumOfChildTables = metaGetStbStats; +// pMeta->getNumOfRowsInMem = tsdbGetNumOfRowsInMemTable; pMeta->getChildTableList = vnodeGetCtbIdList; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d16102037d..b950437f23 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -250,7 +250,9 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { uint8_t *pCont; SEncoder *pCoder = &(SEncoder){0}; SDeleteRes res = {0}; + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + initStorageAPI(&handle.api); code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); if (code) goto _exit; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 8f705a7eeb..d15b6d5754 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1814,9 +1814,13 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); + if (pInfo->pAPI->metaFn.closeTableMetaCursor != NULL) { + pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); + } + pInfo->pCur = NULL; } + if (pInfo->pIdx) { taosArrayDestroy(pInfo->pIdx->uids); taosMemoryFree(pInfo->pIdx); @@ -2200,7 +2204,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { } pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle); + blockDistInfo.numOfInmemRows = (int32_t) pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2d366b6110..cf58892920 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2906,7 +2906,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin } int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, - SStreamState* pState, int32_t keySize, int16_t keyType) { + SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore) { pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput); pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); pSup->gap = gap; @@ -2939,6 +2939,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir); return terrno; } + + pSup->stateStore = *pStore; int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].saveHandle.pBuf = pSup->pResultBuf; @@ -3600,7 +3602,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh } code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap, - pTaskInfo->streamInfo.pState, 0, 0); + pTaskInfo->streamInfo.pState, 0, 0, &pTaskInfo->storageAPI.stateStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4149,7 +4151,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes; int16_t type = pColNode->node.resType.type; code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, - type); + type, &pTaskInfo->storageAPI.stateStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } From da9f33a483ce658098bbc142f223d46166d1a272 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 09:56:34 +0800 Subject: [PATCH 19/25] fix: fix error in set function ptr. --- source/dnode/snode/src/snode.c | 4 +- source/dnode/vnode/src/tq/tq.c | 5 ++- source/libs/executor/src/filloperator.c | 7 ++- source/libs/executor/src/scanoperator.c | 43 +++++++++---------- source/libs/executor/src/timewindowoperator.c | 7 +-- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index e9225f3d6e..24e2b13f46 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -87,9 +87,9 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { } int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); - SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; + SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0); ASSERT(pTask->exec.pExecutor); streamSetupTrigger(pTask); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 362b8204fc..d98fbab192 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -804,9 +804,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); - SReadHandle mgHandle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + SReadHandle handle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + initStorageAPI(&handle.api); - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); if (pTask->exec.pExecutor == NULL) { return -1; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 0b89dbc80b..b448e8f2b0 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -443,7 +443,6 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - ASSERT(0); releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore); //????? pFillSup->cur.pRowVal = NULL; cleanupExprSupp(&pFillSup->notFillExprSup); @@ -493,7 +492,6 @@ static void resetFillWindow(SResultRowData* pRowData) { void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { resetFillWindow(&pFillSup->prev); - ASSERT(0); releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore); //??? resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); @@ -1335,7 +1333,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { } static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols) { + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); if (!pFillSup) { return NULL; @@ -1348,6 +1346,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->type = convertFillType(pPhyFillNode->mode); pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; pFillSup->interval = *pInterval; + pFillSup->pAPI = pAPI; int32_t code = initResultBuf(pFillSup); if (code != TSDB_CODE_SUCCESS) { @@ -1427,7 +1426,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; int32_t numOfFillCols = 0; SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); if (!pInfo->pFillSup) { goto _error; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4bd8faf8e9..39d619172b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1070,9 +1070,9 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou pInfo->groupId = groupCol[rowIndex]; } -void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t version) { +void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) { pTableScanInfo->base.cond.twindows = *pWin; - pTableScanInfo->base.cond.endVersion = version; + pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); @@ -1182,17 +1182,19 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ win.ekey = TMAX(win.ekey, endData[*pRowIndex]); continue; } + if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) { win.skey = TMIN(win.skey, startData[*pRowIndex]); continue; } + ASSERT(!(win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) || !(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0))); break; } - ASSERT(0); -// resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); + STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; + resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -1650,13 +1652,13 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - ASSERT(0); // qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, // pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, // pInfo->tqReader->pWalReader->curVersion); tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; } + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); @@ -1673,7 +1675,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { while (1) { bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id); - ASSERT(0); SSDataBlock* pRes = NULL; struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); @@ -1757,8 +1758,7 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { - ASSERT(0); -// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); checkUpdateData(pInfo, true, pBlock, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); if (pInfo->pUpdateDataRes->info.rows > 0) { @@ -1934,8 +1934,7 @@ FETCH_NEXT_BLOCK: pBlock->info.calWin.ekey = INT64_MAX; pBlock->info.dataLoad = 1; if (pInfo->pUpdateInfo) { - ASSERT(0); -// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); } blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { @@ -3097,9 +3096,9 @@ _error: static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); static void destoryTableCountScanOperator(void* param); static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid); + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI); static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName); + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI); static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, @@ -3385,11 +3384,11 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca } if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); - buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid); + buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid, pAPI); pInfo->currGrpIdx++; } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { - buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName); + buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName, pAPI); pInfo->currGrpIdx++; } else { @@ -3434,7 +3433,7 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO } static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName) { + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI) { char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); @@ -3442,18 +3441,18 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.vnode); - ASSERT(0); - if (ntbNum != 0) { - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + + int64_t numOfTables = 0;//metaGetNtbNum(pInfo->readHandle.vnode); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables); + if (numOfTables != 0) { + fillTableCountScanDataBlock(pSupp, dbName, "", numOfTables, pRes); } } static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI) { char stbName[TSDB_TABLE_NAME_LEN] = {0}; - ASSERT(0); -// metaGetTableSzNameByUid(pInfo->readHandle.vnode, stbUid, stbName); + pAPI->metaFn.getTableNameByUid(pInfo->readHandle.vnode, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index cf58892920..290bc43cbe 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2755,7 +2755,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->pState = taosMemoryCalloc(1, sizeof(void)); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, @@ -2917,8 +2917,10 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, return TSDB_CODE_OUT_OF_MEMORY; } + pSup->stateStore = *pStore; + initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); - pSup->pState = taosMemoryCalloc(1, sizeof(void)); + pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pSup->pState) = *pState; pSup->stateStore.streamStateSetNumber(pSup->pState, -1); @@ -2940,7 +2942,6 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, return terrno; } - pSup->stateStore = *pStore; int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].saveHandle.pBuf = pSup->pResultBuf; From 09662138472bd86beeb1e9f38b1aa92d0f0e8ddc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 14:21:40 +0800 Subject: [PATCH 20/25] fix(arch): set the correct module api dependency. --- include/libs/executor/storageapi.h | 36 +----------- include/libs/function/function.h | 57 ++++++++++++++++--- include/libs/stream/streamState.h | 10 ---- source/dnode/vnode/src/vnd/vnodeInitApi.c | 7 +++ source/libs/executor/inc/executil.h | 2 +- source/libs/executor/inc/executorInt.h | 5 +- source/libs/executor/src/aggregateoperator.c | 8 +-- source/libs/executor/src/cachescanoperator.c | 2 +- .../libs/executor/src/eventwindowoperator.c | 4 +- source/libs/executor/src/executil.c | 3 +- source/libs/executor/src/executorInt.c | 4 +- source/libs/executor/src/filloperator.c | 13 +++-- source/libs/executor/src/groupoperator.c | 12 ++-- source/libs/executor/src/projectoperator.c | 6 +- source/libs/executor/src/scanoperator.c | 10 ++-- source/libs/executor/src/sortoperator.c | 4 +- source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/executor/src/timesliceoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 38 +++++++------ source/libs/function/src/builtinsimpl.c | 32 +++++------ source/libs/stream/src/streamState.c | 1 - 21 files changed, 135 insertions(+), 125 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0fd8a74127..1240f536ae 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -22,6 +22,7 @@ #include "tmsg.h" #include "tcommon.h" #include "index.h" +#include "function.h" #ifdef __cplusplus extern "C" { @@ -404,41 +405,9 @@ int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); */ } SStoreMeta; -typedef struct STdbState { - void* rocksdb; - void** pHandle; - void* writeOpts; - void* readOpts; - void** cfOpts; - void* dbOpt; - struct SStreamTask* pOwner; - void* param; - void* env; - SListNode* pComparNode; - void* pBackendHandle; - char idstr[64]; - void* compactFactory; - void* db; - void* pStateDb; - void* pFuncStateDb; - void* pFillStateDb; // todo refactor - void* pSessionStateDb; - void* pParNameDb; - void* pParTagDb; - void* txn; -} STdbState; -// incremental state storage -typedef struct { - STdbState* pTdbState; - struct SStreamFileState* pFileState; - int32_t number; - SSHashObj* parNameMap; - int64_t checkPointId; - int32_t taskId; - int64_t streamId; -} SStreamState; + typedef struct SUpdateInfo { SArray *pTsBuckets; @@ -546,6 +515,7 @@ typedef struct SStorageAPI { SStoreTqReader tqReaderFn; SStateStore stateStore; SMetaDataFilterAPI metaFilter; + SFunctionStateStore functionStore; } SStorageAPI; #ifdef __cplusplus diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 85f7cf7e2c..de2494ab3c 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -22,6 +22,7 @@ extern "C" { #include "tcommon.h" #include "tvariant.h" +#include "tsimplehash.h" struct SqlFunctionCtx; struct SResultRowEntryInfo; @@ -127,18 +128,59 @@ typedef struct SSerializeDataHandle { void *pState; } SSerializeDataHandle; +// incremental state storage +typedef struct STdbState { + void* rocksdb; + void** pHandle; + void* writeOpts; + void* readOpts; + void** cfOpts; + void* dbOpt; + struct SStreamTask* pOwner; + void* param; + void* env; + SListNode* pComparNode; + void* pBackendHandle; + char idstr[64]; + void* compactFactory; + + void* db; + void* pStateDb; + void* pFuncStateDb; + void* pFillStateDb; // todo refactor + void* pSessionStateDb; + void* pParNameDb; + void* pParTagDb; + void* txn; +} STdbState; + +typedef struct { + STdbState* pTdbState; + struct SStreamFileState* pFileState; + int32_t number; + SSHashObj* parNameMap; + int64_t checkPointId; + int32_t taskId; + int64_t streamId; +} SStreamState; + +typedef struct SFunctionStateStore { + int32_t (*streamStateFuncPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateFuncGet)(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); +} SFunctionStateStore; + // sql function runtime context typedef struct SqlFunctionCtx { SInputColumnInfoData input; SResultDataInfo resDataInfo; - uint32_t order; // data block scanner order: asc|desc - uint8_t isPseudoFunc;// denote current function is pseudo function or not [added for perf reason] - uint8_t isNotNullFunc;// not return null value. - uint8_t scanFlag; // record current running step, default: 0 - int16_t functionId; // function id - char *pOutput; // final result output buffer, point to sdata->data + uint32_t order; // data block scanner order: asc|desc + uint8_t isPseudoFunc; // denote current function is pseudo function or not [added for perf reason] + uint8_t isNotNullFunc; // not return null value. + uint8_t scanFlag; // record current running step, default: 0 + int16_t functionId; // function id + char *pOutput; // final result output buffer, point to sdata->data // input parameter, e.g., top(k, 20), the number of results of top query is kept in param - SFunctParam *param; + SFunctParam *param; // corresponding output buffer for timestamp of each result, e.g., diff/csum SColumnInfoData *pTsOutput; int32_t numOfParams; @@ -155,6 +197,7 @@ typedef struct SqlFunctionCtx { SSerializeDataHandle saveHandle; int32_t exprIdx; char *udfName; + SFunctionStateStore *pStore; } SqlFunctionCtx; typedef struct tExprNode { diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 08357b8251..ca85622d8a 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -66,16 +66,6 @@ int32_t streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); -//typedef struct { -// rocksdb_iterator_t* iter; -// rocksdb_snapshot_t* snapshot; -// rocksdb_readoptions_t* readOpt; -// rocksdb_t* db; -// -// TBC* pCur; -// int64_t number; -//} SStreamStateCur; - int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index d9bb4270d8..923e25f73b 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -24,6 +24,7 @@ static void initTqAPI(SStoreTqReader* pTq); static void initStateStoreAPI(SStateStore* pStore); static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); +static void initFunctionStateStore(SFunctionStateStore* pStore); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -32,6 +33,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initStateStoreAPI(&pAPI->stateStore); initMetaReaderAPI(&pAPI->metaReaderFn); initMetaFilterAPI(&pAPI->metaFilter); + initFunctionStateStore(&pAPI->functionStore); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -206,4 +208,9 @@ void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { pFilter->metaFilterTableIds = metaFilterTableIds; pFilter->metaFilterTableName = metaFilterTableName; pFilter->metaFilterTtl = metaFilterTtl; +} + +void initFunctionStateStore(SFunctionStateStore* pStore) { + pStore->streamStateFuncPut = streamStateFuncPut; + pStore->streamStateFuncGet = streamStateFuncGet; } \ No newline at end of file diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 6dc359c502..966c5fa2e4 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -166,7 +166,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId); void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); -SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); +SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 8188ecd305..79391bc1c5 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -579,12 +579,11 @@ void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); -int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore); void cleanupExprSupp(SExprSupp* pSup); - int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState); + const char* pkey, void* pState, SFunctionStateStore* pStore); void cleanupAggSup(SAggSupporter* pAggSup); void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 00aa52b593..b34c6f4b82 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -84,7 +84,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -95,7 +95,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); } - code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -485,8 +485,8 @@ void cleanupAggSup(SAggSupporter* pAggSup) { } int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState) { - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + const char* pkey, void* pState, SFunctionStateStore* pStore) { + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 1b6c06b6c4..48003b277b 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -126,7 +126,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe if (pScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* p = &pInfo->pseudoExprSup; p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs); - p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); + p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 956d5b714d..b5dea6d94d 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -92,7 +92,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo if (pEventWindowNode->window.pExprs != NULL) { int32_t numOfScalarExpr = 0; SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -110,7 +110,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo initResultSizeInfo(&pOperator->resultInfo, 4096); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 47b08b3323..ebe3d69dbb 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1544,7 +1544,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_SUCCESS; } -SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset) { +SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore) { SqlFunctionCtx* pFuncCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; @@ -1607,6 +1607,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->numOfParams = pExpr->base.numOfParams; pCtx->param = pFunct->pParam; pCtx->saveHandle.currentPage = -1; + pCtx->pStore = pStore; } for (int32_t i = 1; i < numOfOutput; ++i) { diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 45478cf837..5e0e7b0d6a 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -979,11 +979,11 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return NULL; } -int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) { +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore) { pSup->pExprInfo = pExprInfo; pSup->numOfExprs = numOfExpr; if (pSup->pExprInfo != NULL) { - pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset, pStore); if (pSup->pCtx == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index b448e8f2b0..90b7f4a77e 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -338,7 +338,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs); + code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -355,7 +355,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -525,7 +525,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, pFillSup->cur.key = key.ts; pFillSup->cur.pRowVal = curVal; - void* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); + SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; void* preVal = NULL; int32_t preVLen = 0; @@ -1139,7 +1139,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { TSKEY endTs = ts; uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; SWinKey key = {.ts = ts, .groupId = groupId}; - void* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + if (!pCur) { pInfo->srcDelRowIndex++; continue; @@ -1355,7 +1356,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod } SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols); + code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); if (code != TSDB_CODE_SUCCESS) { destroyStreamFillSupporter(pFillSup); return NULL; @@ -1491,7 +1492,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi goto _error; } - code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 77d251bab9..859fad3803 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -451,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -467,7 +467,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -849,7 +849,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition if (pPartNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { terrno = code; pTaskInfo->code = terrno; @@ -1242,7 +1242,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr if (pPartNode->part.pExprs != NULL) { int32_t num = 0; SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num); - code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num); + code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1257,7 +1257,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pPartNode->pSubtable, 0); - code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1); + code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1271,7 +1271,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index ae72627575..dde6f7c0e8 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -115,7 +115,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys initResultSizeInfo(&pOperator->resultInfo, numOfRows); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -370,7 +370,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy if (pPhyNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -392,7 +392,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy blockDataEnsureCapacity(pResBlock, numOfRows); int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 39d619172b..66e200c81b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -962,7 +962,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, if (pScanNode->pScanPseudoCols != NULL) { SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); - pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; @@ -2349,7 +2349,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0); - if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) { + if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore) != 0) { tableListDestroy(pTableListInfo); goto _error; } @@ -2363,7 +2363,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys tableListDestroy(pTableListInfo); goto _error; } - if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tableListDestroy(pTableListInfo); goto _error; @@ -2612,7 +2612,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3033,7 +3033,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pTableScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs); - pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 718bb596c5..0357828732 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -60,7 +60,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* } pOperator->exprSupp.pCtx = - createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); initResultSizeInfo(&pOperator->resultInfo, 1024); code = filterInitFromNode((SNode*)pSortNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -500,7 +500,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pSup->numOfExprs = numOfCols; initResultSizeInfo(&pOperator->resultInfo, 1024); - pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index d15b6d5754..40b45c25cd 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2301,7 +2301,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 209e5cb8fc..6ed247a76d 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -818,7 +818,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs); - int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs); + int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -826,7 +826,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode if (pInterpPhyNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 290bc43cbe..1adc171c5b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1656,7 +1656,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); int32_t code = - initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState); + initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1684,7 +1684,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1884,7 +1884,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi if (pStateNode->window.pExprs != NULL) { int32_t numOfScalarExpr = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr); - int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1910,7 +1910,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initResultSizeInfo(&pOperator->resultInfo, 4096); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1979,7 +1979,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2744,7 +2744,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2757,9 +2757,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2861,9 +2862,9 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock) { + SSDataBlock* pResultBlock, SFunctionStateStore* pStore) { initBasicInfo(pBasicInfo, pResultBlock); - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2936,6 +2937,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, if (bufSize <= pageSize) { bufSize = pageSize * 4; } + if (!osTempSpaceAvailable()) { terrno = TSDB_CODE_NO_DISKSPACE; qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir); @@ -3588,7 +3590,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh if (pSessionNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3597,7 +3599,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4126,7 +4128,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys if (pStateNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4145,7 +4147,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4433,7 +4435,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4719,7 +4721,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4764,7 +4766,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -4948,7 +4950,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pInfo->pState); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4956,7 +4958,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f83873848e..5c47daca5b 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,7 +18,6 @@ #include "function.h" #include "query.h" #include "querynodes.h" -//#include "streamState.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" @@ -1697,7 +1696,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); - int32_t code = getPercentile(pMemBucket, v, &ppInfo->result); + code = getPercentile(pMemBucket, v, &ppInfo->result); if (code != TSDB_CODE_SUCCESS) { goto _fin_error; } @@ -3120,7 +3119,7 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid } static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, SWinKey* key, - STuplePos* pPos) { + STuplePos* pPos, SFunctionStateStore* pStore) { STuplePos p = {0}; if (pHandle->pBuf != NULL) { SFilePage* pPage = NULL; @@ -3153,11 +3152,10 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, pPage->num += length; setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); - } else { - // other tuple save policy -// if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { -// p.streamTupleKey = *key; -// } + } else { // other tuple save policy + if (pStore->streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { + p.streamTupleKey = *key; + } } *pPos = p; @@ -3179,10 +3177,10 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos); + return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos, pCtx->pStore); } -static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) { +static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos, SFunctionStateStore* pStore) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); if (pPage == NULL) { @@ -3192,7 +3190,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { -// streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); + pStore->streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); } return TSDB_CODE_SUCCESS; @@ -3202,10 +3200,10 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc prepareBuf(pCtx); char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos); + return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore); } -static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) { +static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); if (pPage == NULL) { @@ -3217,13 +3215,13 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo } else { void* value = NULL; int32_t vLen; -// streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); + pStore->streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); return (char*)value; } } const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) { - return doLoadTupleData(&pCtx->saveHandle, pPos); + return doLoadTupleData(&pCtx->saveHandle, pPos, pCtx->pStore); } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { @@ -4991,7 +4989,7 @@ static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pI memcpy(pInfo->buf, data, pInfo->colBytes); } - return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos); + return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos, pCtx->pStore); } static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { @@ -5020,7 +5018,7 @@ static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCt } else { pHashItem->count += 1; if (pCtx->subsidiaries.num > 0) { - int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos); + code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 24339337b7..7af92cfb81 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -23,7 +23,6 @@ #include "tcommon.h" #include "tcompare.h" #include "tref.h" -#include "ttimer.h" #define MAX_TABLE_NAME_NUM 2000000 From c3be7b14bed50e8f680bd98fcb381c405cce9659 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 17:51:03 +0800 Subject: [PATCH 21/25] fix: set correct function ptr. --- include/libs/executor/executor.h | 1 - include/libs/executor/storageapi.h | 42 +++++++++++----------- source/dnode/snode/inc/sndInt.h | 2 ++ source/dnode/snode/src/snode.c | 4 +-- source/dnode/vnode/inc/vnode.h | 4 ++- source/dnode/vnode/src/meta/metaQuery.c | 4 +-- source/dnode/vnode/src/tq/tqRead.c | 3 ++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 12 +++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 8 +++++ source/libs/executor/src/executil.c | 6 ++-- source/libs/executor/src/executor.c | 15 ++++---- source/libs/executor/src/operator.c | 4 +-- source/libs/executor/src/querytask.c | 2 +- source/libs/executor/src/scanoperator.c | 42 +++++++++++----------- source/libs/executor/src/sysscanoperator.c | 20 +++++------ 15 files changed, 98 insertions(+), 71 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index c4c2c89fdb..e90bb4688b 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -42,7 +42,6 @@ typedef struct { typedef struct { void* tqReader; -// void* meta; void* config; void* vnode; void* mnd; diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 1240f536ae..5f7a91c057 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -84,7 +84,7 @@ typedef struct SMetaReader { SMetaEntry me; void * pBuf; int32_t szBuf; - struct SStorageAPI *storageAPI; + struct SStoreMeta* pAPI; } SMetaReader; typedef struct SMTbCursor { @@ -256,7 +256,7 @@ typedef struct SStoreCacheReader { void *(*closeReader)(void *pReader); int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUidList); - void (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); + int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); } SStoreCacheReader; /*------------------------------------------------------------------------------------------------------------------*/ @@ -290,6 +290,7 @@ typedef struct SStoreTqReader { int32_t (*tqRetrieveBlock)(); bool (*tqReaderNextBlockInWal)(); bool (*tqNextBlockImpl)(); // todo remove it + SSDataBlock* (*tqGetResultBlock)(); void (*tqReaderSetColIdList)(); int32_t (*tqReaderSetQueryTableList)(); @@ -345,14 +346,6 @@ int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ -typedef struct SStoreMetaReader { - void (*initReader)(SMetaReader *pReader, void *pMeta, int32_t flags); - void (*clearReader)(SMetaReader *pReader); - void (*readerReleaseLock)(SMetaReader *pReader); - int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); - int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); - int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); -} SStoreMetaReader; typedef struct SStoreMeta { SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor @@ -387,7 +380,7 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in void *(*storeGetIndexInfo)(); void *(*getInvertIndex)(void* pVnode); int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] - int32_t (*storeGetTableList)(); // vnodeGetStbIdList & vnodeGetAllTableList + int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList void *storeGetVersionRange; void *storeGetLastTimestamp; @@ -405,9 +398,14 @@ int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); */ } SStoreMeta; - - - +typedef struct SStoreMetaReader { + void (*initReader)(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); + void (*clearReader)(SMetaReader *pReader); + void (*readerReleaseLock)(SMetaReader *pReader); + int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); + int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); + int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); +} SStoreMetaReader; typedef struct SUpdateInfo { SArray *pTsBuckets; @@ -507,14 +505,14 @@ typedef struct SStateStore { } SStateStore; typedef struct SStorageAPI { - SStoreMeta metaFn; // todo: refactor - TsdReader tsdReader; - SStoreMetaReader metaReaderFn; - SStoreCacheReader cacheFn; - SStoreSnapshotFn snapshotFn; - SStoreTqReader tqReaderFn; - SStateStore stateStore; - SMetaDataFilterAPI metaFilter; + SStoreMeta metaFn; // todo: refactor + TsdReader tsdReader; + SStoreMetaReader metaReaderFn; + SStoreCacheReader cacheFn; + SStoreSnapshotFn snapshotFn; + SStoreTqReader tqReaderFn; + SStateStore stateStore; + SMetaDataFilterAPI metaFilter; SFunctionStateStore functionStore; } SStorageAPI; diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 3fcee862a1..68f7f756d5 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -53,6 +53,8 @@ int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId); int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId); #endif +void initStreamStateAPI(SStorageAPI* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 24e2b13f46..678dd34e4a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -62,8 +62,7 @@ FAIL: } int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); - ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); + ASSERT(pTask->taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->childEpInfo) != 0); pTask->refCnt = 1; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; @@ -88,6 +87,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; + initStreamStateAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0); ASSERT(pTask->exec.pExecutor); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2723e869ff..e8ae545ea0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -69,6 +69,7 @@ int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); +int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeIsCatchUp(SVnode *pVnode); ESyncRole vnodeGetRole(SVnode *pVnode); @@ -105,7 +106,7 @@ typedef struct SMetaEntry SMetaEntry; #define META_READER_NOLOCK 0x1 -void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags); +void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); @@ -257,6 +258,7 @@ int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); bool tqNextBlockInWal(STqReader *pReader, const char *idstr); bool tqNextBlockImpl(STqReader *pReader, const char *idstr); SWalReader* tqGetWalReader(STqReader* pReader); +SSDataBlock* tqGetResultBlock (STqReader* pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 255fa10f57..7eb6460af5 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,14 +17,14 @@ #include "osMemory.h" #include "tencode.h" -void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags) { +void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; metaReaderInit(pReader, pMeta, flags); + pReader->pAPI = pAPI; } void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); - pReader->flags = flags; pReader->pMeta = pMeta; if (pReader->pMeta && !(flags & META_READER_NOLOCK)) { metaRLock(pMeta); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2521d956a3..dbb7d564f8 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -446,6 +446,9 @@ SWalReader* tqGetWalReader(STqReader* pReader) { return pReader->pWalReader; } +SSDataBlock* tqGetResultBlock (STqReader* pReader) { + return pReader->pResBlock; +} bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 923e25f73b..5317c0e675 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -25,6 +25,7 @@ static void initStateStoreAPI(SStateStore* pStore); static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); static void initFunctionStateStore(SFunctionStateStore* pStore); +static void initCacheFn(SStoreCacheReader* pCache); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -34,6 +35,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initMetaReaderAPI(&pAPI->metaReaderFn); initMetaFilterAPI(&pAPI->metaFilter); initFunctionStateStore(&pAPI->functionStore); + initCacheFn(&pAPI->cacheFn); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -83,6 +85,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableNameByUid = metaGetTableNameByUid; pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor + pMeta->storeGetTableList = vnodeGetTableList; } void initTqAPI(SStoreTqReader* pTq) { @@ -109,6 +112,8 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqGetResultBlock = tqGetResultBlock; + pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; } @@ -213,4 +218,11 @@ void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { void initFunctionStateStore(SFunctionStateStore* pStore) { pStore->streamStateFuncPut = streamStateFuncPut; pStore->streamStateFuncGet = streamStateFuncGet; +} + +void initCacheFn(SStoreCacheReader* pCache) { + pCache->openReader = tsdbCacherowsReaderOpen; + pCache->closeReader = tsdbCacherowsReaderClose; + pCache->retrieveRows = tsdbRetrieveCacheRows; + pCache->reuseReader = tsdbReuseCacherowsReader; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index ac3e632172..022fc4c951 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -431,6 +431,14 @@ void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* num } } +int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) { + if (type == TSDB_SUPER_TABLE) { + return vnodeGetStbIdList(pVnode, 0, pList); + } else { + return TSDB_CODE_INVALID_PARA; + } +} + int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, uid, 1); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ebe3d69dbb..562a6daf8a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -263,7 +263,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { STagVal tagVal = {0}; tagVal.cid = pSColumnNode->colId; - const char* p = mr->storageAPI->metaFn.extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); + const char* p = mr->pAPI->extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { @@ -306,7 +306,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, int32_t code = TSDB_CODE_SUCCESS; SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, metaHandle, 0); + pAPI->metaReaderFn.initReader(&mr, metaHandle, 0, &pAPI->metaFn); code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, info->uid); if (TSDB_CODE_SUCCESS != code) { pAPI->metaReaderFn.clearReader(&mr); @@ -1168,7 +1168,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, SStorageAPI* pAPI) { SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pVnode, 0); + pAPI->metaReaderFn.initReader(&mr, pVnode, 0, &pAPI->metaFn); if (pAPI->metaReaderFn.getEntryGetUidCache(&mr, uid) != 0) { // table not exist pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 53fad6503e..14f5c3f4fa 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -341,7 +341,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0, &pAPI->metaFn); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); @@ -1091,12 +1091,13 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; - ASSERT(0); -// walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); -// if (tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) { -// qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id); -// return -1; -// } + SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn; + SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader); + walReaderVerifyOffset(pWalReader, pOffset); + if (pReaderAPI->tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) { + qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id); + return -1; + } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { // iterate all tables from tableInfoList, and retrieve rows from each table one-by-one // those data are from the snapshot in tsdb, besides the data in the wal file. diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 8521fb623e..935e6a0b8b 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -381,9 +381,9 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR if (pBlockNode->tableType == TSDB_SUPER_TABLE) { SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); - int32_t code = pTaskInfo->storageAPI.metaFn.storeGetTableList(pHandle->vnode, pBlockNode->uid, pList); + int32_t code = pTaskInfo->storageAPI.metaFn.getChildTableList(pHandle->vnode, pBlockNode->uid, pList); if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = terrno; + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index f79966a021..22d171e74a 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -122,7 +122,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo SStorageAPI* pAPI = &pTaskInfo->storageAPI; - pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pAPI->metaFn); int32_t code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 66e200c81b..8f11cd769a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -531,7 +531,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. @@ -560,7 +560,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { - pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -1330,9 +1330,9 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { - uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); + uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver); // gap must be 0. SSessionKey startWin = {0}; getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], startData[i], groupId, &startWin); @@ -1378,13 +1378,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; if (pInfo->partitionSup.needCalc && srcStartTsCol[0] != srcEndTsCol[0]) { uint64_t srcUid = srcUidData[0]; TSKEY startTs = srcStartTsCol[0]; TSKEY endTs = srcEndTsCol[0]; - SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, version); + SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, ver); printDataBlock(pPreRes, "pre res"); blockDataCleanup(pSrcBlock); int32_t code = blockDataEnsureCapacity(pSrcBlock, pPreRes->info.rows); @@ -1422,7 +1422,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; if (groupId == 0) { - groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver); } TSKEY calStartTs = srcStartTsCol[i]; colDataSetVal(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); @@ -1459,13 +1459,13 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; char* tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0}; if (groupId == 0) { - groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver); } if (pInfo->tbnameCalSup.pExprInfo) { void* parTbname = NULL; @@ -1676,7 +1676,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { while (1) { bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id); - SSDataBlock* pRes = NULL; + SSDataBlock* pRes = pAPI->tqReaderFn.tqGetResultBlock(pInfo->tqReader); struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); // curVersion move to next, so currentOffset = curVersion - 1 @@ -2558,7 +2558,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { char str[512] = {0}; int32_t count = 0; SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); @@ -3185,6 +3185,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo)); return code; } + supp->dbNameSlotId = -1; supp->stbNameSlotId = -1; supp->tbCountSlotId = -1; @@ -3194,6 +3195,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo)); return code; } + code = tblCountScanGetCountSlotId(pseudoCols, supp); if (code != TSDB_CODE_SUCCESS) { qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo)); @@ -3378,7 +3380,7 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca if (pSupp->groupByStbName) { if (pInfo->stbUidList == NULL) { pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, 0, pInfo->stbUidList, TSDB_SUPER_TABLE) < 0) { + if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, TSDB_SUPER_TABLE, pInfo->stbUidList) < 0) { qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); } } @@ -3412,7 +3414,7 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) { - tb_uid_t uid = 0; + uint64_t uid = 0; pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); int64_t numOfChildTables = 0; @@ -3420,7 +3422,8 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes); } else { - int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.vnode); + int64_t tbNumVnode = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { @@ -3444,6 +3447,7 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S int64_t numOfTables = 0;//metaGetNtbNum(pInfo->readHandle.vnode); pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables); + if (numOfTables != 0) { fillTableCountScanDataBlock(pSupp, dbName, "", numOfTables, pRes); } @@ -3456,18 +3460,16 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, varDataVal(stbName)); } else { - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName); + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", varDataVal(stbName)); } uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - SMetaStbStats stats = {0}; -// metaGetStbStats(pInfo->readHandle.vnode, stbUid, &stats); - int64_t ctbNum = stats.ctbNum; - + int64_t ctbNum = 0; + int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum); fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 40b45c25cd..fd766e21f8 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -466,7 +466,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -486,7 +486,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (smrTable.me.type == TSDB_CHILD_TABLE) { int64_t suid = smrTable.me.ctbEntry.suid; pAPI->metaReaderFn.clearReader(&smrTable); - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -569,7 +569,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -658,7 +658,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, condTableName); SMetaReader smrChildTable = {0}; - pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly @@ -676,7 +676,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid @@ -715,7 +715,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { @@ -1131,7 +1131,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn); ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); if (ret < 0) { pAPI->metaReaderFn.clearReader(&mr); @@ -1159,7 +1159,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; - pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); int64_t suid = mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); @@ -1338,7 +1338,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; - pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); @@ -2148,7 +2148,7 @@ static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* ro *rowLen = 0; SMetaReader mr = {0}; - pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); From 30de31e37431ac2e793769141692b0680d6e836f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 18:17:09 +0800 Subject: [PATCH 22/25] fix(test): fix unit test link error. --- source/libs/stream/test/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index 160cddbd4c..049bfbbb3a 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -15,11 +15,11 @@ ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") IF (TD_GRANT) TARGET_LINK_LIBRARIES(streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor grant + PUBLIC os util common gtest gtest_main stream executor index grant ) ELSE () TARGET_LINK_LIBRARIES(streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor + PUBLIC os util common gtest gtest_main stream executor index ) ENDIF() From 1090afd10884d91de4f074e793685100faa7e45e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 18:46:53 +0800 Subject: [PATCH 23/25] fix(stream): add new file. --- source/dnode/snode/src/snodeInitApi.c | 109 ++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 source/dnode/snode/src/snodeInitApi.c diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c new file mode 100644 index 0000000000..f5e9245252 --- /dev/null +++ b/source/dnode/snode/src/snodeInitApi.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "storageapi.h" +#include "tstreamUpdate.h" +#include "streamState.h" + +static void initStateStoreAPI(SStateStore* pStore); +static void initFunctionStateStore(SFunctionStateStore* pStore); + +void initStreamStateAPI(SStorageAPI* pAPI) { + initStateStoreAPI(&pAPI->stateStore); + initFunctionStateStore(&pAPI->functionStore); +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; +} + +void initFunctionStateStore(SFunctionStateStore* pStore) { + pStore->streamStateFuncPut = streamStateFuncPut; + pStore->streamStateFuncGet = streamStateFuncGet; +} \ No newline at end of file From 1e22d875b3d948dfe7c65d8a03481b7c017f7980 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 25 May 2023 20:00:26 +0800 Subject: [PATCH 24/25] fix: fix dead lock. --- source/dnode/vnode/src/meta/metaQuery.c | 1 + source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 7eb6460af5..26182baf7b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -26,6 +26,7 @@ void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMe void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->pMeta = pMeta; + pReader->flags = flags; if (pReader->pMeta && !(flags & META_READER_NOLOCK)) { metaRLock(pMeta); } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 5317c0e675..00c1dacdfd 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -200,7 +200,6 @@ void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { pMetaReader->clearReader = metaReaderClear; pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; - pMetaReader->clearReader = metaReaderClear; pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; pMetaReader->getTableEntryByName = metaGetTableEntryByName; From e24ce2111ff85eac87306edfd49e869155554e37 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 00:00:07 +0800 Subject: [PATCH 25/25] fix: set function ptr. --- include/libs/executor/storageapi.h | 14 +++++++------- source/dnode/vnode/inc/vnode.h | 4 ++-- source/dnode/vnode/src/meta/metaSnapshot.c | 6 +++--- source/dnode/vnode/src/tq/tqMeta.c | 2 ++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 10 +++++++++- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/operator.c | 7 ++++--- source/libs/executor/src/scanoperator.c | 9 +++++---- 8 files changed, 33 insertions(+), 21 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 5f7a91c057..9ed47c94f8 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -196,8 +196,8 @@ typedef struct { // int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); -// int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); -// SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); +// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); +// SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); // int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); // int32_t destroySnapContext(SSnapContext *ctx); @@ -310,15 +310,15 @@ typedef struct SStoreTqReader { typedef struct SStoreSnapshotFn { /* - int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); - SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); + int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); + SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); */ int32_t (*createSnapshot)(); - void (*destroySnapshot)(); - SMetaTableInfo (*getTableInfoFromSnapshot)(); - int32_t (*getMetaInfoFromSnapshot)(); + int32_t (*destroySnapshot)(); + SMetaTableInfo (*getMetaTableInfoFromSnapshot)(); + int32_t (*getTableInfoFromSnapshot)(); } SStoreSnapshotFn; /** diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e8ae545ea0..72c8dce52f 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -282,8 +282,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, SSnapContext **ctxRet); -int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); -SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); +int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); +SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index ff4355fdf8..f4e930e509 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -466,7 +466,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { return c; } -int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { +int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { int32_t ret = 0; void* pKey = NULL; void* pVal = NULL; @@ -598,7 +598,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in return ret; } -SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { +SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext* ctx) { SMetaTableInfo result = {0}; void* pKey = NULL; void* pVal = NULL; @@ -619,7 +619,7 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp); if (ret != 0) { - metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); + metaDebug("tmqsnap getMetaTableInfoFromSnapshot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); continue; } tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index d5cfa86e4c..be5b5706e2 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -304,6 +304,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { .version = handle.snapshotVer }; + initStorageAPI(&reader.api); + if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { handle.execHandle.task = qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 00c1dacdfd..1cc985f7df 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -26,6 +26,7 @@ static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); static void initFunctionStateStore(SFunctionStateStore* pStore); static void initCacheFn(SStoreCacheReader* pCache); +static void initSnapshotFn(SStoreSnapshotFn* pSnapshot); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); @@ -36,6 +37,7 @@ void initStorageAPI(SStorageAPI* pAPI) { initMetaFilterAPI(&pAPI->metaFilter); initFunctionStateStore(&pAPI->functionStore); initCacheFn(&pAPI->cacheFn); + initSnapshotFn(&pAPI->snapshotFn); } void initTsdbReaderAPI(TsdReader* pReader) { @@ -69,7 +71,6 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getBasicInfo = vnodeGetInfo; pMeta->getNumOfChildTables = metaGetStbStats; -// pMeta->getNumOfRowsInMem = tsdbGetNumOfRowsInMemTable; pMeta->getChildTableList = vnodeGetCtbIdList; @@ -224,4 +225,11 @@ void initCacheFn(SStoreCacheReader* pCache) { pCache->closeReader = tsdbCacherowsReaderClose; pCache->retrieveRows = tsdbRetrieveCacheRows; pCache->reuseReader = tsdbReuseCacherowsReader; +} + +void initSnapshotFn(SStoreSnapshotFn* pSnapshot) { + pSnapshot->createSnapshot = setForSnapShot; + pSnapshot->destroySnapshot = destroySnapContext; + pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot; + pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot; } \ No newline at end of file diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 14f5c3f4fa..fb35b211c9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1190,7 +1190,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT return -1; } - SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getTableInfoFromSnapshot(sContext); + SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getMetaTableInfoFromSnapshot(sContext); pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 935e6a0b8b..730252c7ee 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -380,17 +380,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR STableListInfo* pTableListInfo = tableListCreate(); if (pBlockNode->tableType == TSDB_SUPER_TABLE) { - SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); + SArray* pList = taosArrayInit(4, sizeof(uint64_t)); int32_t code = pTaskInfo->storageAPI.metaFn.getChildTableList(pHandle->vnode, pBlockNode->uid, pList); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; + taosArrayDestroy(pList); return NULL; } size_t num = taosArrayGetSize(pList); for (int32_t i = 0; i < num; ++i) { - STableKeyInfo* p = taosArrayGet(pList, i); - tableListAddTableInfo(pTableListInfo, p->uid, 0); + uint64_t* id = taosArrayGet(pList, i); + tableListAddTableInfo(pTableListInfo, *id, 0); } taosArrayDestroy(pList); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8f11cd769a..3984e9a792 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2185,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return pBlock; } - SMetaTableInfo mtInfo = pAPI->snapshotFn.getTableInfoFromSnapshot(pInfo->sContext); + SMetaTableInfo mtInfo = pAPI->snapshotFn.getMetaTableInfoFromSnapshot(pInfo->sContext); STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); @@ -2203,8 +2203,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { int32_t dataLen = 0; int16_t type = 0; int64_t uid = 0; - if (pAPI->snapshotFn.getMetaInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { - qError("tmqsnap getMetafromSnapShot error"); + if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { + qError("tmqsnap getTableInfoFromSnapshot error"); taosMemoryFreeClear(data); return NULL; } @@ -2253,6 +2253,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->pTableListInfo = tableListCreate(); pInfo->vnode = pHandle->vnode; + pInfo->pAPI = &pTaskInfo->storageAPI; pInfo->sContext = pHandle->sContext; setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, @@ -3470,7 +3471,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S int64_t ctbNum = 0; int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum); - fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); + fillTableCountScanDataBlock(pSupp, dbName, varDataVal(stbName), ctbNum, pRes); } static void destoryTableCountScanOperator(void* param) {