diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 3aa1ffc07e..e4d440d76e 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -87,7 +87,7 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_ARM_32 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") - ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") MESSAGE(STATUS "The current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") SET(TD_ARM_64 TRUE) diff --git a/cmake/cmake.version b/cmake/cmake.version index db29644b38..7c895b12ff 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.0.1") + SET(TD_VER_NUMBER "3.0.1.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index f182beed33..eb0faf6d5d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG abed566 + GIT_TAG 71e7ccf SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index b905e30489..adefacc6af 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG a4d9b92 + GIT_TAG 7d5c1c0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 54e898ecc0..04b1262caf 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 6fc47d7 + GIT_TAG e771403 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index e2be419517..35e7ad1933 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -62,7 +62,7 @@ taos> ## 体验查询 -使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。 +使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。 查询超级表下记录总条数： diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index ef1cd4797a..3239eae49b 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -116,7 +116,7 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 参数的具体含义是： - inputDataBlock: 输入的数据块 - - resultColumn: 输出列。输出列 + - resultColumn: 输出列 ### 聚合接口函数 diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 2337f2a272..f6790e3c69 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -23,10 +23,7 @@ create_subtable_clause: { } create_definition: - col_name column_definition - -column_definition: - type_name [comment 'string_value'] + col_name column_type table_options: table_option ... diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 92c814067e..09fea45b7a 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -6,7 +6,8 @@ description: TDengine 保留关键字的详细列表 ## 保留关键字 -目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下： +目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写如果需要用作库名、表名、STable 名、数据列名及标签列名等，需要使用符合``将关键字括起来使用，例如`ADD`。 +关键字列表如下： ### A @@ -239,6 +240,7 @@ description: TDengine 保留关键字的详细列表 - TOPICS - TRIGGER - TSERIES +- TTL ### U diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index e3e1463131..a6ec560d3c 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,11 +6,7 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; -## 3.0.0.1 +## 3.0.1.0 - - - + diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 61129d74e5..9e8757cc4e 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,6 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; -## 2.1.2 +## 2.1.3 - \ No newline at end of file + diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 006ba7f21b..3f917ff0d1 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -272,6 +272,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SET_MNODE_STANDBY, "set-mnode-standby", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) #if defined(TD_MSG_NUMBER_) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2c27509008..afd8de6b1c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -554,6 +554,8 @@ typedef struct { int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateDel(SStreamState* pState, const SWinKey* key); +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); void streamFreeVal(void* val); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 6c95c3c6d7..de2271554d 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -444,6 +444,70 @@ void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg); +// --------------------------------------------- +typedef struct SyncHeartbeat { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + SyncIndex commitIndex; + SyncTerm privateTerm; +} SyncHeartbeat; + +SyncHeartbeat* syncHeartbeatBuild(int32_t vgId); +void syncHeartbeatDestroy(SyncHeartbeat* pMsg); +void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen); +void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg); +char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len); +SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len); +void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg); +void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg); +SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg); +char* syncHeartbeat2Str(const SyncHeartbeat* pMsg); + +// for debug ---------------------- +void syncHeartbeatPrint(const SyncHeartbeat* pMsg); +void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg); +void syncHeartbeatLog(const SyncHeartbeat* pMsg); +void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg); + +// --------------------------------------------- +typedef struct SyncHeartbeatReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + SyncTerm privateTerm; + int64_t startTime; +} SyncHeartbeatReply; + +SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId); +void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg); +void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen); +void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg); +char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len); +SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len); +void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg); +void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg); +SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg); +char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg); + +// for debug ---------------------- +void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg); + // --------------------------------------------- typedef struct SyncApplyMsg { uint32_t bytes; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 939439ca56..feb69bf74f 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndCluster.h" #include "mndDnode.h" #include "mndOffset.h" #include "mndPrivilege.h" @@ -1592,6 +1593,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); if (i == 0) { colDataAppend(pColInfo, rows, buf, false); + } else if (i == 1) { + colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false); } else if (i == 3) { colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); } else if (i == 14) { @@ -1712,18 +1715,18 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, taosMemoryFree(buf); } -static void setInformationSchemaDbCfg(SDbObj *pDbObj) { +static void setInformationSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) { tstrncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name)); - pDbObj->createdTime = 0; + pDbObj->createdTime = mndGetClusterCreateTime(pMnode); pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; } -static void setPerfSchemaDbCfg(SDbObj *pDbObj) { +static void setPerfSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) { tstrncpy(pDbObj->name, TSDB_PERFORMANCE_SCHEMA_DB, tListLen(pDbObj->name)); - pDbObj->createdTime = 0; + pDbObj->createdTime = mndGetClusterCreateTime(pMnode); pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; @@ -1754,7 +1757,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; - setInformationSchemaDbCfg(&infoschemaDb); + setInformationSchemaDbCfg(pMnode, &infoschemaDb); size_t numOfTables = 0; getVisibleInfosTablesNum(sysinfo, &numOfTables); mndDumpDbInfoData(pMnode, pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); @@ -1762,7 +1765,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc numOfRows += 1; SDbObj perfschemaDb = {0}; - setPerfSchemaDbCfg(&perfschemaDb); + setPerfSchemaDbCfg(pMnode, &perfschemaDb); numOfTables = 0; getPerfDbMeta(NULL, &numOfTables); mndDumpDbInfoData(pMnode, pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index c9d1c3d015..165f5da4b6 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -211,6 +211,7 @@ int32_t tdRSmaFSOpen(SSma *pSma, int64_t version); void tdRSmaFSClose(SRSmaFS *fs); int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version); void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version); +int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat); int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile); int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer); int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d45a6f19f0..f77b6d56c4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -643,20 +643,37 @@ typedef struct { TSDBROW row; } SRowInfo; +typedef struct SSttBlockLoadInfo { + SBlockData blockData[2]; + SArray *aSttBlk; + int32_t blockIndex[2]; // to denote the loaded block in the corresponding position. + int32_t currentLoadBlockIndex; + int32_t loadBlocks; + double elapsedTime; +} SSttBlockLoadInfo; + typedef struct SMergeTree { int8_t backward; SRBTree rbt; SArray *pIterList; SLDataIter *pIter; + bool destroyLoadInfo; + SSttBlockLoadInfo* pLoadInfo; + const char *idStr; } SMergeTree; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange); + STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pLoadInfo, const char* idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); +SSttBlockLoadInfo* tCreateLastBlockLoadInfo(); +void resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo); +void getLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo, int64_t* blocks, double* el); +void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo); + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index fd300ef34a..f08cefdb04 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -182,6 +182,7 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { SVnode *pVnode = pSma->pVnode; SRSmaFS *pFS = RSMA_FS(pStat); int64_t committed = pStat->commitAppliedVer; + int64_t fsMaxVer = -1; char qTaskInfoFullName[TSDB_FILENAME_LEN]; taosWLockLatch(RSMA_FS_LOCK(pStat)); @@ -204,10 +205,20 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { ++i; } - SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; - if (tdRSmaFSUpsertQTaskFile(pFS, &qFile) < 0) { - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - return TSDB_CODE_FAILED; + if (taosArrayGetSize(pFS->aQTaskInf) > 0) { + fsMaxVer = ((SQTaskFile *)taosArrayGetLast(pFS->aQTaskInf))->version; + } + + if (fsMaxVer < committed) { + SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; + if (taosArrayPush(pFS->aQTaskInf, &qFile) < 0) { + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } else { + smaDebug("vgId:%d, update qinf, no need as committed %" PRIi64 " not larger than fsMaxVer %" PRIi64, TD_VID(pVnode), + committed, fsMaxVer); } taosWUnLockLatch(RSMA_FS_LOCK(pStat)); @@ -365,7 +376,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } - SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); // step 1: merge qTaskInfo and iQTaskInfo // lock diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index 4387db553e..8e8611f0e8 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -49,7 +49,7 @@ int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) { for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { int32_t vid = 0; int64_t version = -1; - sscanf((const char *)taosArrayGetP(output, i), "v%dqinfo.v%" PRIi64, &vid, &version); + sscanf((const char *)taosArrayGetP(output, i), "v%dqinf.v%" PRIi64, &vid, &version); SQTaskFile qTaskFile = {.version = version, .nRef = 1}; if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { goto _end; @@ -96,6 +96,18 @@ int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { return oldVal; } +int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat) { + SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; + int64_t version = -1; + + taosRLockLatch(RSMA_FS_LOCK(pStat)); + if (taosArrayGetSize(aQTaskInf) > 0) { + version = ((SQTaskFile *)taosArrayGetLast(aQTaskInf))->version; + } + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + return version; +} + void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { SVnode *pVnode = pSma->pVnode; SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index fdd173cdf7..920d04bf6c 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1342,29 +1342,31 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF return TSDB_CODE_FAILED; } - if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) { + if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { return TSDB_CODE_FAILED; } int32_t infoLen = 0; - taosDecodeFixedI32(pIter->qBuf, &infoLen); + taosDecodeFixedI32(pIter->pBuf, &infoLen); if (infoLen > nBytes) { if (infoLen <= RSMA_QTASKINFO_BUFSIZE) { terrno = TSDB_CODE_RSMA_FILE_CORRUPTED; smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); return TSDB_CODE_FAILED; } - pIter->nAlloc = infoLen; - void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); - if (!pBuf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + if (pIter->nAlloc < infoLen) { + pIter->nAlloc = infoLen; + void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); + if (!pBuf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pIter->pBuf = pBuf; } - pIter->pBuf = pBuf; - pIter->qBuf = pIter->pBuf; + nBytes = infoLen; - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { return TSDB_CODE_FAILED; } @@ -1373,6 +1375,7 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF } } + pIter->qBuf = pIter->pBuf; pIter->offset += nBytes; pIter->nBytes = nBytes; pIter->nBufPos = 0; @@ -1450,17 +1453,24 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { return TSDB_CODE_SUCCESS; } + int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); + if (pRSmaStat->commitAppliedVer <= fsMaxVer) { + smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, + pRSmaStat->commitAppliedVer, fsMaxVer); + return TSDB_CODE_SUCCESS; + } + STFile tFile = {0}; #if 0 if (pRSmaStat->commitAppliedVer > 0) { char qTaskInfoFName[TSDB_FILENAME_LEN]; tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); + smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; } if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); + smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); goto _err; } smaDebug("vgId:%d, rsma, serialize qTaskInfo, file %s created", vid, TD_TFILE_FULL_NAME(&tFile)); @@ -1510,11 +1520,11 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { char qTaskInfoFName[TSDB_FILENAME_LEN]; tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); + smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; } if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); + smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); goto _err; } smaDebug("vgId:%d, rsma, table %" PRIi64 " serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid, @@ -1558,7 +1568,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { } return TSDB_CODE_SUCCESS; _err: - smaError("vgId:%d, rsma persit failed since %s", vid, terrstr()); + smaError("vgId:%d, rsma persist failed since %s", vid, terrstr()); if (isFileCreated) { tdRemoveTFile(&tFile); tdDestroyTFile(&tFile); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 9ac62b4b59..3ddba8e656 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -457,7 +457,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}); + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL); bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index a072f22fa9..45fe29f0fa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -22,26 +22,207 @@ struct SLDataIter { SDataFReader *pReader; int32_t iStt; int8_t backward; - SArray *aSttBlk; int32_t iSttBlk; - SBlockData bData[2]; - int32_t loadIndex; int32_t iRow; SRowInfo rInfo; uint64_t uid; STimeWindow timeWindow; SVersionRange verRange; + + SSttBlockLoadInfo* pBlockLoadInfo; }; -static SBlockData *getCurrentBlock(SLDataIter *pIter) { return &pIter->bData[pIter->loadIndex]; } +SSttBlockLoadInfo* tCreateLastBlockLoadInfo() { + SSttBlockLoadInfo* pLoadInfo = taosMemoryCalloc(TSDB_DEFAULT_STT_FILE, sizeof(SSttBlockLoadInfo)); + if (pLoadInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } -static SBlockData *getNextBlock(SLDataIter *pIter) { - pIter->loadIndex ^= 1; - return getCurrentBlock(pIter); + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + pLoadInfo[i].blockIndex[0] = -1; + pLoadInfo[i].blockIndex[1] = -1; + pLoadInfo[i].currentLoadBlockIndex = 1; + + int32_t code = tBlockDataCreate(&pLoadInfo[i].blockData[0]); + if (code) { + terrno = code; + } + + code = tBlockDataCreate(&pLoadInfo[i].blockData[1]); + if (code) { + terrno = code; + } + + pLoadInfo[i].aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); + } + + return pLoadInfo; +} + +void resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) { + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + pLoadInfo[i].currentLoadBlockIndex = 1; + pLoadInfo[i].blockIndex[0] = -1; + pLoadInfo[i].blockIndex[1] = -1; + + taosArrayClear(pLoadInfo[i].aSttBlk); + + pLoadInfo[i].elapsedTime = 0; + pLoadInfo[i].loadBlocks = 0; + } +} + +void getLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo, int64_t* blocks, double* el) { + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + *el += pLoadInfo[i].elapsedTime; + *blocks += pLoadInfo[i].loadBlocks; + } +} + +void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) { + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + pLoadInfo[i].currentLoadBlockIndex = 1; + pLoadInfo[i].blockIndex[0] = -1; + pLoadInfo[i].blockIndex[1] = -1; + + tBlockDataDestroy(&pLoadInfo[i].blockData[0], true); + tBlockDataDestroy(&pLoadInfo[i].blockData[1], true); + + taosArrayDestroy(pLoadInfo[i].aSttBlk); + } + + taosMemoryFree(pLoadInfo); + return NULL; +} + +static SBlockData* loadLastBlock(SLDataIter *pIter, const char* idStr) { + int32_t code = 0; + + SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo; + if (pInfo->blockIndex[0] == pIter->iSttBlk) { + return &pInfo->blockData[0]; + } + + if (pInfo->blockIndex[1] == pIter->iSttBlk) { + return &pInfo->blockData[1]; + } + + pInfo->currentLoadBlockIndex ^= 1; + if (pIter->pSttBlk != NULL) { // current block not loaded yet + int64_t st = taosGetTimestampUs(); + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, &pInfo->blockData[pInfo->currentLoadBlockIndex]); + double el = (taosGetTimestampUs() - st)/ 1000.0; + pInfo->elapsedTime += el; + pInfo->loadBlocks += 1; + + tsdbDebug("read last block, index:%d, last file index:%d, elapsed time:%.2f ms, %s", pIter->iSttBlk, pIter->iStt, el, idStr); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; + pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1; + } + + return &pInfo->blockData[pInfo->currentLoadBlockIndex]; + + _exit: + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + } + + return NULL; +} + +// find the earliest block that contains the required records +static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t num, int32_t backward) { + int32_t i = index; + int32_t step = backward? 1:-1; + while (i >= 0 && i < num && uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid) { + i += step; + } + return i - step; +} + +static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64_t uid, int32_t backward) { + int32_t midPos = -1; + if (num <= 0) { + return -1; + } + + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + // find the first position which is bigger than the key + if ((uid > pBlockList[lastPos].maxUid) || (uid < pBlockList[firstPos].minUid)) { + return -1; + } + + while (1) { + if (uid >= pBlockList[firstPos].minUid && uid <= pBlockList[firstPos].maxUid) { + return findEarliestIndex(firstPos, uid, pBlockList, num, backward); + } + + if (uid > pBlockList[lastPos].maxUid || uid < pBlockList[firstPos].minUid) { + return -1; + } + + int32_t numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (uid < pBlockList[midPos].minUid) { + lastPos = midPos - 1; + } else if (uid > pBlockList[midPos].maxUid) { + firstPos = midPos + 1; + } else { + return findEarliestIndex(midPos, uid, pBlockList, num, backward); + } + } +} + +static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t num, int32_t backward) { + int32_t i = index; + int32_t step = backward? 1:-1; + while (i >= 0 && i < num && uid == uidList[i]) { + i += step; + } + return i - step; +} + +static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint64_t uid, int32_t backward) { + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + // find the first position which is bigger than the key + if ((uid > uidList[lastPos]) || (uid < uidList[firstPos])) { + return -1; + } + + while (1) { + if (uid == uidList[firstPos]) { + return findEarliestRow(firstPos, uid, uidList, num, backward); + } + + if (uid > uidList[lastPos] || uid < uidList[firstPos]) { + return -1; + } + + int32_t numOfRows = lastPos - firstPos + 1; + int32_t midPos = (numOfRows >> 1u) + firstPos; + + if (uid < uidList[midPos]) { + lastPos = midPos - 1; + } else if (uid > uidList[midPos]) { + firstPos = midPos + 1; + } else { + return findEarliestRow(midPos, uid, uidList, num, backward); + } + } } int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, - uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) { + uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) { int32_t code = 0; *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); if (*pIter == NULL) { @@ -55,60 +236,34 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t (*pIter)->backward = backward; (*pIter)->verRange = *pRange; (*pIter)->timeWindow = *pTimeWindow; - (*pIter)->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); - if ((*pIter)->aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + + (*pIter)->pBlockLoadInfo = pBlockLoadInfo; + if (taosArrayGetSize(pBlockLoadInfo->aSttBlk) == 0) { + code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); + if (code) { + goto _exit; + } else { + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + SArray* pTmp = taosArrayInit(size, sizeof(SSttBlk)); + for(int32_t i = 0; i < size; ++i) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + if (p->suid == suid) { + taosArrayPush(pTmp, p); + } + } + + taosArrayDestroy(pBlockLoadInfo->aSttBlk); + pBlockLoadInfo->aSttBlk = pTmp; + } } - code = tBlockDataCreate(&(*pIter)->bData[0]); - if (code) { - goto _exit; - } - - code = tBlockDataCreate(&(*pIter)->bData[1]); - if (code) { - goto _exit; - } - - code = tsdbReadSttBlk(pReader, iStt, (*pIter)->aSttBlk); - if (code) { - goto _exit; - } - - size_t size = taosArrayGetSize((*pIter)->aSttBlk); + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); // find the start block - int32_t index = -1; - if (!backward) { // asc - for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i); - if (p->suid != suid) { - continue; - } - - if (p->minUid <= uid && p->maxUid >= uid) { - index = i; - break; - } - } - } else { // desc - for (int32_t i = size - 1; i >= 0; --i) { - SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i); - if (p->suid != suid) { - continue; - } - - if (p->minUid <= uid && p->maxUid >= uid) { - index = i; - break; - } - } - } - - (*pIter)->iSttBlk = index; - if (index != -1) { - (*pIter)->pSttBlk = taosArrayGet((*pIter)->aSttBlk, (*pIter)->iSttBlk); + (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); + if ((*pIter)->iSttBlk != -1) { + (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk); + (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; } _exit: @@ -116,9 +271,6 @@ _exit: } void tLDataIterClose(SLDataIter *pIter) { - tBlockDataDestroy(&pIter->bData[0], 1); - tBlockDataDestroy(&pIter->bData[1], 1); - taosArrayDestroy(pIter->aSttBlk); taosMemoryFree(pIter); } @@ -127,9 +279,9 @@ void tLDataIterNextBlock(SLDataIter *pIter) { pIter->iSttBlk += step; int32_t index = -1; - size_t size = taosArrayGetSize(pIter->aSttBlk); + size_t size = pIter->pBlockLoadInfo->aSttBlk->size; for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) { - SSttBlk *p = taosArrayGet(pIter->aSttBlk, i); + SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, i); if ((!pIter->backward) && p->minUid > pIter->uid) { break; } @@ -166,32 +318,41 @@ void tLDataIterNextBlock(SLDataIter *pIter) { } } - if (index == -1) { - pIter->pSttBlk = NULL; - } else { - pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); + pIter->pSttBlk = NULL; + if (index != -1) { + pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); } } -static void findNextValidRow(SLDataIter *pIter) { +static void findNextValidRow(SLDataIter *pIter, const char* idStr) { int32_t step = pIter->backward ? -1 : 1; bool hasVal = false; int32_t i = pIter->iRow; - SBlockData *pBlockData = getCurrentBlock(pIter); + + SBlockData *pBlockData = loadLastBlock(pIter, idStr); + + // mostly we only need to find the start position for a given table + if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && pBlockData->aUid != NULL) { + i = binarySearchForStartRowIndex((uint64_t*)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward); + if (i == -1) { + pIter->iRow = -1; + return; + } + } for (; i < pBlockData->nRow && i >= 0; i += step) { if (pBlockData->aUid != NULL) { if (!pIter->backward) { - if (pBlockData->aUid[i] < pIter->uid) { + /*if (pBlockData->aUid[i] < pIter->uid) { continue; - } else if (pBlockData->aUid[i] > pIter->uid) { + } else */if (pBlockData->aUid[i] > pIter->uid) { break; } } else { - if (pBlockData->aUid[i] > pIter->uid) { + /*if (pBlockData->aUid[i] > pIter->uid) { continue; - } else if (pBlockData->aUid[i] < pIter->uid) { + } else */if (pBlockData->aUid[i] < pIter->uid) { break; } } @@ -229,7 +390,7 @@ static void findNextValidRow(SLDataIter *pIter) { pIter->iRow = (hasVal) ? i : -1; } -bool tLDataIterNextRow(SLDataIter *pIter) { +bool tLDataIterNextRow(SLDataIter *pIter, const char* idStr) { int32_t code = 0; int32_t step = pIter->backward ? -1 : 1; @@ -238,23 +399,12 @@ bool tLDataIterNextRow(SLDataIter *pIter) { return false; } - int32_t iBlockL = pIter->iSttBlk; - SBlockData *pBlockData = getCurrentBlock(pIter); - - if (pBlockData->nRow == 0 && pIter->pSttBlk != NULL) { // current block not loaded yet - pBlockData = getNextBlock(pIter); - code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - - pIter->iRow = (pIter->backward) ? pBlockData->nRow : -1; - } - + int32_t iBlockL = pIter->iSttBlk; + SBlockData *pBlockData = loadLastBlock(pIter, idStr); pIter->iRow += step; while (1) { - findNextValidRow(pIter); + findNextValidRow(pIter, idStr); if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter); @@ -266,12 +416,8 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } if (iBlockL != pIter->iSttBlk) { - pBlockData = getNextBlock(pIter); - code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData); - if (code) { - goto _exit; - } - pIter->iRow = pIter->backward ? (pBlockData->nRow - 1) : 0; + pBlockData = loadLastBlock(pIter, idStr); + pIter->iRow += step; } } @@ -313,7 +459,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange) { + STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, const char* idStr) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -321,22 +467,36 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead return TSDB_CODE_OUT_OF_MEMORY; } - tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + pMTree->idStr = idStr; + + tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); + int32_t code = TSDB_CODE_SUCCESS; + + SSttBlockLoadInfo* pLoadInfo = NULL; + if (pBlockLoadInfo == NULL) { + if (pMTree->pLoadInfo == NULL) { + pMTree->destroyLoadInfo = true; + pMTree->pLoadInfo = tCreateLastBlockLoadInfo(); + } + + pLoadInfo = pMTree->pLoadInfo; + } else { + pLoadInfo = pBlockLoadInfo; + } - struct SLDataIter *pIterList[TSDB_DEFAULT_STT_FILE] = {0}; for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file - code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange); + struct SLDataIter* pIter = NULL; + code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pLoadInfo[i]); if (code != TSDB_CODE_SUCCESS) { goto _end; } - bool hasVal = tLDataIterNextRow(pIterList[i]); + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); if (hasVal) { - taosArrayPush(pMTree->pIterList, &pIterList[i]); - tMergeTreeAddIter(pMTree, pIterList[i]); + taosArrayPush(pMTree->pIterList, &pIter); + tMergeTreeAddIter(pMTree, pIter); } else { - tLDataIterClose(pIterList[i]); + tLDataIterClose(pIter); } } @@ -354,7 +514,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { if (pMTree->pIter) { SLDataIter *pIter = pMTree->pIter; - bool hasVal = tLDataIterNextRow(pIter); + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); if (!hasVal) { pMTree->pIter = NULL; } @@ -393,4 +553,9 @@ void tMergeTreeClose(SMergeTree *pMTree) { pMTree->pIterList = taosArrayDestroy(pMTree->pIterList); pMTree->pIter = NULL; + + if (pMTree->destroyLoadInfo) { + pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo); + pMTree->destroyLoadInfo = false; + } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8a51fc73a6..997f903045 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -17,8 +17,6 @@ #include "tsdb.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define ALL_ROWS_CHECKED_INDEX (INT16_MIN) -#define INITIAL_ROW_INDEX_VAL (-1) typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -72,6 +70,8 @@ typedef struct SIOCostSummary { double smaLoadTime; int64_t lastBlockLoad; double lastBlockLoadTime; + int64_t composedBlocks; + double buildComposedBlockTime; } SIOCostSummary; typedef struct SBlockLoadSuppInfo { @@ -88,6 +88,7 @@ typedef struct SLastBlockReader { int32_t order; uint64_t uid; SMergeTree mergeTree; + SSttBlockLoadInfo* pInfo; } SLastBlockReader; typedef struct SFilesetIter { @@ -226,13 +227,14 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK return NULL; } - int32_t step = ASCENDING_TRAVERSE(pTsdbReader->order)? 1:-1; for (int32_t j = 0; j < numOfTables; ++j) { STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { - info.lastKey = pTsdbReader->window.skey - step; + int64_t skey = pTsdbReader->window.skey; + info.lastKey = (skey > INT64_MIN)? (skey - 1):skey; } else { - info.lastKey = pTsdbReader->window.ekey - step; + int64_t ekey = pTsdbReader->window.ekey; + info.lastKey = (ekey < INT64_MAX)? (ekey + 1):ekey; } taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); @@ -319,8 +321,7 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap } // init file iterator -static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, - STsdbReader* pReader /*int32_t order, const char* idstr*/) { +static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader) { size_t numOfFileset = taosArrayGetSize(aDFileSet); pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset; @@ -345,6 +346,14 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, pLReader->uid = 0; tMergeTreeClose(&pLReader->mergeTree); + if (pLReader->pInfo == NULL) { + pLReader->pInfo = tCreateLastBlockLoadInfo(); + if (pLReader->pInfo == NULL) { + tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); + return terrno; + } + } + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -358,8 +367,12 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { return false; } + SIOCostSummary* pSum = &pReader->cost; + getLastBlockLoadInfo(pIter->pLastBlockReader->pInfo, &pSum->lastBlockLoad, &pReader->cost.lastBlockLoadTime); + pIter->pLastBlockReader->uid = 0; tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); + resetLastBlockLoadInfo(pIter->pLastBlockReader->pInfo); // check file the time range of coverage STimeWindow win = {0}; @@ -1230,6 +1243,38 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return false; } +static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { + while (1) { + bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + if (!hasVal) { + return false; + } + + TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBKEY k = TSDBROW_KEY(&row); + if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { + return true; + } + } +} + +static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) { + bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo); + if (hasVal) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 != ts) { + doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + return true; + } + } else { + doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + return true; + } + + return false; +} + static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { // always set the newest schema version in pReader->pSchema if (pReader->pSchema == NULL) { @@ -1377,30 +1422,54 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - // SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; + TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + // only last block exists + if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { + if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); - // merge with block data if ts == key - if (mergeBlockData && (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tRowMerge(&merge, &fRow1); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + } + } else { // not merge block data + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + ASSERT(mergeBlockData); + + // merge with block data if ts == key + if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); - - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1851,57 +1920,40 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { - while (1) { - bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); - if (!hasVal) { - return false; - } - - TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = TSDBROW_KEY(&row); - if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { - return true; - } - } -} - -static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader) { +static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { // the last block reader has been initialized for this table. - if (pLastBlockReader->uid == pBlockScanInfo->uid) { + if (pLBlockReader->uid == pScanInfo->uid) { return true; } - if (pLastBlockReader->uid != 0) { - tMergeTreeClose(&pLastBlockReader->mergeTree); + if (pLBlockReader->uid != 0) { + tMergeTreeClose(&pLBlockReader->mergeTree); } - initMemDataIterator(pBlockScanInfo, pReader); - pLastBlockReader->uid = pBlockScanInfo->uid; + initMemDataIterator(pScanInfo, pReader); + pLBlockReader->uid = pScanInfo->uid; - int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order)? 1:-1; - STimeWindow w = pLastBlockReader->window; - if (ASCENDING_TRAVERSE(pLastBlockReader->order)) { - w.skey = pBlockScanInfo->lastKey + step; + int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order)? 1:-1; + STimeWindow w = pLBlockReader->window; + if (ASCENDING_TRAVERSE(pLBlockReader->order)) { + w.skey = pScanInfo->lastKey + step; } else { - w.ekey = pBlockScanInfo->lastKey + step; + w.ekey = pScanInfo->lastKey + step; } int32_t code = - tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, - pReader->suid, pBlockScanInfo->uid, &w, &pLastBlockReader->verRange); + tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, + pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { return false; } - return nextRowFromLastBlocks(pLastBlockReader, pBlockScanInfo); + return nextRowFromLastBlocks(pLBlockReader, pScanInfo); } static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY key = TSDBROW_KEY(&row); - return key.ts; + return TSDBROW_TS(&row); } static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } @@ -1931,8 +1983,6 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc tRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } - - return TSDB_CODE_SUCCESS; } static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, @@ -2025,13 +2075,16 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { blockDataUpdateTsWindow(pResBlock, 0); setComposedBlockFlag(pReader, true); - int64_t et = taosGetTimestampUs(); + double el = (taosGetTimestampUs() - st)/1000.0; + + pReader->cost.composedBlocks += 1; + pReader->cost.buildComposedBlockTime += el; if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, - pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); + pResBlock->info.rows, el, pReader->idStr); } return TSDB_CODE_SUCCESS; @@ -3074,11 +3127,16 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pCol->info.colId) { + j += 1; + continue; + } + if (pData->cid == pCol->info.colId) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; - } else { // the specified column does not exist in file block, fill with null data + } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data colDataAppendNULL(pCol, outputRowIndex); } @@ -3200,11 +3258,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl } } + // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. if (pCond->suid != 0) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, pCond->endVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, /*pCond->endVersion*/ -1); + if (pReader->pSchema == NULL) { + tsdbError("failed to get table schema, suid:%"PRIu64", ver:%"PRId64" , %s", pReader->suid, -1, pReader->idStr); + } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, pCond->endVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, /*pCond->endVersion*/ -1); + if (pReader->pSchema == NULL) { + tsdbError("failed to get table schema, uid:%"PRIu64", ver:%"PRId64" , %s", pKey->uid, -1, pReader->idStr); + } } int32_t numOfTables = taosArrayGetSize(pTableList); @@ -3301,23 +3366,27 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); + SIOCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pLastBlockReader != NULL) { - tMergeTreeClose(&pFilesetIter->pLastBlockReader->mergeTree); - taosMemoryFree(pFilesetIter->pLastBlockReader); + SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader; + tMergeTreeClose(&pLReader->mergeTree); + + getLastBlockLoadInfo(pLReader->pInfo, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime); + + pLReader->pInfo = destroyLastBlockLoadInfo(pLReader->pInfo); + taosMemoryFree(pLReader); } - SIOCostSummary* pCost = &pReader->cost; - - tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, fileBlocks:%" PRId64 - ", fileBlocks-time:%.2f ms, " - "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 - ", lastBlocks-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", - pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, - pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, - pCost->lastBlockLoadTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); + tsdbDebug( + "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 + " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, + pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, + pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 0722c2b306..a26f1c74f8 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -126,6 +126,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type); +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a9826e0018..fc8f942015 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -411,7 +411,7 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, - STREAM_SCAN_FROM_DELETERES, + STREAM_SCAN_FROM_DELETE_DATA, STREAM_SCAN_FROM_DATAREADER_RETRIEVE, STREAM_SCAN_FROM_DATAREADER_RANGE, } EStreamScanMode; @@ -794,6 +794,7 @@ typedef struct SStreamPartitionOperatorInfo { void* parIte; SSDataBlock* pInputDataBlock; int32_t tsColIndex; + SSDataBlock* pDelRes; } SStreamPartitionOperatorInfo; typedef struct STimeSliceOperatorInfo { @@ -1108,6 +1109,13 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol bool groupbyTbname(SNodeList* pGroupList); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo); +int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo); +int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); +int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 3965b7e5b2..0c1728386b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -475,7 +475,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray if (code != TSDB_CODE_SUCCESS) { terrno = code; qError("failed to create result, reason:%s", tstrerror(code)); - terrno = code; goto end; } @@ -1012,6 +1011,100 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa return pCol; } +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) { + pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); + pExp->pExpr->_function.num = 1; + pExp->pExpr->_function.functionId = -1; + + int32_t type = nodeType(pTargetNode->pExpr); + // it is a project query, or group by column + if (type == QUERY_NODE_COLUMN) { + pExp->pExpr->nodeType = QUERY_NODE_COLUMN; + SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pColNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pColNode->colName); + pExp->base.pParam[0].pCol = + createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; + } else if (type == QUERY_NODE_VALUE) { + pExp->pExpr->nodeType = QUERY_NODE_VALUE; + SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pValNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pValNode->node.aliasName); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; + nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param); + } else if (type == QUERY_NODE_FUNCTION) { + pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; + SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; + + SDataType* pType = &pFuncNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pFuncNode->node.aliasName); + + pExp->pExpr->_function.functionId = pFuncNode->funcId; + pExp->pExpr->_function.pFunctNode = pFuncNode; + + strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, + tListLen(pExp->pExpr->_function.functionName)); +#if 1 + // todo refactor: add the parameter for tbname function + if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + pFuncNode->pParameterList = nodesMakeList(); + ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { // todo handle error + } else { + res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + nodesListAppend(pFuncNode->pParameterList, (SNode*)res); + } + } +#endif + + int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); + + pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam)); + pExp->base.numOfParams = numOfParam; + + for (int32_t j = 0; j < numOfParam; ++j) { + SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); + if (p1->type == QUERY_NODE_COLUMN) { + SColumnNode* pcn = (SColumnNode*)p1; + + pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; + pExp->base.pParam[j].pCol = + createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType); + } else if (p1->type == QUERY_NODE_VALUE) { + SValueNode* pvn = (SValueNode*)p1; + pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; + nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param); + } + } + } else if (type == QUERY_NODE_OPERATOR) { + pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; + SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pNode->node.aliasName); + pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; + } else { + ASSERT(0); + } +} + SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); int32_t numOfGroupKeys = 0; @@ -1035,98 +1128,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } SExprInfo* pExp = &pExprs[i]; - - pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); - pExp->pExpr->_function.num = 1; - pExp->pExpr->_function.functionId = -1; - - int32_t type = nodeType(pTargetNode->pExpr); - // it is a project query, or group by column - if (type == QUERY_NODE_COLUMN) { - pExp->pExpr->nodeType = QUERY_NODE_COLUMN; - SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pColNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pColNode->colName); - pExp->base.pParam[0].pCol = - createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType); - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; - } else if (type == QUERY_NODE_VALUE) { - pExp->pExpr->nodeType = QUERY_NODE_VALUE; - SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pValNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pValNode->node.aliasName); - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; - nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param); - } else if (type == QUERY_NODE_FUNCTION) { - pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; - SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; - - SDataType* pType = &pFuncNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pFuncNode->node.aliasName); - - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; - - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); -#if 1 - // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { - pFuncNode->pParameterList = nodesMakeList(); - ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); - SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); - if (NULL == res) { // todo handle error - } else { - res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; - nodesListAppend(pFuncNode->pParameterList, (SNode*)res); - } - } -#endif - - int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); - - pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam)); - pExp->base.numOfParams = numOfParam; - - for (int32_t j = 0; j < numOfParam; ++j) { - SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); - if (p1->type == QUERY_NODE_COLUMN) { - SColumnNode* pcn = (SColumnNode*)p1; - - pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; - pExp->base.pParam[j].pCol = - createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType); - } else if (p1->type == QUERY_NODE_VALUE) { - SValueNode* pvn = (SValueNode*)p1; - pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; - nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param); - } - } - } else if (type == QUERY_NODE_OPERATOR) { - pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; - SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pNode->node.aliasName); - pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; - } else { - ASSERT(0); - } + createExprFromTargetNode(pExp, pTargetNode); } return pExprs; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 205bcd58df..815d6a71ef 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3439,6 +3439,44 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } } +static bool isWstartColumnExist(SFillOperatorInfo* pInfo) { + if (pInfo->numOfNotFillExpr == 0) { + return false; + } + for (int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) { + SExprInfo* exprInfo = pInfo->pNotFillExprInfo + i; + if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN && + exprInfo->base.numOfParams == 1 && + exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) { + return true; + } + } + return false; +} + +static int32_t createWStartTsAsNotFillExpr(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode) { + bool wstartExist = isWstartColumnExist(pInfo); + if (wstartExist == false) { + if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) { + qError("pWStartTs of fill physical node is not a target node"); + return TSDB_CODE_QRY_SYS_ERROR; + } + + SExprInfo* notFillExprs = taosMemoryRealloc(pInfo->pNotFillExprInfo, (pInfo->numOfNotFillExpr + 1) * sizeof(SExprInfo)); + if (notFillExprs == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + createExprFromTargetNode(notFillExprs + pInfo->numOfNotFillExpr, (STargetNode*)pPhyFillNode->pWStartTs); + + ++pInfo->numOfNotFillExpr; + pInfo->pNotFillExprInfo = notFillExprs; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); @@ -3450,7 +3488,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr); - + int32_t code = createWStartTsAsNotFillExpr(pInfo, pPhyFillNode); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval @@ -3471,7 +3512,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); - int32_t code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, + code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type, order); if (code != TSDB_CODE_SUCCESS) { @@ -3938,7 +3979,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode, pTaskInfo, isStream); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; @@ -4410,3 +4451,108 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF return code; } + +int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo) { + SWinKey key = { + .ts = win->skey, + .groupId = tableGroupId, + }; + char* value = NULL; + int32_t size = pAggSup->resultRowSize; + if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + *pResult = (SResultRow*)value; + ASSERT(*pResult); + // set time window for current result + (*pResult)->win = (*win); + setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); + return TSDB_CODE_SUCCESS; +} + +int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) { + streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult); + return TSDB_CODE_SUCCESS; +} + +int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { + streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize); + return TSDB_CODE_SUCCESS; +} + +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { + SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); + int32_t size = 0; + void* pVal = NULL; + SWinKey key = { + .ts = *(TSKEY*)pPos->key, + .groupId = pPos->groupId, + }; + int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size); + ASSERT(code == 0); + SResultRow* pRow = (SResultRow*)pVal; + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); + // no results, continue to check the next one + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + releaseOutputBuf(pTaskInfo, &key, pRow); + continue; + } + + if (pBlock->info.groupId == 0) { + pBlock->info.groupId = pPos->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.groupId != pPos->groupId) { + releaseOutputBuf(pTaskInfo, &key, pRow); + break; + } + } + + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + releaseOutputBuf(pTaskInfo, &key, pRow); + break; + } + + pGroupResInfo->index += 1; + + for (int32_t j = 0; j < numOfExprs; ++j) { + int32_t slotId = pExprInfo[j].base.resSchema.slotId; + + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.finalize) { + int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, code); + } + } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { + // do nothing, todo refactor + } else { + // expand the result into multiple rows. E.g., _wstart, top(k, 20) + // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } + } + } + releaseOutputBuf(pTaskInfo, &key, pRow); + pBlock->info.rows += pRow->numOfRows; + } + blockDataUpdateTsWindow(pBlock, 0); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 599edb0722..5eb6557dbd 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -13,26 +13,26 @@ * along with this program. If not, see . */ -#include "os.h" #include "function.h" +#include "os.h" #include "tname.h" #include "tdatablock.h" #include "tmsg.h" +#include "executorInt.h" #include "executorimpl.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "executorInt.h" static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); -static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); +static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, + int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); static void freeGroupKey(void* param) { - SGroupKeys* pKey = (SGroupKeys*) param; + SGroupKeys* pKey = (SGroupKeys*)param; taosMemoryFree(pKey->pData); } @@ -62,13 +62,13 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupColList, i); - (*keyLen) += pCol->bytes; // actual data + null_flag + (*keyLen) += pCol->bytes; // actual data + null_flag SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; + key.bytes = pCol->bytes; + key.type = pCol->type; key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); + key.pData = taosMemoryCalloc(1, pCol->bytes); if (key.pData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -87,7 +87,8 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** return TSDB_CODE_SUCCESS; } -static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { +static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, + int32_t numOfGroupCols) { SColumnDataAgg* pColAgg = NULL; for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupCols, i); @@ -112,7 +113,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo if (pkey->type == TSDB_DATA_TYPE_JSON) { int32_t dataLen = getJsonValueLen(val); - if (memcmp(pkey->pData, val, dataLen) == 0){ + if (memcmp(pkey->pData, val, dataLen) == 0) { continue; } else { return false; @@ -154,7 +155,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); if (pkey->type == TSDB_DATA_TYPE_JSON) { - if(tTagIsJson(val)){ + if (tTagIsJson(val)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; return; } @@ -198,13 +199,13 @@ static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { } } - return (int32_t) (pStart - (char*)pKey); + return (int32_t)(pStart - (char*)pKey); } // assign the group keys or user input constant values if required static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) { for (int32_t i = 0; i < numOfOutput; ++i) { - if (pCtx[i].functionId == -1) { // select count(*),key from t group by key. + if (pCtx[i].functionId == -1) { // select count(*),key from t group by key. SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]); SColumnInfoData* pColInfoData = pCtx[i].input.pData[0]; @@ -221,7 +222,7 @@ static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t } else { memcpy(dest, data, pColInfoData->info.bytes); } - } else { // it is a NULL value + } else { // it is a NULL value pEntryInfo->isNullRes = 1; } @@ -275,7 +276,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, + len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -291,9 +293,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (num > 0) { len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = - setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, - pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, + len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -308,7 +309,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; SSDataBlock* pRes = pInfo->binfo.pRes; - while(1) { + while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes, NULL); @@ -323,7 +324,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { } pOperator->resultInfo.totalRows += pRes->info.rows; - return (pRes->info.rows == 0)? NULL:pRes; + return (pRes->info.rows == 0) ? NULL : pRes; } static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { @@ -334,7 +335,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SGroupbyOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { return buildGroupResultDataBlock(pOperator); @@ -343,7 +344,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -362,7 +363,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -403,8 +405,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx goto _error; } - pInfo->pGroupCols = pGroupColList; - pInfo->pCondition = pCondition; + pInfo->pGroupCols = pGroupColList; + pInfo->pCondition = pCondition; int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); if (code != TSDB_CODE_SUCCESS) { @@ -425,14 +427,15 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx initBasicInfo(&pInfo->binfo, pResultBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pOperator->name = "GroupbyAggOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "GroupbyAggOperator"; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, + destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -440,7 +443,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; destroyGroupOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); @@ -448,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx } static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { -// SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SPartitionOperatorInfo* pInfo = pOperator->info; @@ -457,7 +460,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); SDataGroupInfo* pGroupInfo = NULL; - void *pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); + void* pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); pGroupInfo->numOfRows += 1; @@ -467,32 +470,32 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } // number of rows - int32_t* rows = (int32_t*) pPage; + int32_t* rows = (int32_t*)pPage; size_t numOfCols = pOperator->exprSupp.numOfExprs; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; - int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); int32_t bytes = pColInfoData->info.bytes; int32_t startOffset = pInfo->columnOffset[i]; - int32_t* columnLen = NULL; + int32_t* columnLen = NULL; int32_t contentLen = 0; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { int32_t* offset = (int32_t*)((char*)pPage + startOffset); - columnLen = (int32_t*) ((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); - char* data = (char*)((char*) columnLen + sizeof(int32_t)); + columnLen = (int32_t*)((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); + char* data = (char*)((char*)columnLen + sizeof(int32_t)); if (colDataIsNull_s(pColInfoData, j)) { offset[(*rows)] = -1; contentLen = 0; - } else if(pColInfoData->info.type == TSDB_DATA_TYPE_JSON){ + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { offset[*rows] = (*columnLen); - char* src = colDataGetData(pColInfoData, j); + char* src = colDataGetData(pColInfoData, j); int32_t dataLen = getJsonValueLen(src); memcpy(data + (*columnLen), src, dataLen); @@ -511,8 +514,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } else { char* bitmap = (char*)pPage + startOffset; - columnLen = (int32_t*) ((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); - char* data = (char*) columnLen + sizeof(int32_t); + columnLen = (int32_t*)((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); + char* data = (char*)columnLen + sizeof(int32_t); bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); if (isNull) { @@ -539,7 +542,7 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf SDataGroupInfo* p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len); void* pPage = NULL; - if (p == NULL) { // it is a new group + if (p == NULL) { // it is a new group SDataGroupInfo gi = {0}; gi.pPageList = taosArrayInit(100, sizeof(int32_t)); taosHashPut(pInfo->pGroupSet, pInfo->keyBuf, len, &gi, sizeof(SDataGroupInfo)); @@ -550,12 +553,12 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf pPage = getNewBufPage(pInfo->pBuf, &pageId); taosArrayPush(p->pPageList, &pageId); - *(int32_t *) pPage = 0; + *(int32_t*)pPage = 0; } else { int32_t* curId = taosArrayGetLast(p->pPageList); pPage = getBufPage(pInfo->pBuf, *curId); - int32_t *rows = (int32_t*) pPage; + int32_t* rows = (int32_t*)pPage; if (*rows >= pInfo->rowCapacity) { // release buffer releaseBufPage(pInfo->pBuf, pPage); @@ -585,17 +588,18 @@ uint64_t calcGroupId(char* pData, int32_t len) { } int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); - offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format + offset[0] = sizeof(int32_t) + + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format - for(int32_t i = 0; i < numOfCols - 1; ++i) { + for (int32_t i = 0; i < numOfCols - 1; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); int32_t bytes = pColInfoData->info.bytes; int32_t payloadLen = bytes * rowCapacity; - + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { // offset segment + content length + payload offset[i + 1] = rowCapacity * sizeof(int32_t) + sizeof(int32_t) + payloadLen + offset[i]; @@ -609,9 +613,9 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { } static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { - void *ite = NULL; - while( (ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL ) { - taosArrayDestroy( ((SDataGroupInfo *)ite)->pPageList); + void* ite = NULL; + while ((ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL) { + taosArrayDestroy(((SDataGroupInfo*)ite)->pPageList); } taosArrayClear(pInfo->sortedGroupArray); clearDiskbasedBuf(pInfo->pBuf); @@ -626,13 +630,14 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { return 0; } - return (pGroupInfo1->groupId < pGroupInfo2->groupId)? -1:1; + return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1; } static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; - SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; + SDataGroupInfo* pGroupInfo = + (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { // try next group data ++pInfo->groupIndex; @@ -647,7 +652,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { } int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); - void* page = getBufPage(pInfo->pBuf, *pageId); + void* page = getBufPage(pInfo->pBuf, *pageId); blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); @@ -670,14 +675,14 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SPartitionOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { blockDataCleanup(pRes); return buildPartitionResult(pOperator); } - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -688,7 +693,8 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -727,7 +733,7 @@ static void destroyPartitionOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); taosArrayDestroy(pInfo->pGroupCols); - for(int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++){ + for (int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++) { SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->pGroupColVals, i); taosMemoryFree(key.pData); } @@ -743,24 +749,25 @@ static void destroyPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo) { SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); if (pPartNode->pExprs != NULL) { - int32_t num = 0; + int32_t num = 0; SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -772,7 +779,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - uint32_t defaultPgsz = 0; + uint32_t defaultPgsz = 0; uint32_t defaultBufsz = 0; getBufferPgSize(pResBlock->info.rowSize, &defaultPgsz, &defaultBufsz); @@ -794,15 +801,15 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pOperator->name = "PartitionOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "PartitionOperator"; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pInfo->binfo.pRes = pResBlock; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pInfo->binfo.pRes = pResBlock; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL, NULL, NULL); @@ -810,16 +817,16 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; } -int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, + int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; @@ -833,37 +840,36 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId) { if (pExprSup->pExprInfo != NULL) { - int32_t code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + int32_t code = + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { qError("calaculate group id error, code:%d", code); } } recordNewGroupKeys(pParSup->pGroupCols, pParSup->pGroupColVals, pBlock, rowId); - int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals); + int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals); uint64_t groupId = calcGroupId(pParSup->keyBuf, len); return groupId; } -static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { - return pInfo->parIte != NULL; -} +static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo->parIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { SStreamPartitionOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pDest = pInfo->binfo.pRes; + SSDataBlock* pDest = pInfo->binfo.pRes; ASSERT(hasRemainPartion(pInfo)); SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte; blockDataCleanup(pDest); - int32_t rows = taosArrayGetSize(pParInfo->rowIds); + int32_t rows = taosArrayGetSize(pParInfo->rowIds); SSDataBlock* pSrc = pInfo->pInputDataBlock; for (int32_t i = 0; i < rows; i++) { int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i); for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; j++) { - int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId; + int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId; SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, slotId); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j); - bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL); - char* pSrcData = colDataGetData(pSrcCol, rowIndex); + bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL); + char* pSrcData = colDataGetData(pSrcCol, rowIndex); colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull); } pDest->info.rows++; @@ -881,9 +887,9 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat pInfo->pInputDataBlock = pBlock; for (int32_t i = 0; i < pBlock->info.rows; ++i) { recordNewGroupKeys(pInfo->partitionSup.pGroupCols, pInfo->partitionSup.pGroupColVals, pBlock, i); - int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals); - SPartitionDataInfo* pParData = - (SPartitionDataInfo*) taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen); + int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals); + SPartitionDataInfo* pParData = + (SPartitionDataInfo*)taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen); if (pParData) { taosArrayPush(pParData->rowIds, &i); } else { @@ -891,8 +897,7 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat newParData.groupId = calcGroupId(pInfo->partitionSup.keyBuf, keyLen); newParData.rowIds = taosArrayInit(64, sizeof(int32_t)); taosArrayPush(newParData.rowIds, &i); - taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, - sizeof(SPartitionDataInfo)); + taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, sizeof(SPartitionDataInfo)); } } } @@ -902,13 +907,13 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamPartitionOperatorInfo* pInfo = pOperator->info; if (hasRemainPartion(pInfo)) { return buildStreamPartitionResult(pOperator); } - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; { pInfo->pInputDataBlock = NULL; @@ -924,14 +929,19 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { case STREAM_INVALID: pInfo->binfo.pRes->info.type = pBlock->info.type; break; + case STREAM_DELETE_DATA: { + copyDataBlock(pInfo->pDelRes, pBlock); + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; + return pInfo->pDelRes; + } break; default: return pBlock; } // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, - pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, pTaskInfo->code); } @@ -940,7 +950,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { doStreamHashPartitionImpl(pInfo, pBlock); } pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - + pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); return buildStreamPartitionResult(pOperator); } @@ -950,7 +960,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); taosArrayDestroy(pInfo->partitionSup.pGroupCols); - for(int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++){ + for (int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++) { SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->partitionSup.pGroupColVals, i); taosMemoryFree(key.pData); } @@ -958,6 +968,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFree(pInfo->partitionSup.keyBuf); cleanupExprSupp(&pInfo->scalarSup); + blockDataDestroy(pInfo->pDelRes); taosMemoryFreeClear(param); } @@ -970,7 +981,8 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup pScanInfo->pPartScalarSup = pExpr; } -SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo) { SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -980,7 +992,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); if (pPartNode->pExprs != NULL) { - int32_t num = 0; + int32_t num = 0; SExprInfo* pCalExprInfo = createExprInfo(pPartNode->pExprs, NULL, &num); code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num); if (code != TSDB_CODE_SUCCESS) { @@ -989,7 +1001,8 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } int32_t keyLen = 0; - code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); + code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, + pInfo->partitionSup.pGroupCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1000,35 +1013,35 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr goto _error; } blockDataEnsureCapacity(pResBlock, 4096); - pInfo->binfo.pRes = pResBlock; - pInfo->parIte = NULL; - pInfo->pInputDataBlock = NULL; + pInfo->binfo.pRes = pResBlock; + pInfo->parIte = NULL; + pInfo->pInputDataBlock = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - pInfo->tsColIndex = 0; + pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + pInfo->tsColIndex = 0; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); - pOperator->name = "StreamPartitionOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "StreamPartitionOperator"; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, destroyStreamPartitionOperatorInfo, - NULL, NULL, NULL); + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, + destroyStreamPartitionOperatorInfo, NULL, NULL, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFreeClear(pInfo); + destroyStreamPartitionOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); return NULL; } - diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c099c1c24c..ad9cd1ffe7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1057,24 +1057,24 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ return true; } -static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, - int32_t* pRowIndex, bool hasGroup) { +static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, SInterval* pInterval, + SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; while (1) { if (hasGroup) { (*pRowIndex) += 1; } else { - (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); + (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, startTsCol, *pRowIndex, endWin.ekey, binarySearchForKey, + NULL, TSDB_ORDER_ASC); } do { preWin = endWin; getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC); - } while (tsCol[(*pRowIndex) - 1] >= endWin.skey); + } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey); endWin = preWin; if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows) { win.ekey = endWin.ekey; @@ -1102,6 +1102,11 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 return NULL; } + doFilter(pInfo->pCondition, pResult, NULL); + if (pResult->info.rows == 0) { + continue; + } + if (pInfo->partitionSup.needCalc) { SSDataBlock* tmpBlock = createOneDataBlock(pResult, true); blockDataCleanup(pResult); @@ -1188,13 +1193,15 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS return code; } - SColumnInfoData* pSrcTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData; - ASSERT(pSrcTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* tsCol = (TSKEY*)pSrcTsCol->pData; + ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); @@ -1204,12 +1211,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < rows;) { uint64_t srcUid = srcUidData[i]; - uint64_t groupId = getGroupIdByData(pInfo, srcUid, tsCol[i], version); + uint64_t groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); uint64_t srcGpId = srcGp[i]; - TSKEY calStartTs = tsCol[i]; + TSKEY calStartTs = srcStartTsCol[i]; colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); - STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc); - TSKEY calEndTs = tsCol[i - 1]; + STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, &pInfo->interval, &pSrcBlock->info, &i, + pInfo->partitionSup.needCalc); + TSKEY calEndTs = srcStartTsCol[i - 1]; colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); @@ -1229,11 +1237,49 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS return TSDB_CODE_SUCCESS; } +static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + if (pSrcBlock->info.rows == 0) { + return TSDB_CODE_SUCCESS; + } + blockDataCleanup(pDestBlock); + int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3); + SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startData = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endData = (TSKEY*)pEndTsCol->pData; + SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pUidCol->pData; + + SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + int32_t dummy = 0; + int64_t version = pSrcBlock->info.version - 1; + for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { + uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); + colDataAppend(pDestStartCol, i, (const char*)(startData + i), false); + colDataAppend(pDestEndCol, i, (const char*)(endData + i), false); + colDataAppendNULL(pDestUidCol, i); + colDataAppend(pDestGpCol, i, (const char*)&groupId, false); + colDataAppendNULL(pDestCalStartTsCol, i); + colDataAppendNULL(pDestCalEndTsCol, i); + pDestBlock->info.rows++; + } + return TSDB_CODE_SUCCESS; +} + static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { int32_t code = TSDB_CODE_SUCCESS; if (isIntervalWindow(pInfo)) { code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock); - } else { + } else if (isSessionWindow(pInfo) || isStateWindow(pInfo)) { code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock); } pDestBlock->info.type = STREAM_CLEAR; @@ -1510,14 +1556,23 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } break; case STREAM_DELETE_DATA: { - pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; - pInfo->updateResIndex = 0; - generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); - pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - return pInfo->pDeleteDataRes; + printDataBlock(pBlock, "stream scan delete recv"); + if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { + generateDeleteResultBlock(pInfo, pBlock, pInfo->pDeleteDataRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; + printDataBlock(pBlock, "stream scan delete result"); + return pInfo->pDeleteDataRes; + } else { + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; + pInfo->updateResIndex = 0; + generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + printDataBlock(pBlock, "stream scan delete data"); + return pInfo->pDeleteDataRes; + } } break; default: break; @@ -1532,7 +1587,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; return pInfo->pRes; } break; - case STREAM_SCAN_FROM_DELETERES: { + case STREAM_SCAN_FROM_DELETE_DATA: { generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; @@ -1646,7 +1701,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_RES; return pInfo->pUpdateDataRes; } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { - pInfo->scanMode = STREAM_SCAN_FROM_DELETERES; + pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; } } } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d773f8a629..b3a85807ae 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -955,8 +955,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder); - int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -983,7 +983,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); + numOfOutput); doCloseWindow(pResultRowInfo, pInfo, pResult); @@ -1406,20 +1406,25 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pUpdatedMap) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* tsEnds = (TSKEY*)pEndCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIds = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC); - doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); - SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]}; - if (pDelWins) { - taosArrayPush(pDelWins, &winRes); - } - if (pUpdatedMap) { - taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); - } + do { + doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); + SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]}; + if (pDelWins) { + taosArrayPush(pDelWins, &winRes); + } + if (pUpdatedMap) { + taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + } + getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + } while (win.skey <= tsEnds[i]); } } @@ -2775,7 +2780,7 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); } - if (num > 1 && pUpdatedMap) { + if (num > 0 && pUpdatedMap) { saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur); } @@ -2807,15 +2812,14 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } -static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SHashObj* pUpdatedMap) { +static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, + SHashObj* pUpdatedMap) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExprSupp* pSup = &pOperatorInfo->exprSupp; int32_t numOfOutput = pSup->numOfExprs; int32_t step = 1; - bool ascScan = true; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t forwardRows = 0; @@ -2824,7 +2828,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = {0}; if (IS_FINAL_OP(pInfo)) { @@ -3165,7 +3169,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); - doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); + doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3183,7 +3187,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); - doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); + doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL); } } @@ -3591,7 +3595,8 @@ SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { // don't add new window SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); + STimeWindow searchWin = {.skey = startTs, .ekey = endTs}; + SArray* pWinInfos = getWinInfos(pAggSup, groupId); pAggSup->pCurWins = pWinInfos; int32_t size = taosArrayGetSize(pWinInfos); @@ -3603,7 +3608,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start SResultWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); - if (isInWindow(pWin, startTs, gap)) { + if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { *pIndex = index; return pWin; } @@ -3611,7 +3616,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start if (index + 1 < size) { pWin = taosArrayGet(pWinInfos, index + 1); - if (isInWindow(pWin, startTs, gap)) { + if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { *pIndex = index + 1; return pWin; } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { @@ -3789,7 +3794,7 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true); compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); - if (pWinInfo->isOutput) { + if (pWinInfo->isOutput && pStDeleted) { SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); pWinInfo->isOutput = false; @@ -3882,19 +3887,24 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - int32_t winIndex = 0; - while (1) { - SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); - if (!pCurWin) { - break; - } + int32_t winIndex = 0; + SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); + if (!pCurWin) { + continue; + } + + do { SResultWindowInfo delWin = *pCurWin; deleteWindow(pAggSup->pCurWins, winIndex, fp); if (result) { delWin.groupId = gpDatas[i]; taosArrayPush(result, &delWin); } - } + if (winIndex >= taosArrayGetSize(pAggSup->pCurWins)) { + break; + } + pCurWin = taosArrayGet(pAggSup->pCurWins, winIndex); + } while (pCurWin->win.skey <= endDatas[i]); } } @@ -3975,26 +3985,16 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It } static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput, - SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) { + SOperatorInfo* pOperator, SHashObj* pStUpdated) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t size = taosArrayGetSize(pWinArray); + int32_t size = taosArrayGetSize(pWinArray); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; uint64_t groupId = pParentWin->groupId; - int32_t winIndex = 0; - if (needCreate) { - pParentWin = - getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex); - } - setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); - int32_t num = 0; + int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; @@ -4007,31 +4007,36 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin for (int32_t k = index; k < chWinSize; k++) { SResultWindowInfo* pChWin = taosArrayGet(pChWins, k); if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) { + int32_t winIndex = 0; + SResultWindowInfo* pNewParWin = + getSessionTimeWindow(&pInfo->streamAggSup, pChWin->win.skey, pChWin->win.ekey, groupId, 0, &winIndex); + SResultRow* pPareResult = NULL; + setWindowOutputBuf(pNewParWin, &pPareResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, + &pInfo->streamAggSup, pTaskInfo); SResultRow* pChResult = NULL; setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput, pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pChWin->win, true); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pNewParWin->win, true); compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + + int32_t winNum = getNumCompactWindow(pInfo->streamAggSup.pCurWins, winIndex, pInfo->gap); + if (winNum > 0) { + compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, NULL, pOperator); + } + SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId); releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage); - num++; - continue; + + bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pNewParWin->pos.pageId); + setBufPageDirty(bufPage, true); + releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); + SWinKey value = {.ts = pNewParWin->win.skey, .groupId = groupId}; + taosHashPut(pStUpdated, &pNewParWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); } else if (!pChWin->isClosed) { break; } } } - if (num == 0 && needCreate) { - deleteWindow(pInfo->streamAggSup.pCurWins, winIndex, NULL); - } - if (pStUpdated && num > 0) { - SWinKey value = {.ts = pParentWin->win.skey, .groupId = groupId}; - taosHashPut(pStUpdated, &pParentWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); - } - SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId); - ASSERT(size > 0); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); } } @@ -4192,7 +4197,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, pChildOp->exprSupp.numOfExprs, 0, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, NULL, false); + rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); } taosArrayDestroy(pWins); continue; @@ -4206,7 +4211,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated, true); + rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); removeSessionResults(pStUpdated, pWins); @@ -4743,7 +4748,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_CLEAR) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted); continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA) { + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo); copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); @@ -5468,25 +5473,24 @@ _error: return NULL; } -static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, - SSDataBlock* pBlock, int32_t scanFlag, SHashObj* pUpdatedMap) { +static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, + int32_t scanFlag, SHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t startPos = 0; - int32_t numOfOutput = pSup->numOfExprs; + int32_t startPos = 0; + int32_t numOfOutput = pSup->numOfExprs; SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; - uint64_t tableGroupId = pBlock->info.groupId; - bool ascScan = true; - TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); - SResultRow* pResult = NULL; + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = true; + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; - STimeWindow win = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); - int32_t ret = TSDB_CODE_SUCCESS; + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + int32_t ret = TSDB_CODE_SUCCESS; if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, @@ -5547,11 +5551,88 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo } } +static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, + SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pSup = &pOperatorInfo->exprSupp; + int32_t numOfOutput = pSup->numOfExprs; + int32_t step = 1; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; + int32_t aa = 4; + + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + + int32_t startPos = 0; + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = + getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + while (1) { + bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); + if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } + + int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, + &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + pSDataBlock->info.rows, numOfOutput); + SWinKey key = { + .ts = nextWin.skey, + .groupId = tableGroupId, + }; + saveOutput(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize); + releaseOutputBuf(pTaskInfo, &key, pResult); + int32_t prevEndPos = (forwardRows - 1) * step + startPos; + ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); + startPos = + getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (startPos < 0) { + break; + } + } +} + +void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + return; + } + + // clear the existed group id + pBlock->info.groupId = 0; + buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo); +} + static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int64_t maxTs = INT64_MIN; - SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int64_t maxTs = INT64_MIN; + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -5594,7 +5675,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA) { + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -5622,6 +5703,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { maxTs = TMAX(maxTs, pBlock->info.window.ekey); doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); + // new disc buf + // doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); @@ -5664,6 +5747,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } taosArraySort(pUpdated, resultrowComparAsc); + // new disc buf finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -5676,6 +5760,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + // new disc buf + // doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; } @@ -5697,25 +5783,29 @@ void destroyStreamIntervalOperatorInfo(void* param) { SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); ASSERT(numOfCols > 0); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SInterval interval = {.interval = pIntervalPhyNode->interval, - .sliding = pIntervalPhyNode->sliding, - .intervalUnit = pIntervalPhyNode->intervalUnit, - .slidingUnit = pIntervalPhyNode->slidingUnit, - .offset = pIntervalPhyNode->offset, - .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, }; - STimeWindowAggSupp twAggSupp = {.waterMark = pIntervalPhyNode->window.watermark, - .calTrigger = pIntervalPhyNode->window.triggerType, - .maxTs = INT64_MIN, }; + SInterval interval = { + .interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, + }; + STimeWindowAggSupp twAggSupp = { + .waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + }; ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); pOperator->pTaskInfo = pTaskInfo; pInfo->interval = interval; @@ -5732,11 +5822,11 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys } } - pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;; + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -5758,8 +5848,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, - destroyStreamIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index dcf58759b0..0d7fd1a6da 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2795,6 +2795,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); return 0; } @@ -2871,7 +2873,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } } #endif - + if (numOfElems == 0) { + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -2892,6 +2897,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); return 0; } @@ -2952,7 +2959,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } } #endif - + if (numOfElems == 0) { + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -4701,7 +4711,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { colDataAppendNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } continue; } @@ -4714,11 +4724,11 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { } else { pInfo->count = 0; } - colDataAppend(pOutput, i, (char*)&output, false); + colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5de9c52cc1..3f472b53a0 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1183,7 +1183,9 @@ void onUdfcPipeClose(uv_handle_t *handle) { QUEUE_REMOVE(&task->procTaskQueue); uv_sem_post(&task->taskSem); } - conn->session->udfUvPipe = NULL; + if (conn->session != NULL) { + conn->session->udfUvPipe = NULL; + } taosMemoryFree(conn->readBuf.buf); taosMemoryFree(conn); taosMemoryFree((uv_pipe_t *) handle); @@ -1803,6 +1805,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { if (session->udfUvPipe == NULL) { fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName); + taosMemoryFree(session); return TSDB_CODE_UDF_PIPE_NO_PIPE; } @@ -1821,7 +1824,11 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { udfcRunUdfUvTask(task, UV_TASK_DISCONNECT); fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); - + //TODO: synchronization refactor between libuv event loop and request thread + if (session->udfUvPipe != NULL && session->udfUvPipe->data != NULL) { + SClientUvConn *conn = session->udfUvPipe->data; + conn->session = NULL; + } taosMemoryFree(session); taosMemoryFree(task); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 691eb6771c..ecf9136073 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -81,28 +81,28 @@ __compar_fn_t idxGetCompar(int8_t type) { } return getComparFunc(type, 0); } -static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_THAN, a, b, type); } -static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_EQUAL, a, b, type); } -static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_THAN, a, b, type); } -static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } -static TExeCond tCompareContains(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareContains(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } -static TExeCond tCompareEqual(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index e7f221de3d..b65acc4672 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -88,7 +88,7 @@ typedef struct SIFCtx { SIndexMetaArg arg; } SIFCtx; -static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { +static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { if (src == OP_TYPE_GREATER_THAN) { *dst = QUERY_GREATER_THAN; } else if (src == OP_TYPE_GREATER_EQUAL) { @@ -110,10 +110,9 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { } typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); - static sif_func_t sifNullFunc = NULL; -static void sifFreeParam(SIFParam *param) { +static FORCE_INLINE void sifFreeParam(SIFParam *param) { if (param == NULL) return; taosArrayDestroy(param->result); @@ -123,7 +122,7 @@ static void sifFreeParam(SIFParam *param) { param->pFilter = NULL; } -static int32_t sifGetOperParamNum(EOperatorType ty) { +static FORCE_INLINE int32_t sifGetOperParamNum(EOperatorType ty) { if (OP_TYPE_IS_NULL == ty || OP_TYPE_IS_NOT_NULL == ty || OP_TYPE_IS_TRUE == ty || OP_TYPE_IS_NOT_TRUE == ty || OP_TYPE_IS_FALSE == ty || OP_TYPE_IS_NOT_FALSE == ty || OP_TYPE_IS_UNKNOWN == ty || OP_TYPE_IS_NOT_UNKNOWN == ty || OP_TYPE_MINUS == ty) { @@ -131,14 +130,14 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } -static int32_t sifValidOp(EOperatorType ty) { +static FORCE_INLINE int32_t sifValidOp(EOperatorType ty) { if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) || (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) { return -1; } return 0; } -static int32_t sifValidColumn(SColumnNode *cn) { +static FORCE_INLINE int32_t sifValidColumn(SColumnNode *cn) { // add more check if (cn == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -149,7 +148,7 @@ static int32_t sifValidColumn(SColumnNode *cn) { return TSDB_CODE_SUCCESS; } -static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) { +static FORCE_INLINE SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) { // enh rule later if (type == LOGIC_COND_TYPE_AND) { if (ls == SFLT_NOT_INDEX || rs == SFLT_NOT_INDEX) { @@ -167,7 +166,7 @@ static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SI return SFLT_NOT_INDEX; } -static int32_t sifGetValueFromNode(SNode *node, char **value) { +static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { // covert data From snode; SValueNode *vn = (SValueNode *)node; @@ -205,7 +204,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) { return TSDB_CODE_SUCCESS; } -static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { +static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SOperatorNode *nd = (SOperatorNode *)node; assert(nodeType(node) == QUERY_NODE_OPERATOR); SColumnNode *l = (SColumnNode *)nd->pLeft; @@ -355,30 +354,30 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu return TSDB_CODE_QRY_INVALID_INPUT; } -typedef int (*Filter)(void *a, void *b, int16_t dtype); +typedef int (*FilterFunc)(void *a, void *b, int16_t dtype); -int sifGreaterThan(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifGreaterThan(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_GREATER_THAN, a, b); } -int sifGreaterEqual(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifGreaterEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_GREATER_EQUAL, a, b); } -int sifLessEqual(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifLessEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_LESS_EQUAL, a, b); } -int sifLessThan(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifLessThan(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return (int)tDoCompare(func, QUERY_LESS_THAN, a, b); } -int sifEqual(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); //__compar_fn_t func = idxGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } -static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { +static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reverse) { if (type == QUERY_LESS_EQUAL || type == QUERY_LESS_THAN) { *reverse = true; } else { @@ -470,8 +469,8 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP indexMultiTermQueryAdd(mtm, tm, qtype); ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); } else { - bool reverse; - Filter filterFunc = sifGetFilterFunc(qtype, &reverse); + bool reverse; + FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse); SMetaFltParam param = {.suid = arg->suid, .cid = left->colId, @@ -498,72 +497,72 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP return ret; } -static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_THAN; return sifDoIndex(left, right, id, output); } -static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_THAN; return sifDoIndex(left, right, id, output); } -static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_IN; return sifDoIndex(left, right, id, output); } -static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_IN; return sifDoIndex(left, right, id, output); } -static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LIKE; return sifDoIndex(left, right, id, output); } -static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_LIKE; return sifDoIndex(left, right, id, output); } -static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_MATCH; return sifDoIndex(left, right, id, output); } -static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NMATCH; return sifDoIndex(left, right, id, output); } -static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_JSON_CONTAINS; return sifDoIndex(left, right, id, output); } -static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) { +static FORCE_INLINE int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) { // return 0 return 0; } -static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; } -static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *status) { +static FORCE_INLINE int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *status) { // impl later *status = SFLT_ACCURATE_INDEX; switch (funcId) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 240aa0d6c0..ad9a467dee 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6604,7 +6604,17 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->colId = pSchema->colId; SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema); - if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { + + if (QUERY_NODE_VALUE != pStmt->pVal->node.type) { + SValueNode *pVal = NULL; + pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal); + if (pCxt->errCode) { + return pCxt->errCode; + } + + nodesDestroyNode((SNode*)pStmt->pVal); + pStmt->pVal = pVal; + } else if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { return pCxt->errCode; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index dfd6f012cc..5efdbb4679 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -112,6 +112,29 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); } +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { + // todo refactor + int32_t size = *pVLen; + if (streamStateGet(pState, key, pVal, pVLen) == 0) { + return 0; + } + void* tmp = taosMemoryCalloc(1, size); + if (streamStatePut(pState, key, &tmp, size) == 0) { + taosMemoryFree(tmp); + int32_t code = streamStateGet(pState, key, pVal, pVLen); + ASSERT(code == 0); + return code; + } + taosMemoryFree(tmp); + return -1; +} + +int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) { + // todo refactor + streamFreeVal(pVal); + return 0; +} + SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 51098374b0..6f29b54f80 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2766,8 +2766,6 @@ const char* syncStr(ESyncState state) { } int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { - SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); - if (ths->state != TAOS_SYNC_STATE_FOLLOWER) { syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); return 0; @@ -2799,6 +2797,8 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p } */ + SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index b42aba560f..faebe5bbec 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1992,6 +1992,313 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { } } +// ---- message process SyncHeartbeat---- +SyncHeartbeat* syncHeartbeatBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncHeartbeat); + SyncHeartbeat* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_HEARTBEAT; + return pMsg; +} + +void syncHeartbeatDestroy(SyncHeartbeat* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncHeartbeatSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncHeartbeat* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncHeartbeatDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncHeartbeatSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg) { + syncHeartbeatDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncHeartbeat* pMsg = syncHeartbeatDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncHeartbeat", pRoot); + return pJson; +} + +char* syncHeartbeat2Str(const SyncHeartbeat* pMsg) { + cJSON* pJson = syncHeartbeat2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncHeartbeatPrint(const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + printf("syncHeartbeatPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + printf("syncHeartbeatPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatLog(const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + sTrace("syncHeartbeatLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncHeartbeat2Str(pMsg); + sTrace("syncHeartbeatLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// ---- message process SyncHeartbeatReply---- +SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncHeartbeatReply); + SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_HEARTBEAT_REPLY; + return pMsg; +} + +void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncHeartbeatReplySerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncHeartbeatReplyDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncHeartbeatReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg) { + syncHeartbeatReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncHeartbeatReply* pMsg = syncHeartbeatReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncHeartbeatReply", pRoot); + return pJson; +} + +char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg) { + cJSON* pJson = syncHeartbeatReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + printf("syncHeartbeatReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + printf("syncHeartbeatReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + sTrace("syncHeartbeatReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncHeartbeatReply2Str(pMsg); + sTrace("syncHeartbeatReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + // ---- message process SyncApplyMsg---- SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncApplyMsg) + dataLen; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 72845d0c1d..b9cc7a391d 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -57,6 +57,8 @@ add_executable(syncLeaderTransferTest "") add_executable(syncReconfigFinishTest "") add_executable(syncRestoreFromSnapshot "") add_executable(syncRaftCfgIndexTest "") +add_executable(syncHeartbeatTest "") +add_executable(syncHeartbeatReplyTest "") target_sources(syncTest @@ -295,6 +297,14 @@ target_sources(syncRaftCfgIndexTest PRIVATE "syncRaftCfgIndexTest.cpp" ) +target_sources(syncHeartbeatTest + PRIVATE + "syncHeartbeatTest.cpp" +) +target_sources(syncHeartbeatReplyTest + PRIVATE + "syncHeartbeatReplyTest.cpp" +) target_include_directories(syncTest @@ -592,6 +602,16 @@ target_include_directories(syncRaftCfgIndexTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncHeartbeatTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncHeartbeatReplyTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -830,6 +850,14 @@ target_link_libraries(syncRaftCfgIndexTest sync gtest_main ) +target_link_libraries(syncHeartbeatTest + sync + gtest_main +) +target_link_libraries(syncHeartbeatReplyTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp new file mode 100644 index 0000000000..0ccd7b70bb --- /dev/null +++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp @@ -0,0 +1,105 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncHeartbeatReply *createMsg() { + SyncHeartbeatReply *pMsg = syncHeartbeatReplyBuild(1000); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + + pMsg->term = 33; + pMsg->privateTerm = 44; + pMsg->startTime = taosGetTimestampMs(); + return pMsg; +} + +void test1() { + SyncHeartbeatReply *pMsg = createMsg(); + syncHeartbeatReplyLog2((char *)"test1:", pMsg); + syncHeartbeatReplyDestroy(pMsg); +} + +void test2() { + SyncHeartbeatReply *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncHeartbeatReplySerialize(pMsg, serialized, len); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000); + syncHeartbeatReplyDeserialize(serialized, len, pMsg2); + syncHeartbeatReplyLog2((char *)"test2: syncHeartbeatReplySerialize -> syncHeartbeatReplyDeserialize ", + pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test3() { + SyncHeartbeatReply *pMsg = createMsg(); + uint32_t len; + char * serialized = syncHeartbeatReplySerialize2(pMsg, &len); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyDeserialize2(serialized, len); + syncHeartbeatReplyLog2((char *)"test3: syncHeartbeatReplySerialize3 -> syncHeartbeatReplyDeserialize2 ", + pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test4() { + SyncHeartbeatReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000); + syncHeartbeatReplyFromRpcMsg(&rpcMsg, pMsg2); + syncHeartbeatReplyLog2((char *)"test4: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg ", + pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test5() { + SyncHeartbeatReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyFromRpcMsg2(&rpcMsg); + syncHeartbeatReplyLog2((char *)"test5: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg2 ", + pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +int main() { + gRaftDetailLog = true; + + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp new file mode 100644 index 0000000000..d910c828f1 --- /dev/null +++ b/source/libs/sync/test/syncHeartbeatTest.cpp @@ -0,0 +1,99 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncHeartbeat *createMsg() { + SyncHeartbeat *pMsg = syncHeartbeatBuild(789); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 8; + pMsg->commitIndex = 33; + pMsg->privateTerm = 44; + return pMsg; +} + +void test1() { + SyncHeartbeat *pMsg = createMsg(); + syncHeartbeatLog2((char *)"test1:", pMsg); + syncHeartbeatDestroy(pMsg); +} + +void test2() { + SyncHeartbeat *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncHeartbeatSerialize(pMsg, serialized, len); + SyncHeartbeat *pMsg2 = syncHeartbeatBuild(789); + syncHeartbeatDeserialize(serialized, len, pMsg2); + syncHeartbeatLog2((char *)"test2: syncHeartbeatSerialize -> syncHeartbeatDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test3() { + SyncHeartbeat *pMsg = createMsg(); + uint32_t len; + char * serialized = syncHeartbeatSerialize2(pMsg, &len); + SyncHeartbeat *pMsg2 = syncHeartbeatDeserialize2(serialized, len); + syncHeartbeatLog2((char *)"test3: syncHeartbeatSerialize2 -> syncHeartbeatDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test4() { + SyncHeartbeat *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeat2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeat *pMsg2 = (SyncHeartbeat *)taosMemoryMalloc(rpcMsg.contLen); + syncHeartbeatFromRpcMsg(&rpcMsg, pMsg2); + syncHeartbeatLog2((char *)"test4: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test5() { + SyncHeartbeat *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeat2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeat *pMsg2 =syncHeartbeatFromRpcMsg2(&rpcMsg); + syncHeartbeatLog2((char *)"test5: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index f90c392788..bec07b935e 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -502,7 +502,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { i64 offset; int ret; - offset = pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); + offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { ASSERT(0); return -1; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 275e7b42cc..1559c85e23 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -126,22 +126,22 @@ _OVER: return code; } -static void destroyHttpClient(SHttpClient* cli) { +static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) { taosMemoryFree(cli->wbuf); taosMemoryFree(cli->rbuf); taosMemoryFree(cli->addr); taosMemoryFree(cli); } -static void clientCloseCb(uv_handle_t* handle) { +static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) { SHttpClient* cli = handle->data; destroyHttpClient(cli); } -static void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SHttpClient* cli = handle->data; buf->base = cli->rbuf; buf->len = HTTP_RECV_BUF_SIZE; } -static void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { +static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { SHttpClient* cli = handle->data; if (nread < 0) { uError("http-report recv error:%s", uv_err_name(nread)); @@ -173,7 +173,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); } -static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { +static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4b5441f738..62cdf4e784 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -69,11 +69,9 @@ typedef struct SCliThrd { SAsyncPool* asyncPool; uv_prepare_t* prepare; void* pool; // conn pool - + // timer handles SArray* timerList; - // msg queue - queue msg; TdThreadMutex msgMtx; SDelayQueue* delayQueue; @@ -108,7 +106,7 @@ static void cliReadTimeoutCb(uv_timer_t* handle); // register timer in each thread to clear expire conn // static void cliTimeoutCb(uv_timer_t* handle); // alloc buffer for recv -static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static FORCE_INLINE void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); // callback after recv nbytes from socket static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); // callback after send data to socket @@ -132,10 +130,10 @@ static void cliSend(SCliConn* pConn); static void cliDestroyConnMsgs(SCliConn* conn, bool destroy); // cli util func -static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx); -static void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); +static FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx); +static FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); -static int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp); +static FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); @@ -150,13 +148,11 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd); static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL, cliHandleUpdate}; -static void cliSendQuit(SCliThrd* thrd); -static void destroyUserdata(STransMsg* userdata); +static FORCE_INLINE void destroyUserdata(STransMsg* userdata); +static FORCE_INLINE void destroyCmsg(void* cmsg); +static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); +static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); -static int cliRBChoseIdx(STrans* pTransInst); - -static void destroyCmsg(void* cmsg); -static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static SCliThrd* createThrdObj(); static void destroyThrdObj(SCliThrd* pThrd); @@ -434,6 +430,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (pCtx == NULL || pCtx->pSem == NULL) { if (transMsg.info.ahandle == NULL) { + if (REQUEST_NO_RESP(&pMsg->msg)) destroyCmsg(pMsg); once = true; continue; } @@ -885,26 +882,23 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { } return conn; } -void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { +FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { if (pCvtAddr->cvt == false) { return; } - for (int i = 0; i < pEpSet->numOfEps && pEpSet->numOfEps == 1; i++) { - if (strncmp(pEpSet->eps[i].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) { - memset(pEpSet->eps[i].fqdn, 0, TSDB_FQDN_LEN); - memcpy(pEpSet->eps[i].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN); - } + if (pEpSet->numOfEps == 1 && strncmp(pEpSet->eps[0].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) { + memset(pEpSet->eps[0].fqdn, 0, TSDB_FQDN_LEN); + memcpy(pEpSet->eps[0].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN); } } -bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { +FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { if (code != 0) return false; if (pCtx->retryCnt == 0) return false; if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; return true; } - -int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) { +FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) { if (pMsg == NULL) return -1; memset(pResp, 0, sizeof(STransMsg)); @@ -980,6 +974,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { return; } } + STraceId* trace = &pMsg->msg.info.traceId; + tGTrace("%s conn %p ready", pTransInst->label, conn); } static void cliAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; @@ -1128,14 +1124,15 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, return cli; } -static void destroyUserdata(STransMsg* userdata) { +static FORCE_INLINE void destroyUserdata(STransMsg* userdata) { if (userdata->pCont == NULL) { return; } transFreeMsg(userdata->pCont); userdata->pCont = NULL; } -static void destroyCmsg(void* arg) { + +static FORCE_INLINE void destroyCmsg(void* arg) { SCliMsg* pMsg = arg; if (pMsg == NULL) { return; @@ -1201,7 +1198,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(pThrd); } -static void transDestroyConnCtx(STransConnCtx* ctx) { +static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx) { // taosMemoryFree(ctx); } @@ -1220,7 +1217,7 @@ void cliWalkCb(uv_handle_t* handle, void* arg) { } } -int cliRBChoseIdx(STrans* pTransInst) { +FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst) { int8_t index = pTransInst->index; if (pTransInst->numOfThreads == 0) { return -1; @@ -1230,7 +1227,7 @@ int cliRBChoseIdx(STrans* pTransInst) { } return index % pTransInst->numOfThreads; } -static void doDelayTask(void* param) { +static FORCE_INLINE void doDelayTask(void* param) { STaskArg* arg = param; SCliMsg* pMsg = arg->param1; SCliThrd* pThrd = arg->param2; @@ -1264,13 +1261,13 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); } -void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { +FORCE_INLINE void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { if (*val != exp) { *val = newVal; } } -bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { +FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { if ((pResp == NULL || pResp->info.hasEpSet == 0)) { return false; } @@ -1300,7 +1297,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STrans* pTransInst = pThrd->pTransInst; if (pMsg == NULL || pMsg->ctx == NULL) { - tTrace("%s conn %p handle resp", pTransInst->label, pConn); + tDebug("%s conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, NULL); return 0; } @@ -1402,7 +1399,7 @@ void transUnrefCliHandle(void* handle) { cliDestroyConn((SCliConn*)handle, true); } } -SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) { +static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) { SCliThrd* pThrd = NULL; SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle); if (exh == NULL) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index dea96fa3ac..5f3171ee0e 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -424,7 +424,7 @@ void transQueueDestroy(STransQueue* queue) { taosArrayDestroy(queue->q); } -static int32_t timeCompare(const HeapNode* a, const HeapNode* b) { +static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { SDelayTask* arg1 = container_of(a, SDelayTask, node); SDelayTask* arg2 = container_of(b, SDelayTask, node); if (arg1->execTime > arg2->execTime) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7007079f87..c00cafb708 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -125,17 +125,17 @@ static void uvWorkAfterTask(uv_work_t* req, int status); static void uvWalkCb(uv_handle_t* handle, void* arg); static void uvFreeCb(uv_handle_t* handle); -static void uvStartSendRespImpl(SSvrMsg* smsg); +static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg); + static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static void destroySmsg(SSvrMsg* smsg); -// check whether already read complete packet -static SSvrConn* createConn(void* hThrd); -static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); -static void destroyConnRegArg(SSvrConn* conn); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE SSvrConn* createConn(void* hThrd); +static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); +static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); static int reallocConnRef(SSvrConn* conn); @@ -413,7 +413,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { wb->len = len; } -static void uvStartSendRespImpl(SSvrMsg* smsg) { +static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) { SSvrConn* pConn = smsg->pConn; if (pConn->broken) { return; @@ -447,7 +447,7 @@ static void uvStartSendResp(SSvrMsg* smsg) { return; } -static void destroySmsg(SSvrMsg* smsg) { +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) { if (smsg == NULL) { return; } @@ -812,7 +812,7 @@ void* transWorkerThread(void* arg) { return NULL; } -static SSvrConn* createConn(void* hThrd) { +static FORCE_INLINE SSvrConn* createConn(void* hThrd) { SWorkThrd* pThrd = hThrd; SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn)); @@ -842,7 +842,7 @@ static SSvrConn* createConn(void* hThrd) { return pConn; } -static void destroyConn(SSvrConn* conn, bool clear) { +static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear) { if (conn == NULL) { return; } @@ -854,7 +854,7 @@ static void destroyConn(SSvrConn* conn, bool clear) { } } } -static void destroyConnRegArg(SSvrConn* conn) { +static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn) { if (conn->regArg.init == 1) { transFreeMsg(conn->regArg.msg.pCont); conn->regArg.init = 0; diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 1a057c5875..534c177587 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -23,9 +23,9 @@ void simpleTest() { ASSERT_EQ(getTotalBufSize(pBuf), 1024); - SIDList list = getDataBufPagesIdList(pBuf, groupId); + SIDList list = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); + //ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); releaseBufPage(pBuf, pBufPage); @@ -98,7 +98,7 @@ void writeDownTest() { SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId)); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); - SArray* pa = getDataBufPagesIdList(pBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(pa), 5); destroyDiskbasedBuf(pBuf); @@ -152,7 +152,7 @@ void recyclePageTest() { SFilePage* pBufPagex1 = static_cast(getBufPage(pBuf, 1)); - SArray* pa = getDataBufPagesIdList(pBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(pa), 6); destroyDiskbasedBuf(pBuf); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 46bae734ea..57cf477bf8 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -248,6 +248,12 @@ ./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/stream/ignoreExpiredData.sim ./test.sh -f tsim/stream/sliding.sim +#./test.sh -f tsim/stream/partitionbyColumnInterval.sim +#./test.sh -f tsim/stream/partitionbyColumnSession.sim +#./test.sh -f tsim/stream/partitionbyColumnState.sim +#./test.sh -f tsim/stream/deleteInterval.sim +#./test.sh -f tsim/stream/deleteSession.sim +#./test.sh -f tsim/stream/deleteState.sim # ---- transaction ---- ./test.sh -f tsim/trans/lossdata1.sim diff --git a/tests/script/tsim/column/table.sim b/tests/script/tsim/column/table.sim index 4f1d32c373..03c4799681 100644 --- a/tests/script/tsim/column/table.sim +++ b/tests/script/tsim/column/table.sim @@ -159,6 +159,7 @@ if $data01 != 10 then return -1 endi if $data02 != 4.500000000 then + print expect 4.500000000, actual: $data02 return -1 endi if $data03 != 4.500000000 then diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim index ae19a4803b..a9bf8b26eb 100644 --- a/tests/script/tsim/parser/commit.sim +++ b/tests/script/tsim/parser/commit.sim @@ -97,6 +97,7 @@ while $loop <= $loops endw sql select count(*) from $stb if $data00 != $totalNum then + print expect $totalNum , actual: $data00 return -1 endi $loop = $loop + 1 diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index 70ca1179b0..8942f7f702 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -5,7 +5,7 @@ sleep 50 sql connect print =============== create database -sql create database test vgroups 1 +sql create database test vgroups 1; sql select * from information_schema.ins_databases if $rows != 3 then return -1 @@ -13,7 +13,7 @@ endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim new file mode 100644 index 0000000000..dfd0ddc9d3 --- /dev/null +++ b/tests/script/tsim/stream/deleteInterval.sim @@ -0,0 +1,419 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,2,2,2,2.0); +sql insert into t1 values(1648791213002,3,3,3,3.0); +sql insert into t1 values(1648791213003,4,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,2,2,2,2.0); +sql insert into t1 values(1648791213006,3,3,3,3.0); +sql insert into t1 values(1648791213007,4,4,4,4.0); + +sql insert into t1 values(1648791223000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,3,3,3,3.0); +sql insert into t1 values(1648791223003,4,4,4,4.0); + +sql insert into t1 values(1648791233000,1,1,1,1.0); +sql insert into t1 values(1648791233001,2,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,4,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop7 +endi + +sleep 200 + +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop8: +sleep 200 + +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001; + +$loop_count = 0 + +loop11: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop11 +endi + +sleep 200 + +sql delete from st where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop12: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop12 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop12 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop12 +endi + +sql insert into t1 values(1648791213004,3,2,3,1.0); +sql insert into t1 values(1648791213005,3,2,3,1.0); +sql insert into t1 values(1648791213006,3,2,3,1.0); +sql insert into t1 values(1648791223004,1,2,3,1.0); +sql insert into t2 values(1648791213004,3,2,3,1.0); +sql insert into t2 values(1648791213005,3,2,3,1.0); +sql insert into t2 values(1648791213006,3,2,3,1.0); +sql insert into t2 values(1648791223004,1,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006; + +$loop_count = 0 + +loop13: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 3 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t1 values(1648791223005,1,2,3,1.0); +sql insert into t1 values(1648791223006,1,2,3,1.0); +sql insert into t2 values(1648791223005,1,2,3,1.0); +sql insert into t2 values(1648791223006,1,2,3,1.0); + +sql insert into t1 values(1648791233005,4,2,3,1.0); +sql insert into t1 values(1648791233006,2,2,3,1.0); +sql insert into t2 values(1648791233005,5,2,3,1.0); +sql insert into t2 values(1648791233006,3,2,3,1.0); + +sleep 200 + +sql delete from st where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop14: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop14 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop14 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop14 +endi + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim new file mode 100644 index 0000000000..541609633b --- /dev/null +++ b/tests/script/tsim/stream/deleteSession.sim @@ -0,0 +1,532 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,2,2,2,2.0); +sql insert into t1 values(1648791213002,3,3,3,3.0); +sql insert into t1 values(1648791213003,4,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,2,2,2,2.0); +sql insert into t1 values(1648791213006,3,3,3,3.0); +sql insert into t1 values(1648791213007,4,4,4,4.0); + +sql insert into t1 values(1648791223000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,3,3,3,3.0); +sql insert into t1 values(1648791223003,4,4,4,4.0); + +sql insert into t1 values(1648791233000,1,1,1,1.0); +sql insert into t1 values(1648791233001,2,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,4,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop7 +endi + +sleep 200 + +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop8: +sleep 200 + +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001; + +$loop_count = 0 + +loop11: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop11 +endi + +sleep 200 + +sql delete from st where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop12: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop12 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop12 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop12 +endi + +sql insert into t1 values(1648791213004,3,2,3,1.0); +sql insert into t1 values(1648791213005,3,2,3,1.0); +sql insert into t1 values(1648791213006,3,2,3,1.0); +sql insert into t1 values(1648791223004,1,2,3,1.0); +sql insert into t2 values(1648791213004,3,2,3,1.0); +sql insert into t2 values(1648791213005,3,2,3,1.0); +sql insert into t2 values(1648791213006,3,2,3,1.0); +sql insert into t2 values(1648791223004,1,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006; + +$loop_count = 0 + +loop13: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 3 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t1 values(1648791223005,1,2,3,1.0); +sql insert into t1 values(1648791223006,1,2,3,1.0); +sql insert into t2 values(1648791223005,1,2,3,1.0); +sql insert into t2 values(1648791223006,1,2,3,1.0); + +sql insert into t1 values(1648791233005,4,2,3,1.0); +sql insert into t1 values(1648791233006,2,2,3,1.0); +sql insert into t2 values(1648791233005,5,2,3,1.0); +sql insert into t2 values(1648791233006,3,2,3,1.0); + +sleep 200 + +sql delete from st where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop14: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop14 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop14 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop14 +endi + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop database if exists test3; +sql drop database if exists test; +sql create database test3 vgroups 4; +sql create database test vgroups 1; +sql use test3; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); + +sql insert into t1 values(1648791210000,1,1,1,NULL); +sql insert into t1 values(1648791210001,2,2,2,NULL); +sql insert into t2 values(1648791213001,3,3,3,NULL); +sql insert into t2 values(1648791213003,4,4,4,NULL); +sql insert into t1 values(1648791216000,5,5,5,NULL); +sql insert into t1 values(1648791216002,6,6,6,NULL); +sql insert into t1 values(1648791216004,7,7,7,NULL); +sql insert into t2 values(1648791218001,8,8,8,NULL); +sql insert into t2 values(1648791218003,9,9,9,NULL); +sql insert into t1 values(1648791222000,10,10,10,NULL); +sql insert into t1 values(1648791222003,11,11,11,NULL); +sql insert into t1 values(1648791222005,12,12,12,NULL); + +sql insert into t1 values(1648791232005,13,13,13,NULL); +sql insert into t2 values(1648791242005,14,14,14,NULL); + +$loop_count = 0 + +loop19: +sleep 200 +sql select * from test.streamt3 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop19 +endi + +sql delete from t2 where ts >= 1648791213001 and ts <= 1648791218003; + +$loop_count = 0 + +loop20: +sleep 200 +sql select * from test.streamt3 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 5 then + print =====rows=$rows + goto loop20 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop20 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop20 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop20 +endi + +if $data12 != 7 then + print =====data12=$data12 + goto loop20 +endi + +if $data21 != 3 then + print =====data21=$data21 + goto loop20 +endi + +if $data22 != 12 then + print =====data22=$data22 + goto loop20 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop20 +endi + +if $data32 != 13 then + print =====data32=$data32 + goto loop20 +endi + +if $data41 != 1 then + print =====data41=$data41 + goto loop20 +endi + +if $data42 != 14 then + print =====data42=$data42 + goto loop20 +endi + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim new file mode 100644 index 0000000000..ecd9f55340 --- /dev/null +++ b/tests/script/tsim/stream/deleteState.sim @@ -0,0 +1,198 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,1,2,2,2.0); +sql insert into t1 values(1648791213002,1,3,3,3.0); +sql insert into t1 values(1648791213003,1,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,2,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.0); +sql insert into t1 values(1648791223002,2,2,3,1.0); +sql insert into t1 values(1648791223003,2,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,1,2,2,2.0); +sql insert into t1 values(1648791213006,1,3,3,3.0); +sql insert into t1 values(1648791213007,1,4,4,4.0); + +sql insert into t1 values(1648791223000,2,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,2,3,3,3.0); +sql insert into t1 values(1648791223003,2,4,4,4.0); + +sql insert into t1 values(1648791233000,3,1,1,1.0); +sql insert into t1 values(1648791233001,3,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,3,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/partitionbyColumn0.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim similarity index 100% rename from tests/script/tsim/stream/partitionbyColumn0.sim rename to tests/script/tsim/stream/partitionbyColumnInterval.sim diff --git a/tests/script/tsim/stream/partitionbyColumn1.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim similarity index 100% rename from tests/script/tsim/stream/partitionbyColumn1.sim rename to tests/script/tsim/stream/partitionbyColumnSession.sim diff --git a/tests/script/tsim/stream/partitionbyColumn2.sim b/tests/script/tsim/stream/partitionbyColumnState.sim similarity index 100% rename from tests/script/tsim/stream/partitionbyColumn2.sim rename to tests/script/tsim/stream/partitionbyColumnState.sim diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 11df13d451..8987ba3bbd 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -333,7 +333,7 @@ python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py python3 ./test.py -f 7-tmq/tmq_taosx.py -# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py #------------querPolicy 2-----------