From 03bc2d7c6ddb1272c8067747c542bec547b77847 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Feb 2024 10:27:02 +0800 Subject: [PATCH 01/37] test(stream): add unit test cases. --- source/dnode/mnode/impl/inc/mndStream.h | 1 + source/dnode/mnode/impl/src/mndStreamHb.c | 4 +- source/dnode/mnode/impl/src/mndStreamTrans.c | 34 ++++-- source/dnode/mnode/impl/src/mndStreamUtil.c | 2 + .../dnode/mnode/impl/test/stream/stream.cpp | 114 +++++++++++++++--- 5 files changed, 121 insertions(+), 34 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 1084340dc2..aed49809dd 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -119,6 +119,7 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj * int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); void destroyStreamTaskIter(SStreamTaskIter *pIter); diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 5de442951c..3f3a6edf35 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -65,7 +65,7 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI taosArrayPush(pList, pInfo); } -int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint"); if (pTrans == NULL) { return terrno; @@ -115,7 +115,7 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, in } else { mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name, pStream->uid, transId); - code = createStreamResetStatusTrans(pMnode, pStream); + code = mndCreateStreamResetStatusTrans(pMnode, pStream); } } diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 5bfd3933b5..2241d93465 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -261,22 +261,30 @@ int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msg return mndTransAppendRedoAction(pTrans, &action); } +static bool identicalName(const char* pDb, const char* pParam, int32_t len) { + return (strlen(pDb) == len) && (strncmp(pDb, pParam, len) == 0); +} + int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { - // data in the hash table will be removed automatically, no need to remove it here. - SStreamTransInfo *pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len); - if (pTransInfo == NULL) { - return TSDB_CODE_SUCCESS; - } + void *pIter = NULL; - // not checkpoint trans, ignore - if (strcmp(pTransInfo->name, MND_STREAM_CHECKPOINT_NAME) != 0) { - mDebug("not checkpoint trans, not kill it, name:%s, transId:%d", pTransInfo->name, pTransInfo->transId); - return TSDB_CODE_SUCCESS; - } + while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { + SStreamTransInfo *pTransInfo = (SStreamTransInfo *)pIter; + if (strcmp(pTransInfo->name, MND_STREAM_CHECKPOINT_NAME) != 0) { + continue; + } - char *pDupDBName = strndup(pDBName, len); - mndKillTransImpl(pMnode, pTransInfo->transId, pDupDBName); - taosMemoryFree(pDupDBName); + SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId); + if (pStream != NULL) { + if (identicalName(pStream->sourceDb, pDBName, len)) { + mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb); + } else if (identicalName(pStream->targetDb, pDBName, len)) { + mndKillTransImpl(pMnode, pTransInfo->transId, pStream->targetDb); + } + + mndReleaseStream(pMnode, pStream); + } + } return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 3cabce2201..1ae85a2cc6 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -231,6 +231,8 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT taosMemoryFree(pReq); return -1; } + + mDebug("set the resume action for trans:%d", pTrans->id); return 0; } diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index e3bfdb5d6c..369053c7bb 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -29,6 +29,9 @@ #include "../../inc/mndStream.h" namespace { + +static int64_t defStreamId = 999; + SRpcMsg buildHbReq() { SStreamHbMsg msg = {0}; msg.vgId = 1; @@ -40,7 +43,7 @@ SRpcMsg buildHbReq() { entry.nodeId = i + 1; entry.stage = 1; entry.id.taskId = i + 1; - entry.id.streamId = 999; + entry.id.streamId = defStreamId; if (i == 0) { entry.stage = 4; @@ -57,7 +60,7 @@ SRpcMsg buildHbReq() { entry.stage = 1; entry.id.taskId = 5; - entry.id.streamId = 999; + entry.id.streamId = defStreamId; entry.checkpointId = 1; entry.checkpointFailed = true; @@ -118,15 +121,16 @@ void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskI taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry)); taosArrayPush(pExecNode->pTaskList, &id); } + void initStreamExecInfo() { SStreamExecInfo* pExecNode = &execInfo; SStreamTask task = {0}; - setTask(&task, 1, 999, 1); - setTask(&task, 1, 999, 2); - setTask(&task, 1, 999, 3); - setTask(&task, 1, 999, 4); - setTask(&task, 2, 999, 5); + setTask(&task, 1, defStreamId, 1); + setTask(&task, 1, defStreamId, 2); + setTask(&task, 1, defStreamId, 3); + setTask(&task, 1, defStreamId, 4); + setTask(&task, 2, defStreamId, 5); } void initNodeInfo() { @@ -138,38 +142,109 @@ void initNodeInfo() { } } // namespace +class StreamTest : public testing::Test { // 继承了 testing::Test + protected: + + static void SetUpTestSuite() { + mndInitExecInfo(); + initStreamExecInfo(); + initNodeInfo(); + + std::cout<<"setup env for streamTest suite"<(taosMemoryCalloc(1, sizeof(SMnode))); + {// init sdb + SSdbOpt opt = {0}; + opt.path = pMnode->path; + opt.pMnode = pMnode; + opt.pWal = pMnode->pWal; + + pMnode->pSdb = sdbInit(&opt); + } + + SVgroupChangeInfo info; + info.pUpdateNodeList = taosArrayInit(4, sizeof(SNodeUpdateInfo)); + info.pDBMap = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + + const char* pDbName = "test_db_name"; + int32_t len = strlen(pDbName); + + taosHashPut(info.pDBMap, pDbName, len, NULL, 0); + + killAllCheckpointTrans(pMnode, &info); + + SStreamObj stream; + memset(&stream, 0, sizeof(SStreamObj)); + + stream.uid = defStreamId; + stream.lock = 0; + stream.tasks = taosArrayInit(1, POINTER_BYTES); + stream.pHTasksList = taosArrayInit(1, POINTER_BYTES); + + SArray* pLevel = taosArrayInit(1, POINTER_BYTES); + SStreamTask* pTask = static_cast(taosMemoryCalloc(1, sizeof(SStreamTask))); + pTask->id.streamId = defStreamId; + pTask->id.taskId = 1; + pTask->exec.qmsg = (char*)taosMemoryMalloc(1); + + taosArrayPush(pLevel, &pTask); + + taosArrayPush(stream.tasks, &pLevel); + mndCreateStreamResetStatusTrans(pMnode, &stream); + + tFreeStreamObj(&stream); + sdbCleanup(pMnode->pSdb); + taosMemoryFree(pMnode); + + taosArrayDestroy(info.pUpdateNodeList); + taosHashCleanup(info.pDBMap); +} + +TEST_F(StreamTest, plan_Test) { char* ast = "{\"NodeType\":\"101\",\"Name\":\"SelectStmt\",\"SelectStmt\":{\"Distinct\":false,\"Projections\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_1\",\"UserAlias\":\"_wstart\",\"Name\":\"_wstart\",\"Id\":\"89\",\"Type\":\"3505\",\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_2\",\"UserAlias\":\"sum(voltage)\",\"Name\":\"sum\",\"Id\":\"1\",\"Type\":\"14\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"voltage\",\"UserAlias\":\"voltage\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"3\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"voltage\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"#expr_3\",\"UserAlias\":\"groupid\",\"Name\":\"_group_key\",\"Id\":\"96\",\"Type\":\"3754\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"#expr_3\",\"UserAlias\":\"groupid\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"5\",\"ProjId\":\"0\",\"ColType\":\"2\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"groupid\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}}],\"From\":{\"NodeType\":\"6\",\"Name\":\"RealTable\",\"RealTable\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"0\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"DbName\":\"test\",\"tableName\":\"meters\",\"tableAlias\":\"meters\",\"MetaSize\":\"475\",\"Meta\":{\"VgId\":\"0\",\"TableType\":\"1\",\"Uid\":\"6555383776122680534\",\"Suid\":\"6555383776122680534\",\"Sversion\":\"1\",\"Tversion\":\"1\",\"ComInfo\":{\"NumOfTags\":\"2\",\"Precision\":\"0\",\"NumOfColumns\":\"4\",\"RowSize\":\"20\"},\"ColSchemas\":[{\"Type\":\"9\",\"ColId\":\"1\",\"bytes\":\"8\",\"Name\":\"ts\"},{\"Type\":\"6\",\"ColId\":\"2\",\"bytes\":\"4\",\"Name\":\"current\"},{\"Type\":\"4\",\"ColId\":\"3\",\"bytes\":\"4\",\"Name\":\"voltage\"},{\"Type\":\"6\",\"ColId\":\"4\",\"bytes\":\"4\",\"Name\":\"phase\"},{\"Type\":\"4\",\"ColId\":\"5\",\"bytes\":\"4\",\"Name\":\"groupid\"},{\"Type\":\"8\",\"ColId\":\"6\",\"bytes\":\"26\",\"Name\":\"location\"}]},\"VgroupsInfoSize\":\"1340\",\"VgroupsInfo\":{\"Num\":\"2\",\"Vgroups\":[{\"VgId\":\"2\",\"HashBegin\":\"0\",\"HashEnd\":\"2147483646\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"},{\"VgId\":\"3\",\"HashBegin\":\"2147483647\",\"HashEnd\":\"4294967295\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"}]}}},\"PartitionBy\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"groupid\",\"UserAlias\":\"groupid\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"5\",\"ProjId\":\"0\",\"ColType\":\"2\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"groupid\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"Window\":{\"NodeType\":\"14\",\"Name\":\"IntervalWindow\",\"IntervalWindow\":{\"Interval\":{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"115\",\"Bytes\":\"8\"},\"AliasName\":\"c804c3a15ebe05b5baf40ad5ee12be1f\",\"UserAlias\":\"2s\",\"LiteralSize\":\"2\",\"Literal\":\"2s\",\"Duration\":true,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"115\",\"Datum\":\"2000\"}},\"TsPk\":{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}}},\"StmtName\":\"0x1580095ba\",\"HasAggFuncs\":true}}"; // char* ast = "{\"NodeType\":\"101\",\"Name\":\"SelectStmt\",\"SelectStmt\":{\"Distinct\":false,\"Projections\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_1\",\"UserAlias\":\"wstart\",\"Name\":\"_wstart\",\"Id\":\"89\",\"Type\":\"3505\",\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"#expr_2\",\"UserAlias\":\"min(c1)\",\"Name\":\"min\",\"Id\":\"2\",\"Type\":\"8\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"c1\",\"UserAlias\":\"c1\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"2\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c1\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"3\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"#expr_3\",\"UserAlias\":\"max(c2)\",\"Name\":\"max\",\"Id\":\"3\",\"Type\":\"7\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"3\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"c2\",\"UserAlias\":\"c2\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"3\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c2\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_4\",\"UserAlias\":\"sum(c3)\",\"Name\":\"sum\",\"Id\":\"1\",\"Type\":\"14\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"c3\",\"UserAlias\":\"c3\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"4\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c3\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_5\",\"UserAlias\":\"first(c4)\",\"Name\":\"first\",\"Id\":\"33\",\"Type\":\"504\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c4\",\"UserAlias\":\"c4\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"5\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c4\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}},{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"11\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"#expr_6\",\"UserAlias\":\"last(c5)\",\"Name\":\"last\",\"Id\":\"36\",\"Type\":\"506\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"11\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"c5\",\"UserAlias\":\"c5\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"6\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c5\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}},{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"12\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_7\",\"UserAlias\":\"apercentile(c6, 50)\",\"Name\":\"apercentile\",\"Id\":\"12\",\"Type\":\"1\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"12\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"c6\",\"UserAlias\":\"c6\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"7\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c6\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c0c7c76d30bd3dcaefc96f40275bdc0a\",\"UserAlias\":\"50\",\"LiteralSize\":\"2\",\"Literal\":\"50\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"50\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"13\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_8\",\"UserAlias\":\"avg(c7)\",\"Name\":\"avg\",\"Id\":\"8\",\"Type\":\"2\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"13\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"c7\",\"UserAlias\":\"c7\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"8\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c7\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"14\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_9\",\"UserAlias\":\"count(c8)\",\"Name\":\"count\",\"Id\":\"0\",\"Type\":\"3\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"14\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c8\",\"UserAlias\":\"c8\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"9\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c8\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"6\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_10\",\"UserAlias\":\"spread(c1)\",\"Name\":\"spread\",\"Id\":\"17\",\"Type\":\"11\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"c1\",\"UserAlias\":\"c1\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"2\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c1\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_11\",\"UserAlias\":\"stddev(c2)\",\"Name\":\"stddev\",\"Id\":\"4\",\"Type\":\"12\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"3\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"c2\",\"UserAlias\":\"c2\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"3\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c2\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"8\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_12\",\"UserAlias\":\"hyperloglog(c11)\",\"Name\":\"hyperloglog\",\"Id\":\"43\",\"Type\":\"17\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"8\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c11\",\"UserAlias\":\"c11\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"12\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c11\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"10\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"26\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_13\",\"UserAlias\":\"timediff(1, 0, 1h)\",\"Name\":\"timediff\",\"Id\":\"81\",\"Type\":\"2501\",\"Parameters\":[{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c4ca4238a0b923820dcc509a6f75849b\",\"UserAlias\":\"1\",\"LiteralSize\":\"1\",\"Literal\":\"1\",\"Duration\":false,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"1\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"cfcd208495d565ef66e7dff9f98764da\",\"UserAlias\":\"0\",\"LiteralSize\":\"1\",\"Literal\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"104\",\"Bytes\":\"8\"},\"AliasName\":\"7c68645d71b803bf0ba2f22519f73e08\",\"UserAlias\":\"1h\",\"LiteralSize\":\"2\",\"Literal\":\"1h\",\"Duration\":true,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"104\",\"Datum\":\"3600000\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"1\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"8\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"96\"},\"AliasName\":\"#expr_14\",\"UserAlias\":\"timezone()\",\"Name\":\"timezone\",\"Id\":\"84\",\"Type\":\"2503\",\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}}],\"From\":{\"NodeType\":\"6\",\"Name\":\"RealTable\",\"RealTable\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"0\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"DbName\":\"test\",\"tableName\":\"at_once_interval_ext_stb\",\"tableAlias\":\"at_once_interval_ext_stb\",\"MetaSize\":\"2008\",\"Meta\":{\"VgId\":\"0\",\"TableType\":\"1\",\"Uid\":\"5129202035162885657\",\"Suid\":\"5129202035162885657\",\"Sversion\":\"1\",\"Tversion\":\"1\",\"ComInfo\":{\"NumOfTags\":\"13\",\"Precision\":\"0\",\"NumOfColumns\":\"14\",\"RowSize\":\"85\"},\"ColSchemas\":[{\"Type\":\"9\",\"ColId\":\"1\",\"bytes\":\"8\",\"Name\":\"ts\"},{\"Type\":\"2\",\"ColId\":\"2\",\"bytes\":\"1\",\"Name\":\"c1\"},{\"Type\":\"3\",\"ColId\":\"3\",\"bytes\":\"2\",\"Name\":\"c2\"},{\"Type\":\"4\",\"ColId\":\"4\",\"bytes\":\"4\",\"Name\":\"c3\"},{\"Type\":\"5\",\"ColId\":\"5\",\"bytes\":\"8\",\"Name\":\"c4\"},{\"Type\":\"11\",\"ColId\":\"6\",\"bytes\":\"1\",\"Name\":\"c5\"},{\"Type\":\"12\",\"ColId\":\"7\",\"bytes\":\"2\",\"Name\":\"c6\"},{\"Type\":\"13\",\"ColId\":\"8\",\"bytes\":\"4\",\"Name\":\"c7\"},{\"Type\":\"14\",\"ColId\":\"9\",\"bytes\":\"8\",\"Name\":\"c8\"},{\"Type\":\"6\",\"ColId\":\"10\",\"bytes\":\"4\",\"Name\":\"c9\"},{\"Type\":\"7\",\"ColId\":\"11\",\"bytes\":\"8\",\"Name\":\"c10\"},{\"Type\":\"8\",\"ColId\":\"12\",\"bytes\":\"8\",\"Name\":\"c11\"},{\"Type\":\"10\",\"ColId\":\"13\",\"bytes\":\"26\",\"Name\":\"c12\"},{\"Type\":\"1\",\"ColId\":\"14\",\"bytes\":\"1\",\"Name\":\"c13\"},{\"Type\":\"2\",\"ColId\":\"15\",\"bytes\":\"1\",\"Name\":\"t1\"},{\"Type\":\"3\",\"ColId\":\"16\",\"bytes\":\"2\",\"Name\":\"t2\"},{\"Type\":\"4\",\"ColId\":\"17\",\"bytes\":\"4\",\"Name\":\"t3\"},{\"Type\":\"5\",\"ColId\":\"18\",\"bytes\":\"8\",\"Name\":\"t4\"},{\"Type\":\"11\",\"ColId\":\"19\",\"bytes\":\"1\",\"Name\":\"t5\"},{\"Type\":\"12\",\"ColId\":\"20\",\"bytes\":\"2\",\"Name\":\"t6\"},{\"Type\":\"13\",\"ColId\":\"21\",\"bytes\":\"4\",\"Name\":\"t7\"},{\"Type\":\"14\",\"ColId\":\"22\",\"bytes\":\"8\",\"Name\":\"t8\"},{\"Type\":\"6\",\"ColId\":\"23\",\"bytes\":\"4\",\"Name\":\"t9\"},{\"Type\":\"7\",\"ColId\":\"24\",\"bytes\":\"8\",\"Name\":\"t10\"},{\"Type\":\"8\",\"ColId\":\"25\",\"bytes\":\"8\",\"Name\":\"t11\"},{\"Type\":\"10\",\"ColId\":\"26\",\"bytes\":\"26\",\"Name\":\"t12\"},{\"Type\":\"1\",\"ColId\":\"27\",\"bytes\":\"1\",\"Name\":\"t13\"}]},\"VgroupsInfoSize\":\"1340\",\"VgroupsInfo\":{\"Num\":\"2\",\"Vgroups\":[{\"VgId\":\"14\",\"HashBegin\":\"0\",\"HashEnd\":\"2147483646\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"},{\"VgId\":\"15\",\"HashBegin\":\"2147483647\",\"HashEnd\":\"4294967295\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"}]}}},\"Tags\":[{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}}],\"Window\":{\"NodeType\":\"14\",\"Name\":\"IntervalWindow\",\"IntervalWindow\":{\"Interval\":{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"115\",\"Bytes\":\"8\"},\"AliasName\":\"1fd7635317edfeca9054894ac9ef9b5e\",\"UserAlias\":\"14s\",\"LiteralSize\":\"3\",\"Literal\":\"14s\",\"Duration\":true,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"115\",\"Datum\":\"14000\"}},\"TsPk\":{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}}},\"StmtName\":\"0x150146d14\",\"HasAggFuncs\":true}}"; SNode * pAst = NULL; SQueryPlan *pPlan = NULL; if (taosCreateLog("taoslog", 10, "/etc/taos", NULL, NULL, NULL, NULL, 1) != 0) { - // ignore create log failed, only print - printf(" WARING: Create failed:%s. configDir\n", strerror(errno)); + // ignore create log failed, only print + printf(" WARING: Create failed:%s. configDir\n", strerror(errno)); } if (nodesStringToNode(ast, &pAst) < 0) { - ASSERT(0); + ASSERT(0); } - SPlanContext cxt = { 0 }; + SPlanContext cxt = {0}; cxt.pAstRoot = pAst; cxt.topicQuery = false; cxt.streamQuery = true; @@ -181,10 +256,11 @@ TEST(testCase, plan_Test) { // using ast and param to build physical plan if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { - ASSERT(0); + ASSERT(0); } + if (pAst != NULL) nodesDestroyNode(pAst); - nodesDestroyNode((SNode *)pPlan); + nodesDestroyNode((SNode*)pPlan); } #pragma GCC diagnostic pop \ No newline at end of file From b3c4fe489ff398ab188241c7356b44ac120234bb Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 20 Feb 2024 13:22:35 +0800 Subject: [PATCH 02/37] feat: remove todo --- source/libs/executor/src/scanoperator.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index dc9fceee84..cb83cb7599 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3317,12 +3317,6 @@ _error: // table merge scan operator // table merge scan operator -// TODO: limit / duration optimization -// TODO: get block from tsdReader function, with task killed, func_data all filter out, skip, finish -// TODO: error processing, memory freeing -// TODO: add log for error and perf -// TODO: tsdb reader open/close dynamically -// TODO: blockdata deep cleanup static int32_t subTblRowCompareFn(const void* pLeft, const void* pRight, void* param) { int32_t left = *(int32_t*)pLeft; From b2ad162a8b770032594020bd18f600f57b761610 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Feb 2024 17:13:58 +0800 Subject: [PATCH 03/37] enh: remove obsolete codes --- include/common/tmsg.h | 1 - source/dnode/mnode/impl/src/mndDnode.c | 3 --- 2 files changed, 4 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1b438ba026..87d8ed76a8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1613,7 +1613,6 @@ typedef struct { SEp ep; char active[TSDB_ACTIVE_KEY_LEN]; char connActive[TSDB_CONN_ACTIVE_KEY_LEN]; - char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeInfo; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cb8a24e675..6974c6b0e8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -410,9 +410,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.ep.port = pDnode->port; dInfo.offlineReason = pDnode->offlineReason; tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN); - tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN); - tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1); sdbRelease(pSdb, pDnode); if (mndIsMnode(pMnode, pDnode->id)) { dInfo.isMnode = 1; From 87ad111b2c7177eab7751ffc32098fe41227a884 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 20 Feb 2024 18:40:25 +0800 Subject: [PATCH 04/37] coverage: snapshot add stop taosd and start --- tests/army/community/cluster/snapshot.py | 18 +++++++++++++++++- tests/army/frame/srvCtl.py | 4 ++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index b21cbb1ad8..eef650cc77 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -25,6 +25,7 @@ from frame.cases import * from frame.sql import * from frame.caseBase import * from frame import * +from frame.srvCtl import * class TDTestCase(TBase): @@ -65,6 +66,21 @@ class TDTestCase(TBase): sql = f"select avg(dc) from {self.db}.{self.stb}" tdSql.checkFirstValue(sql, 200) + def alterReplica3(self): + sql = f"alter database {self.db} replica 3" + tdSql.execute(sql, show=True) + time.sleep(2) + sc.dnodeStop(2) + sc.dnodeStop(3) + time.sleep(5) + sc.dnodeStart(2) + sc.dnodeStart(3) + + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + def doAction(self): tdLog.info(f"do action.") self.flushDb() @@ -81,7 +97,7 @@ class TDTestCase(TBase): self.alterReplica(1) self.checkAggCorrect() self.compactDb() - self.alterReplica(3) + self.alterReplica3() vgids = self.getVGroup(self.db) selid = random.choice(vgids) diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 091856056b..3a9b0cdf4b 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -33,14 +33,14 @@ class srvCtl: # control server # - # start + # start idx base is 1 def dnodeStart(self, idx): if clusterDnodes.getModel() == 'cluster': return clusterDnodes.starttaosd(idx) return tdDnodes.starttaosd(idx) - # stop + # stop idx base is 1 def dnodeStop(self, idx): if clusterDnodes.getModel() == 'cluster': return clusterDnodes.stoptaosd(idx) From 73ec02ceb4cfb8a8195ab5cd04a2c024201ce7bf Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 20 Feb 2024 18:48:06 +0800 Subject: [PATCH 05/37] coverage: s3 and stop and start taosd --- tests/army/enterprise/s3/s3_basic.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index e7bc188ca5..58dbd12d7b 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -81,11 +81,15 @@ class TDTestCase(TBase): cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 - while len(eos.runRetList(cmd)) > 0 and loop < 40: + while len(eos.runRetList(cmd)) > 0 and loop < 100: time.sleep(5) self.trimDb(True) loop += 1 tdLog.info(f"loop={loop} wait 5s...") + if loop == 4: + sc.dnodeStop(1) + time.sleep(2) + sc.dnodeStart(1) def checkStreamCorrect(self): sql = f"select count(*) from {self.db}.stm1" From db03fbd8960f98011c9ac7d2ac73a37917f2d612 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 20 Feb 2024 19:11:21 +0800 Subject: [PATCH 06/37] add test case --- source/util/src/tskiplist.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index eadd9a2413..f90509bec1 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -21,7 +21,7 @@ #include "tutil.h" static int32_t initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); +static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); @@ -131,12 +131,14 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { return pNode; } +#ifdef BUILD_NO_CALL + void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate) { SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0}; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; bool hasDup = false; - char *pKey = NULL; - char *pDataKey = NULL; + char * pKey = NULL; + char * pDataKey = NULL; int32_t compare = 0; tSkipListWLock(pSkipList); @@ -260,6 +262,7 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { tSkipListCorrectLevel(pSkipList); tSkipListUnlock(pSkipList); } +#endif SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { if (pSkipList == NULL) return NULL; @@ -350,6 +353,7 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) { return NULL; } +#ifdef BUILD_NO_CALL void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { return; @@ -358,7 +362,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); int32_t id = 1; - char *prev = NULL; + char * prev = NULL; while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); @@ -392,6 +396,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1); } } +#endif static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) { for (int32_t i = 0; i < pNode->level; ++i) { @@ -460,7 +465,7 @@ static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) { static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) { int32_t compare = 0; bool hasDupKey = false; - char *pDataKey = pSkipList->keyFn(pData); + char * pDataKey = pSkipList->keyFn(pData); if (pSkipList->size == 0) { for (int32_t i = 0; i < pSkipList->maxLevel; i++) { @@ -516,6 +521,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, return hasDupKey; } +#ifdef BUILD_NO_CALL static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { int32_t level = pNode->level; uint8_t dupMode = SL_DUP_MODE(pSkipList); @@ -540,6 +546,7 @@ static void tSkipListCorrectLevel(SSkipList *pSkipList) { pSkipList->level -= 1; } } +#endif UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level From c7932675ce271bb98506cea5a1be7600c0583304 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 21 Feb 2024 09:07:38 +0800 Subject: [PATCH 07/37] enh: get machine id --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 7 +------ source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 11 +++++++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a3a29f6f77..fa59f56496 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "dmInt.h" #include "systable.h" -#include "tgrant.h" extern SConfig *tsCfg; @@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.memTotal = tsTotalMemoryKB * 1024; req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024; tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN); - char *machine = tGetMachineId(); - if (machine) { - tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1); - taosMemoryFreeClear(machine); - } + tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1); req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 6cbf31b15f..049b1bdf84 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -22,6 +22,7 @@ #ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" +#include "tgrant.h" #endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { @@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) { pData->rebootTime = taosGetTimestampMs(); pData->dropped = 0; pData->stopped = 0; + char *machineId = tGetMachineId(); + if (machineId) { + tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1); + taosMemoryFreeClear(machineId); + } else { +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) + terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; + return -1; +#endif + } pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pData->dnodeHash == NULL) { diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 4769ef8538..504c405506 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -109,6 +109,7 @@ typedef struct { SMsgCb msgCb; bool validMnodeEps; int64_t ipWhiteVer; + char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeData; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6974c6b0e8..c65c228224 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -141,7 +141,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN); taosMemoryFreeClear(machineId); } else { -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; goto _OVER; #endif From e91586c484a305992ab1c8dcead539ce2fa3a485 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 21 Feb 2024 09:47:42 +0800 Subject: [PATCH 08/37] fix: free fd to reuse the space --- source/os/src/osFile.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index bab9ba0cea..e6491639dc 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1329,7 +1329,6 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { char *data = taosMemoryMalloc(compressSize); gzFile dstFp = NULL; - TdFilePtr pFile = NULL; TdFilePtr pSrcFile = NULL; pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ | TD_FILE_STREAM); @@ -1369,8 +1368,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { } cmp_end: - if (pFile) { - taosCloseFile(&pFile); + if (fd >= 0) { + close(fd); } if (pSrcFile) { taosCloseFile(&pSrcFile); From 6b331594d9ff7151b73b8d9cdb6153765f120f5f Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 21 Feb 2024 13:38:49 +0800 Subject: [PATCH 09/37] fix: change typo error --- source/libs/nodes/src/nodesCodeFuncs.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index a9907295b9..689886c366 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -698,7 +698,7 @@ static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx"; static const char* jkScanLogicPlanFilesetDelimited = "FilesetDelimited"; -static const char* jkScanLogicPlanparaTablesSort = "paraTablesSort"; +static const char* jkScanLogicPlanParaTablesSort = "ParaTablesSort"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -747,7 +747,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddBoolToObject(pJson, jkScanLogicPlanFilesetDelimited, pNode->filesetDelimited); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkScanLogicPlanFilesetDelimited, pNode->paraTablesSort); + code = tjsonAddBoolToObject(pJson, jkScanLogicPlanParaTablesSort, pNode->paraTablesSort); } return code; } @@ -800,7 +800,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->filesetDelimited); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->paraTablesSort); + code = tjsonGetBoolValue(pJson, jkScanLogicPlanParaTablesSort, &pNode->paraTablesSort); } return code; } @@ -1895,7 +1895,7 @@ static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable"; -static const char* jkTableScanPhysiPlanparaTablesSort = "paraTablesSort"; +static const char* jkTableScanPhysiPlanParaTablesSort = "ParaTablesSort"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1971,7 +1971,7 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, pNode->needCountEmptyTable); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanparaTablesSort, pNode->paraTablesSort); + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanParaTablesSort, pNode->paraTablesSort); } return code; } @@ -2050,7 +2050,7 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, &pNode->needCountEmptyTable); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanparaTablesSort, &pNode->paraTablesSort); + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanParaTablesSort, &pNode->paraTablesSort); } return code; } From 35aed7653955340e9457b47e09e60753a944d5dd Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 21 Feb 2024 13:39:15 +0800 Subject: [PATCH 10/37] docs: add para_tables_sort() hint --- docs/en/12-taos-sql/06-select.md | 6 ++++-- docs/zh/12-taos-sql/06-select.md | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 1fc6fb7e67..0cd2d06989 100755 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */ hint: - BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP + BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT select_list: select_expr [, select_expr] ... @@ -87,12 +87,13 @@ Hints are a means of user control over query optimization for individual stateme The list of currently supported Hints is as follows: -| **Hint** | **Params** | **Comment** | **Scopt** | +| **Hint** | **Params** | **Comment** | **Scope** | | :-----------: | -------------- | -------------------------- | -----------------------------------| | BATCH_SCAN | None | Batch table scan | JOIN statment for stable | | NO_BATCH_SCAN | None | Sequential table scan | JOIN statment for stable | | SORT_FOR_GROUP| None | Use sort for partition, conflict with PARTITION_FIRST | With normal column in partition by list | | PARTITION_FIRST| None | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list | +| PARA_TABLES_SORT| None | When sorting the supertable rows by timestamp, No temporary disk space is used | Sorting the supertable rows by timestamp | For example: @@ -100,6 +101,7 @@ For example: SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts; SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1; +SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts; ``` ## Lists diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 2a7dff6f7d..43b32efe32 100755 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */ hint: - BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP + BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT select_list: select_expr [, select_expr] ... @@ -93,13 +93,14 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适 | NO_BATCH_SCAN | 无 | 采用顺序读表的方式 | 超级表 JOIN 语句 | | SORT_FOR_GROUP| 无 | 采用sort方式进行分组, 与PARTITION_FIRST冲突 | partition by 列表有普通列时 | | PARTITION_FIRST| 无 | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时 | - +| PARA_TABLES_SORT| 无 | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存 | 超级表的数据按时间戳排序时 | 举例： ```sql SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts; SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1; +SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts; ``` ## 列表 From d0b26bf0370703d4d7ff3d9258b64de8bd26ae72 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 21 Feb 2024 14:07:46 +0800 Subject: [PATCH 11/37] add coveriry --- source/libs/index/test/CMakeLists.txt | 4 ++ source/libs/index/test/fstTest.cc | 53 +++++++++++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 1ee835573b..eff74f93d2 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -175,4 +175,8 @@ IF(NOT TD_DARWIN) NAME idxFstUT COMMAND idxFstUT ) + add_test( + NAME idxFstTest + COMMAND idxFstTest + ) ENDIF () diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index b889a2209a..2b1744c28d 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -1,4 +1,5 @@ +#include #include #include #include @@ -14,6 +15,12 @@ #include "tutil.h" void* callback(void* s) { return s; } +class FstEnv : public ::testing::Test { + protected: + virtual void SetUp() {} + virtual void TearDown() {} +}; + static std::string fileName = TD_TMP_DIR_PATH "tindex.tindex"; class FstWriter { public: @@ -154,7 +161,7 @@ class FstReadMemory { int32_t _size; }; -#define L 100 +#define L 200 #define M 100 #define N 100 @@ -637,23 +644,31 @@ void iterTFileReader(char* path, char* uid, char* colName, char* ver) { tfileIteratorDestroy(iter); } -int main(int argc, char* argv[]) { - // tool to check all kind of fst test - // if (argc > 1) { validateTFile(argv[1]); } - // if (argc > 4) { - // path suid colName ver - // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); - //} - checkFstCheckIterator1(); - // checkFstCheckIterator2(); - // checkFstCheckIteratorPrefix(); - // checkFstCheckIteratorRange1(); - // checkFstCheckIteratorRange2(); - // checkFstCheckIteratorRange3(); - // checkFstLongTerm(); - // checkFstPrefixSearch(); +// int main(int argc, char* argv[]) { +// // tool to check all kind of fst test +// // if (argc > 1) { validateTFile(argv[1]); } +// // if (argc > 4) { +// // path suid colName ver +// // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); +// //} +// checkFstCheckIterator1(); +// // checkFstCheckIterator2(); +// // checkFstCheckIteratorPrefix(); +// // checkFstCheckIteratorRange1(); +// // checkFstCheckIteratorRange2(); +// // checkFstCheckIteratorRange3(); +// // checkFstLongTerm(); +// // checkFstPrefixSearch(); - // checkMillonWriteAndReadOfFst(); +// // checkMillonWriteAndReadOfFst(); - return 1; -} +// return 1; +// } +TEST_F(FstEnv, checkIterator1) { checkFstCheckIterator1(); } +TEST_F(FstEnv, checkItertor2) { checkFstCheckIterator2(); } +TEST_F(FstEnv, checkPrefix) { checkFstCheckIteratorPrefix(); } +TEST_F(FstEnv, checkRange1) { checkFstCheckIteratorRange1(); } +TEST_F(FstEnv, checkRange2) { checkFstCheckIteratorRange2(); } +TEST_F(FstEnv, checkRange3) { checkFstCheckIteratorRange3(); } +TEST_F(FstEnv, checkLongTerm) { checkFstLongTerm(); } +TEST_F(FstEnv, checkMillonWriteData) { checkMillonWriteAndReadOfFst(); } From d28a450882e322e6efec1f6e532208898a78cca2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 21 Feb 2024 14:39:41 +0800 Subject: [PATCH 12/37] add coveriry --- source/libs/index/test/fstTest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 2b1744c28d..adf783d2eb 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -207,7 +207,7 @@ void checkMillonWriteAndReadOfFst() { FstWriter* fw = new FstWriter; Performance_fstWriteRecords(fw); delete fw; - FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); + FstReadMemory* fr = new FstReadMemory(1024 * 8 * 1024); if (fr->init()) { printf("success to init fst read"); From 705ae7d8274be5e74314ca4ad8ef70a4a54473e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 09:04:57 +0800 Subject: [PATCH 13/37] test(stream): init mnode lock. --- source/dnode/mnode/impl/test/stream/stream.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 369053c7bb..1e163d1c8e 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -189,6 +189,7 @@ TEST_F(StreamTest, kill_checkpoint_trans) { opt.pWal = pMnode->pWal; pMnode->pSdb = sdbInit(&opt); + taosThreadMutexInit(&pMnode->syncMgmt.lock, NULL); } SVgroupChangeInfo info; From 4c258d7187c9e54b23ccd4a221f3e122f8d59d5b Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 22 Feb 2024 09:27:53 +0800 Subject: [PATCH 14/37] docs: add implication of para_tables_sort --- docs/en/12-taos-sql/06-select.md | 2 +- docs/zh/12-taos-sql/06-select.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 0cd2d06989..a2e6bca46c 100755 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -93,7 +93,7 @@ The list of currently supported Hints is as follows: | NO_BATCH_SCAN | None | Sequential table scan | JOIN statment for stable | | SORT_FOR_GROUP| None | Use sort for partition, conflict with PARTITION_FIRST | With normal column in partition by list | | PARTITION_FIRST| None | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list | -| PARA_TABLES_SORT| None | When sorting the supertable rows by timestamp, No temporary disk space is used | Sorting the supertable rows by timestamp | +| PARA_TABLES_SORT| None | When sorting the supertable rows by timestamp, No temporary disk space is used. When there are numerous tables, each with long rows, the corresponding algorithm associated with this prompt may consume a substantial amount of memory, potentially leading to an Out Of Memory (OOM) situation. | Sorting the supertable rows by timestamp | For example: diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 43b32efe32..eec947ea23 100755 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -93,7 +93,7 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适 | NO_BATCH_SCAN | 无 | 采用顺序读表的方式 | 超级表 JOIN 语句 | | SORT_FOR_GROUP| 无 | 采用sort方式进行分组, 与PARTITION_FIRST冲突 | partition by 列表有普通列时 | | PARTITION_FIRST| 无 | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时 | -| PARA_TABLES_SORT| 无 | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存 | 超级表的数据按时间戳排序时 | +| PARA_TABLES_SORT| 无 | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存。当子表数量多, 行长比较大时候, 会使用大量内存, 可能发生OOM | 超级表的数据按时间戳排序时 | 举例： ```sql From 5c817d101a6ef510788fac05d1a181da01e18d13 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 09:29:40 +0800 Subject: [PATCH 15/37] test(stream): add print info. --- source/dnode/mnode/impl/test/stream/stream.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 1e163d1c8e..c4449f613b 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -222,6 +222,8 @@ TEST_F(StreamTest, kill_checkpoint_trans) { taosArrayPush(stream.tasks, &pLevel); mndCreateStreamResetStatusTrans(pMnode, &stream); + std::cout<<"finish create reset status trans"<< std::endl; + tFreeStreamObj(&stream); sdbCleanup(pMnode->pSdb); taosMemoryFree(pMnode); From 637e6043682a6797d2eb4ea1051df72a4c035043 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 09:38:44 +0800 Subject: [PATCH 16/37] test(stream): add print info. --- source/dnode/mnode/impl/test/stream/stream.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index c4449f613b..fdc108cbe3 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -222,14 +222,21 @@ TEST_F(StreamTest, kill_checkpoint_trans) { taosArrayPush(stream.tasks, &pLevel); mndCreateStreamResetStatusTrans(pMnode, &stream); - std::cout<<"finish create reset status trans"<< std::endl; + std::cout << "finish create reset status trans" << std::endl; tFreeStreamObj(&stream); + + std::cout << "end1" << std::endl; sdbCleanup(pMnode->pSdb); + + std::cout << "end2" << std::endl; + taosMemoryFree(pMnode); taosArrayDestroy(info.pUpdateNodeList); taosHashCleanup(info.pDBMap); + + std::cout << "end3" << std::endl; } TEST_F(StreamTest, plan_Test) { From 045a30035f28c9ed5a638e44ba4b22adcaf51638 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 09:52:47 +0800 Subject: [PATCH 17/37] test(stream): init lock. --- source/dnode/mnode/impl/test/stream/stream.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index fdc108cbe3..7dfb944d55 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -216,6 +216,7 @@ TEST_F(StreamTest, kill_checkpoint_trans) { pTask->id.streamId = defStreamId; pTask->id.taskId = 1; pTask->exec.qmsg = (char*)taosMemoryMalloc(1); + taosThreadMutexInit(&pTask->lock, NULL); taosArrayPush(pLevel, &pTask); From dbd3b4e2b92334bd86df363f7b3fd9c59b859340 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 10:02:25 +0800 Subject: [PATCH 18/37] test: remove print info. --- source/dnode/mnode/impl/test/stream/stream.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 7dfb944d55..8d106b1ede 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -223,21 +223,13 @@ TEST_F(StreamTest, kill_checkpoint_trans) { taosArrayPush(stream.tasks, &pLevel); mndCreateStreamResetStatusTrans(pMnode, &stream); - std::cout << "finish create reset status trans" << std::endl; - tFreeStreamObj(&stream); - - std::cout << "end1" << std::endl; sdbCleanup(pMnode->pSdb); - std::cout << "end2" << std::endl; - taosMemoryFree(pMnode); taosArrayDestroy(info.pUpdateNodeList); taosHashCleanup(info.pDBMap); - - std::cout << "end3" << std::endl; } TEST_F(StreamTest, plan_Test) { From 6e13e4aa638dacfc34e6652355fa9ecf2deb5ed1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 10:58:32 +0800 Subject: [PATCH 19/37] fix:add excluded msg for meta in tmq --- include/common/tmsg.h | 16 ++++----- source/client/src/clientRawBlockWrite.c | 41 ++++++++++++++++++----- source/client/src/clientTmq.c | 2 +- source/common/src/tmsg.c | 22 ++++++++---- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndStb.c | 10 ++++-- source/dnode/vnode/src/sma/smaTimeRange.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 4 +-- source/dnode/vnode/src/tq/tqSink.c | 4 +-- source/dnode/vnode/src/tq/tqUtil.c | 29 ++++++++++++++++ source/libs/parser/src/parInsertUtil.c | 3 +- 12 files changed, 100 insertions(+), 36 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1b438ba026..587e2f9f3e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -207,9 +207,6 @@ typedef enum _mgmt_table { #define TD_CHILD_TABLE TSDB_CHILD_TABLE #define TD_NORMAL_TABLE TSDB_NORMAL_TABLE -#define TD_REQ_FROM_APP 0 -#define TD_REQ_FROM_TAOX 1 - typedef enum ENodeType { // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN, // VALUE, OPERATOR, FUNCTION and so on. @@ -759,7 +756,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; - int8_t source; // 1-taosX or 0-taosClient + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t reserved[6]; tb_uid_t suid; int64_t delay1; @@ -802,7 +799,7 @@ void tFreeSMCreateStbRsp(SMCreateStbRsp* pRsp); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; - int8_t source; // 1-taosX or 0-taosClient + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t reserved[6]; tb_uid_t suid; int32_t sqlLen; @@ -2661,6 +2658,7 @@ typedef struct SVCreateStbReq { SRSmaParam rsmaParam; int32_t alterOriDataLen; void* alterOriData; + int8_t source; } SVCreateStbReq; int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); @@ -2730,6 +2728,7 @@ typedef struct { SVCreateTbReq* pReqs; SArray* pArray; }; + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient } SVCreateTbBatchReq; int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq); @@ -2822,6 +2821,7 @@ typedef struct { int32_t newCommentLen; char* newComment; int64_t ctimeMs; // fill by vnode + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient } SVAlterTbReq; int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); @@ -3919,12 +3919,13 @@ int32_t tDeatroySMqHbRsp(SMqHbRsp* pRsp); int32_t tSerializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); +#define TD_REQ_FROM_APP 0x0 #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 #define SUBMIT_REQ_FROM_FILE 0x4 +#define TD_REQ_FROM_TAOX 0x8 -#define SOURCE_NULL 0 -#define SOURCE_TAOSX 1 +#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility typedef struct { int32_t flags; @@ -3937,7 +3938,6 @@ typedef struct { SArray* aCol; }; int64_t ctimeMs; - int8_t source; } SSubmitTbData; typedef struct { diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 448243cc3d..f143624bab 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1001,6 +1001,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); taosArrayPush(tBatch.req.pArray, pCreateReq); + tBatch.req.source = TD_REQ_FROM_TAOX; taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); } else { // add to the correct vgroup @@ -1276,7 +1277,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { return terrno; } SVAlterTbReq req = {0}; - SDecoder coder = {0}; + SDecoder dcoder = {0}; int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; @@ -1297,8 +1298,8 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { // decode and process req void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeSVAlterTbReq(&coder, &req) < 0) { + tDecoderInit(&dcoder, data, len); + if (tDecodeSVAlterTbReq(&dcoder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; goto end; } @@ -1340,14 +1341,36 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } pVgData->vg = pInfo; - pVgData->pData = taosMemoryMalloc(metaLen); - if (NULL == pVgData->pData) { + + int tlen = 0; + req.source = TD_REQ_FROM_TAOX; + tEncodeSize(tEncodeSVAlterTbReq, &req, tlen, code); + if(code != 0){ code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - memcpy(pVgData->pData, meta, metaLen); - ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId); - pVgData->size = metaLen; + tlen += sizeof(SMsgHead); + void* pMsg = taosMemoryMalloc(tlen); + if (NULL == pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + ((SMsgHead*)pMsg)->vgId = htonl(pInfo.vgId); + ((SMsgHead*)pMsg)->contLen = htonl(tlen); + void* pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead)); + SEncoder coder = {0}; + tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead)); + code = tEncodeSVAlterTbReq(&coder, &req); + if(code != 0){ + tEncoderClear(&coder); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tEncoderClear(&coder); + + pVgData->pData = pMsg; + pVgData->size = tlen; + pVgData->numOfTables = 1; taosArrayPush(pArray, &pVgData); @@ -1387,7 +1410,7 @@ end: if (pVgData) taosMemoryFreeClear(pVgData->pData); taosMemoryFreeClear(pVgData); destroyRequest(pRequest); - tDecoderClear(&coder); + tDecoderClear(&dcoder); qDestroyQuery(pQuery); terrno = code; return code; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 1904874a0b..b141921a77 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -389,7 +389,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } if (strcasecmp(key, "msg.consume.excluded") == 0) { - conf->sourceExcluded = taosStr2int64(value); + conf->sourceExcluded = (taosStr2int64(value) != 0) ? TD_REQ_FROM_TAOX : 0; return TMQ_CONF_OK; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2ffa12f2c1..85f5d462c7 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7513,6 +7513,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { if (pReq->alterOriDataLen > 0) { if (tEncodeBinary(pCoder, pReq->alterOriData, pReq->alterOriDataLen) < 0) return -1; } + if (tEncodeI8(pCoder, pReq->source) < 0) return -1; tEndEncode(pCoder); return 0; @@ -7535,6 +7536,10 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (tDecodeBinary(pCoder, (uint8_t **)&pReq->alterOriData, NULL) < 0) return -1; } + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI8(pCoder, &pReq->source) < 0) return -1; + } + tEndDecode(pCoder); return 0; } @@ -7663,6 +7668,8 @@ int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq) if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1; } + if (tEncodeI8(pCoder, pReq->source) < 0) return -1; + tEndEncode(pCoder); return 0; } @@ -7677,6 +7684,10 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) { if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1; } + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI8(pCoder, &pReq->source) < 0) return -1; + } + tEndDecode(pCoder); return 0; } @@ -8034,6 +8045,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { break; } if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->source) < 0) return -1; tEndEncode(pEncoder); return 0; @@ -8094,6 +8106,9 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { if (!tDecodeIsEnd(pDecoder)) { if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1; } + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI8(pDecoder, &pReq->source) < 0) return -1; + } tEndDecode(pDecoder); return 0; @@ -8670,7 +8685,6 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm } } if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1; - if (tEncodeI8(pCoder, pSubmitTbData->source) < 0) return -1; tEndEncode(pCoder); return 0; @@ -8758,12 +8772,6 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa goto _exit; } } - if (!tDecodeIsEnd(pCoder)) { - if (tDecodeI8(pCoder, &pSubmitTbData->source) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _exit; - } - } tEndDecode(pCoder); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index fc58fabad5..4a092057ce 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -464,6 +464,7 @@ typedef struct { char* pAst1; char* pAst2; SRWLatch lock; + int8_t source; } SStbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 1ddd2f34e6..aad1dc881b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -458,6 +458,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3 req.rollup = pStb->ast1Len > 0 ? 1 : 0; req.alterOriData = alterOriData; req.alterOriDataLen = alterOriDataLen; + req.source = pStb->source; // todo req.schemaRow.nCols = pStb->numOfColumns; req.schemaRow.version = pStb->colVer; @@ -774,7 +775,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->createdTime = taosGetTimestampMs(); pDst->updateTime = pDst->createdTime; pDst->uid = - (pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX) + ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); pDst->dbUid = pDb->uid; pDst->tagVer = 1; pDst->colVer = 1; @@ -790,6 +792,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->numOfFuncs = pCreate->numOfFuncs; pDst->commentLen = pCreate->commentLen; pDst->pFuncs = pCreate->pFuncs; + pDst->source = pCreate->source; pCreate->pFuncs = NULL; if (pDst->commentLen > 0) { @@ -1033,6 +1036,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq memcpy(pDst, pStb, sizeof(SStbObj)); taosRUnLockLatch(&pStb->lock); + pDst->source = createReq->source; pDst->updateTime = taosGetTimestampMs(); pDst->numOfColumns = createReq->numOfColumns; pDst->numOfTags = createReq->numOfTags; @@ -1141,7 +1145,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { goto _OVER; - } else if (createReq.source == TD_REQ_FROM_TAOX && (createReq.tagVer != 1 || createReq.colVer != 1)) { + } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) { mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name); code = 0; goto _OVER; @@ -2572,7 +2576,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { } } - if (dropReq.source == TD_REQ_FROM_TAOX && pStb->uid != dropReq.suid) { + if ((dropReq.source == TD_REQ_FROM_TAOX_OLD || dropReq.source == TD_REQ_FROM_TAOX) && pStb->uid != dropReq.suid) { code = 0; goto _OVER; } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 54d859992c..fb898c02f8 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -193,7 +193,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * continue; } - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE, .source = SOURCE_NULL}; + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE}; int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1; tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 727157a2f8..7a737fbb5d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -392,7 +392,7 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { pReader->msg.ver); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if ((pSubmitTbData->source & sourceExcluded) != 0) { + if ((pSubmitTbData->flags & sourceExcluded) != 0) { pReader->nextBlk += 1; continue; } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index f108cedcf5..103007eb57 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -267,7 +267,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table; } - if ((pSubmitTbDataRet->source & sourceExcluded) != 0) { + if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_table; } if (pRsp->withTbName) { @@ -335,7 +335,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db; } - if ((pSubmitTbDataRet->source & sourceExcluded) != 0) { + if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_db; } if (pRsp->withTbName) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b56bf3e0fe..42bd1bc59c 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -815,7 +815,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { return; } - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL}; + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { continue; @@ -860,7 +860,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { pTask->execInfo.sink.numOfBlocks += 1; uint64_t groupId = pDataBlock->info.id.groupId; - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL}; + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId)); if (index == NULL) { // no data yet, append it diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 31267dbf52..dad1211326 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -171,6 +171,22 @@ end : { } } +#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \ + SDecoder decoder = {0};\ + TYPE req = {0}; \ + void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \ + int32_t len = pHead->bodyLen - sizeof(SMsgHead); \ + tDecoderInit(&decoder, data, len); \ + if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \ + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d", \ + pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \ + fetchVer++; \ + tDecoderClear(&decoder); \ + continue; \ + } \ + tDecoderClear(&decoder); + + static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { int code = 0; @@ -239,6 +255,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, goto end; } + if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) { + if (pHead->msgType == TDMT_VND_CREATE_TABLE) { + PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq) + } else if (pHead->msgType == TDMT_VND_ALTER_TABLE) { + PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq) + } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) { + PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq) + } else if (pHead->msgType == TDMT_VND_DELETE) { + fetchVer++; + continue; + } + } + tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1); metaRsp.resMsgType = pHead->msgType; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 8900ef8e7f..cd38b67fe9 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -285,7 +285,6 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { pTmp->suid = pSrc->suid; pTmp->uid = pSrc->uid; pTmp->sver = pSrc->sver; - pTmp->source = pSrc->source; pTmp->pCreateTbReq = NULL; if (pTmp->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { if (pSrc->pCreateTbReq) { @@ -653,7 +652,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } - pTableCxt->pData->source = SOURCE_TAOSX; + pTableCxt->pData->flags |= TD_REQ_FROM_TAOX; if(tmp == NULL){ ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { From c22360a529ef92e3773d3399684ff3b5a9fe2831 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 11:32:45 +0800 Subject: [PATCH 20/37] fix:add excluded msg for meta in tmq --- tests/system-test/7-tmq/tmq_taosx.py | 4 ++-- utils/test/c/tmq_taosx_ci.c | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index ac8a6f377c..c547385d70 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -323,7 +323,7 @@ class TDTestCase: tdSql.query("select * from st") tdSql.checkRows(8) - tdSql.execute(f'create topic topic_excluded with meta as database d1') + tdSql.execute(f'create topic topic_all with meta as database d1') consumer_dict = { "group.id": "g1", "td.connect.user": "root", @@ -333,7 +333,7 @@ class TDTestCase: consumer = Consumer(consumer_dict) try: - consumer.subscribe(["topic_excluded"]) + consumer.subscribe(["topic_all"]) except TmqError: tdLog.exit(f"subscribe error") diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 056b7dc6cf..2257089f06 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -20,6 +20,7 @@ #include #include "taos.h" #include "types.h" +#include "tmsg.h" static int running = 1; TdFilePtr g_fp = NULL; @@ -966,7 +967,14 @@ void testConsumeExcluded(int topic_type){ tmq_raw_data raw = {0}; tmq_get_raw(msg, &raw); if(topic_type == 1){ - assert(raw.raw_type != 2 && raw.raw_type != 4); + assert(raw.raw_type != 2 && raw.raw_type != 4 && + raw.raw_type != TDMT_VND_CREATE_STB && + raw.raw_type != TDMT_VND_ALTER_STB && + raw.raw_type != TDMT_VND_CREATE_TABLE && + raw.raw_type != TDMT_VND_ALTER_TABLE && + raw.raw_type != TDMT_VND_DELETE); + assert(raw.raw_type == TDMT_VND_DROP_STB || + raw.raw_type == TDMT_VND_DROP_TABLE); }else if(topic_type == 2){ assert(0); } From 8c6ea4079c0c72918c8494d54f82f55a78454451 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 22 Feb 2024 14:15:35 +0800 Subject: [PATCH 21/37] delete invalid result --- .../executor/src/streamcountwindowoperator.c | 16 +++++++++++++--- source/libs/stream/src/streamSessionState.c | 3 +-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 080f9d4e2b..689ba54ca6 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -115,8 +115,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, } } -static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, int32_t start, int32_t rows, int32_t maxRows, - SSHashObj* pStDeleted, bool* pRebuild) { +static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { + SSessionKey key = {0}; + getSessionHashKey(pKey, &key); + tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); +} + +static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, + int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, + SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; int32_t num = 0; for (int32_t i = start; i < rows; i++) { @@ -148,6 +156,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI if (needDelState) { memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); + removeCountResult(pStUpdated, pAggSup->pResultRows, &sWinKey); if (pWinInfo->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &sWinKey); } @@ -242,7 +251,8 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl setSessionWinOutputInfo(pStUpdated, &curWin.winInfo); slidingRows = *curWin.pWindowCount; if (!buffInfo.rebuildWindow) { - winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStDeleted, &buffInfo.rebuildWindow); + winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated, + pStDeleted, &buffInfo.rebuildWindow); } if (buffInfo.rebuildWindow) { SSessionKey range = {0}; diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index bd28d2bca9..e2aae130e5 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -736,6 +736,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C void* pRockVal = NULL; SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId); code = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &pRockVal, pVLen); + streamStateFreeCur(pCur); if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code); if (code == TSDB_CODE_SUCCESS) { @@ -743,7 +744,6 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)) ); if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); - streamStateFreeCur(pCur); goto _end; } } @@ -751,7 +751,6 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C pWinKey->win.ekey = endTs; (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL); taosMemoryFree(pRockVal); - streamStateFreeCur(pCur); } else { (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = TSDB_CODE_FAILED; From 7c8eb21ee541cf73d632e2e97ecaf0b8e6122407 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 15:01:04 +0800 Subject: [PATCH 22/37] fix(stream): halt the correct task. --- source/dnode/mnode/impl/src/mndScheduler.c | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index cbc0ace75d..ef9a7205e1 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -44,9 +44,8 @@ static bool hasCountWindowNode(SPhysiNode* pNode) { } } -static bool countWindowStreamTask(SSubplan* pPlan) { - SPhysiNode* pNode = pPlan->pNode; - return hasCountWindowNode(pNode); +static bool isCountWindowStreamTask(SSubplan* pPlan) { + return hasCountWindowNode((SPhysiNode*)pPlan->pNode); } int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, @@ -342,13 +341,13 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe } } -static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan) { - bool hasCountWindowNode = countWindowStreamTask(pPlan); - bool isRelStreamTask = (pTask->hTaskInfo.id.taskId != 0); - if (hasCountWindowNode && isRelStreamTask) { +static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFillhistoryTask) { + bool hasCountWindowNode = isCountWindowStreamTask(pPlan); + + if (hasCountWindowNode && (!isFillhistoryTask)) { SStreamStatus* pStatus = &pTask->status; - mDebug("s-task:0x%x status is set to %s from %s for count window agg task with fill-history option set", - pTask->id.taskId, streamTaskGetStatusStr(pStatus->taskStatus), streamTaskGetStatusStr(TASK_STATUS__HALT)); + mDebug("s-task:0x%x status set %s from %s for count window agg task with fill-history option set", + pTask->id.taskId, streamTaskGetStatusStr(TASK_STATUS__HALT), streamTaskGetStatusStr(pStatus->taskStatus)); pStatus->taskStatus = TASK_STATUS__HALT; } } @@ -398,15 +397,17 @@ static void setHTasksId(SStreamObj* pStream) { static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey, SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) { - // new stream task SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } + mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); - haltInitialTaskStatus(pTask, plan); + if (pStream->conf.fillHistory) { + haltInitialTaskStatus(pTask, plan, isFillhistory); + } streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId); @@ -415,6 +416,7 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre terrno = code; return terrno; } + return TDB_CODE_SUCCESS; } From c859213ce54a9faba038ca89ddbb0a6e90d7b89d Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 22 Feb 2024 15:34:12 +0800 Subject: [PATCH 23/37] fix: free datablocks that is skipped decided by firt row ts --- source/libs/executor/src/tsort.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index fde5f22aaa..9ff903cdb9 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1125,8 +1125,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { int64_t firstRowTs = *(int64_t*)tsCol->pData; if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { - continue; - } + if (bExtractedBlock) { + blockDataDestroy(pBlk); + } + continue; + } } if (pBlk != NULL) { @@ -1149,10 +1152,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { tSimpleHashClear(mUidBlk); code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); if (code != TSDB_CODE_SUCCESS) { - tSimpleHashCleanup(mUidBlk); - taosArrayDestroy(aBlkSort); - taosArrayDestroy(aExtSrc); - return code; + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } + taosArrayClear(aBlkSort); + break; } int64_t el = taosGetTimestampUs() - p; @@ -1165,6 +1169,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { szSort = 0; qDebug("source %zu created", taosArrayGetSize(aExtSrc)); } + if (pBlk == NULL) { break; } @@ -1180,6 +1185,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { } tSimpleHashCleanup(mUidBlk); + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } taosArrayDestroy(aBlkSort); tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); if (!tsortIsClosed(pHandle)) { @@ -1188,7 +1196,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { taosArrayDestroy(aExtSrc); tSimpleHashCleanup(mTableNumRows); pHandle->type = SORT_SINGLESOURCE_SORT; - return TSDB_CODE_SUCCESS; + return code; } static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { From 4037a7a1a5070666b5bb312e995a10ce90b3b6bb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 15:44:50 +0800 Subject: [PATCH 24/37] fix:change datablock to old version for compatibility --- include/common/tdatablock.h | 5 ++++- source/client/src/clientImpl.c | 10 +++++----- source/client/src/clientMsgHandler.c | 4 ++-- source/common/src/tdatablock.c | 10 ++++++---- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/parser/src/parInsertUtil.c | 4 ++-- source/libs/stream/src/streamDispatch.c | 4 ++-- 13 files changed, 28 insertions(+), 23 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index bcb40f4175..e305f4c2ec 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -32,6 +32,9 @@ typedef struct SBlockOrderInfo { SColumnInfoData* pColData; } SBlockOrderInfo; +#define BLOCK_VERSION_1 1 +#define BLOCK_VERSION_2 2 + #define NBIT (3u) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) #define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) @@ -253,7 +256,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); int32_t blockGetEncodeSize(const SSDataBlock* pBlock); -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion); const char* blockDecode(SSDataBlock* pBlock, const char* pData); // for debug diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 9800d233e9..a8ebef763a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1811,7 +1811,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i char* pStart = p + len; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; + int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i]; if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; @@ -1910,8 +1910,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* pStart = p; char* pStart1 = p1; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; - int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i]; + int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i]; + int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i]; if (ASSERT(colLen < dataLen)) { tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -1970,7 +1970,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int } colLen1 = len; totalLen += colLen1; - colLength1[i] = (blockVersion == 1) ? htonl(len) : len; + colLength1[i] = (blockVersion == BLOCK_VERSION_1) ? htonl(len) : len; } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { len = numOfRows * sizeof(int32_t); memcpy(pStart1, pStart, len); @@ -2059,7 +2059,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { - if(blockVersion == 1){ + if(blockVersion == BLOCK_VERSION_1){ colLength[i] = htonl(colLength[i]); } if (colLength[i] >= dataLen) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e0cedb9924..3aa46d135e 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -499,7 +499,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS, BLOCK_VERSION_1); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { @@ -611,7 +611,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS, BLOCK_VERSION_1); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5382259899..c6f004f83b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2191,12 +2191,12 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha return TSDB_CODE_SUCCESS; } -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion) { int32_t dataLen = 0; // todo extract method int32_t* version = (int32_t*)data; - *version = 2; + *version = bVersion; data += sizeof(int32_t); int32_t* actualLen = (int32_t*)data; @@ -2277,7 +2277,9 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { data += colSizes[col]; } -// colSizes[col] = htonl(colSizes[col]); + if(bVersion == BLOCK_VERSION_1){ + colSizes[col] = htonl(colSizes[col]); + } // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, // htonl(colSizes[col]), colSizes[col]); } @@ -2338,7 +2340,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - if(version == 1){ + if(version == BLOCK_VERSION_1){ colLen[i] = htonl(colLen[i]); } ASSERT(colLen[i] >= 0); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 14853009e0..b36dc71322 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -401,7 +401,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, numOfCols); + int32_t len = blockEncode(pBlock, pStart, numOfCols, BLOCK_VERSION_1); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 8e7e72aa0e..e46c057129 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -328,7 +328,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 1ddd2f34e6..ed1192c490 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -3150,7 +3150,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // pStart += sizeof(SSysTableSchema); // } // -// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); +// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); // } // // pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 5432637482..4e4e1481f9 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -28,7 +28,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_2); actualLen += sizeof(SRetrieveTableRspForTmq); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6bae0e1022..e74c735497 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -40,7 +40,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols, BLOCK_VERSION_1); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 66b50bcb47..d2d14bb9f3 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1817,7 +1817,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock), BLOCK_VERSION_1); rsp->compLen = htonl(len); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index abe566473f..fb57d6b531 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols, BLOCK_VERSION_1); // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 6b655bfae6..514f2ca64e 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -718,7 +718,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == 1) { + if (needChangeLength && version == BLOCK_VERSION_1) { pStart += htonl(colLength[j]); } else { pStart += colLength[j]; @@ -749,7 +749,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == 1) { + if (needChangeLength && version == BLOCK_VERSION_1) { pStart += htonl(colLength[i]); } else { pStart += colLength[i]; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index b51845d152..080c2c9556 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -188,7 +188,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); SStreamRetrieveReq req = { .streamId = pTask->id.streamId, @@ -772,7 +772,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From da0265ddde86bdb0136ab5ba14ff1ee4a39f8e26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 15:52:47 +0800 Subject: [PATCH 25/37] fix(tsdb): set correct error code info extract method. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 86f58717e2..90a26d17dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2543,7 +2543,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr while (1) { // only check here, since the iterate data in memory is very fast. if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } @@ -2694,7 +2694,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } @@ -2909,7 +2909,7 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_ while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } @@ -2951,7 +2951,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } From ac36dd46d5e5f4b27b5639b6bc0bffa29afcaa78 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 22 Feb 2024 16:32:09 +0800 Subject: [PATCH 26/37] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/tms_memleak.py | 51 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/system-test/2-query/tms_memleak.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cc15df33a1..12f1f62f63 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -49,6 +49,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tms_memleak.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3 diff --git a/tests/system-test/2-query/tms_memleak.py b/tests/system-test/2-query/tms_memleak.py new file mode 100644 index 0000000000..0d5cdd8272 --- /dev/null +++ b/tests/system-test/2-query/tms_memleak.py @@ -0,0 +1,51 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-] + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tms_memleak") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tms_memleak") + tdSql.execute("create database if not exists tms_memleak") + tdSql.execute('use tms_memleak') + + tdSql.execute('create table st(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)('2021-04-19 00:00:02', 2)('2021-04-19 00:00:03', 3)('2021-04-19 00:00:04', 4)") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-20 00:00:01', 5)('2021-04-20 00:00:02', 6)('2021-04-20 00:00:03', 7)('2021-04-20 00:00:04', 8)") + + tdSql.execute("insert into ct3 using st tags(3) values('2021-04-21 00:00:01', 5)('2021-04-21 00:00:02', 6)('2021-04-21 00:00:03', 7)('2021-04-21 00:00:04', 8)") + + tdSql.execute("insert into ct4 using st tags(4) values('2021-04-22 00:00:01', 5)('2021-04-22 00:00:02', 6)('2021-04-22 00:00:03', 7)('2021-04-22 00:00:04', 8)") + + tdSql.query("select * from st order by ts limit 1 "); + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1); + + tdSql.execute('drop database tms_memleak') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From d8361e979c50911b281ae1b6237b75ab65e79f84 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 22 Feb 2024 16:47:27 +0800 Subject: [PATCH 27/37] fix: add check s3 upload over logic --- tests/army/enterprise/s3/s3_basic.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 58dbd12d7b..5d9a52ebc4 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -81,15 +81,23 @@ class TDTestCase(TBase): cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 - while len(eos.runRetList(cmd)) > 0 and loop < 100: - time.sleep(5) + rets = [] + while loop < 500: + rets = eos.runRetList(cmd) + cnt = len(rets) + if cnt == 0: + tdLog.info("All data file upload to server over.") + break + time.sleep(3) self.trimDb(True) loop += 1 - tdLog.info(f"loop={loop} wait 5s...") + tdLog.info(f"loop={loop} data files have {cnt} wait 5s...") if loop == 4: sc.dnodeStop(1) time.sleep(2) sc.dnodeStart(1) + if len(rets) > 0: + tdLog.exit(f"s3 can not upload all data to server. data files={rets}") def checkStreamCorrect(self): sql = f"select count(*) from {self.db}.stm1" From 6ac6e8f2bd382c2e3b41dd604e43f197135a770c Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 17:46:36 +0800 Subject: [PATCH 28/37] enh: unit test and optimization for base58 --- include/util/tbase58.h | 3 ++ source/util/src/tbase58.c | 34 +++++++++--- source/util/test/CMakeLists.txt | 8 +++ source/util/test/tbaseCodecTest.cpp | 80 +++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 source/util/test/tbaseCodecTest.cpp diff --git a/include/util/tbase58.h b/include/util/tbase58.h index e1b03f8a8f..ab62e1926e 100644 --- a/include/util/tbase58.h +++ b/include/util/tbase58.h @@ -22,6 +22,9 @@ extern "C" { #endif +#define TBASE_MAX_ILEN 4096 +#define TBASE_MAX_OLEN 5653 + uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen); char *base58_encode(const uint8_t *value, int32_t vlen); diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index fa3ecadd14..e1cee72b47 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -18,24 +18,29 @@ #include #include -#define BASE_BUF_SIZE 256 +#define TBASE_BUF_SIZE 256 static const char *basis_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pb = value; const uint8_t *pe = pb + vlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; + if (vlen > TBASE_MAX_ILEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + while (pb != pe && *pb == 0) { ++pb; ++nz; } size = (pe - pb) * 69 / 50 + 1; - if (size > BASE_BUF_SIZE) { + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -47,7 +52,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { int32_t num = *pb; int32_t i = 0; for (int32_t j = (int32_t)size - 1; (num != 0 || i < len) && j >= 0; --j, ++i) { - num += ((int32_t)buf[j]) << 8; + num += ((int32_t)pbuf[j]) << 8; pbuf[j] = num % 58; num /= 58; } @@ -57,7 +62,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pi = pbuf + (size - len); while (pi != pbuf + size && *pi == 0) ++pi; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); if (!result) { terrno = TSDB_CODE_OUT_OF_MEMORY; if (bfree) taosMemoryFree(pbuf); @@ -83,11 +88,23 @@ static const signed char index_58[256] = { uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const char *pe = value + inlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; + if (inlen > TBASE_MAX_OLEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + for (int32_t i = 0; i < inlen; ++i) { + if (value[i] == 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + } + while (*value && isspace(*value)) ++value; while (*value == '1') { ++nz; @@ -95,7 +112,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { } size = (int32_t)(pe - value) * 733 / 1000 + 1; - if (size > BASE_BUF_SIZE) { + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -106,6 +123,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { while (*value && !isspace(*value)) { int32_t num = index_58[(uint8_t)*value]; if (num == -1) { + terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); return NULL; } @@ -127,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, inlen + 1); if (!result) { if (bfree) taosMemoryFree(pbuf); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index f4f3880388..5e32bd82b7 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -100,3 +100,11 @@ add_test( NAME talgoTest COMMAND talgoTest ) + +# tbaseCodecTest +add_executable(tbaseCodecTest "tbaseCodecTest.cpp") +target_link_libraries(tbaseCodecTest os util common gtest_main) +add_test( + NAME tbaseCodecTest + COMMAND tbaseCodecTest + diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp new file mode 100644 index 0000000000..70639e7ec9 --- /dev/null +++ b/source/util/test/tbaseCodecTest.cpp @@ -0,0 +1,80 @@ +#include +#include + +#include +#include "os.h" +#include "taos.h" +#include "taoserror.h" +#include "tbase58.h" +#include "tbase64.h" +#include "tglobal.h" + +using namespace std; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { + char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + ASSERT_NE(nullptr, pEnc); + + int32_t encLen = strlen(pEnc); + std::cout << "index:" << index << ", encLen is " << encLen << std::endl; + int32_t decLen = 0; + char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); + std::cout << "index:" << index << ", decLen is " << decLen << std::endl; + ASSERT_NE(nullptr, pDec); + ASSERT_EQ(rawLen, decLen); + ASSERT_LE(rawLen, encLen); + ASSERT_EQ(0, strncmp((char *)pRaw, pDec, rawLen)); + taosMemoryFreeClear(pDec); + taosMemoryFreeClear(pEnc); +} + +TEST(TD_BASE_CODEC_TEST, tbase58_test) { + const int32_t TEST_LEN_MAX = TBASE_MAX_ILEN; + const int32_t TEST_LEN_STEP = 10; + int32_t rawLen = 0; + uint8_t *pRaw = NULL; + + pRaw = (uint8_t *)taosMemoryCalloc(1, TEST_LEN_MAX); + ASSERT_NE(nullptr, pRaw); + + // 1. normal case + // string blend with char and '\0' + rawLen = TEST_LEN_MAX; + for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + checkBase58Codec(pRaw, rawLen, i); + pRaw[i] = i & 127; + } + + // string without '\0' + for (int32_t i = 0; i < TEST_LEN_MAX; ++i) { + pRaw[i] = i & 127; + } + checkBase58Codec(pRaw, TEST_LEN_MAX, 0); + for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + rawLen = i; + checkBase58Codec(pRaw, rawLen, i); + } + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); + + // 2. overflow case + char tmp[1]; + char *pEnc = base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1); + ASSERT_EQ(nullptr, pEnc); + char *pDec = (char *)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL); + ASSERT_EQ(nullptr, pDec); + + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); +} \ No newline at end of file From 604ae220f306d5a794d4082560aa3e2367f1aaf8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 17:48:33 +0800 Subject: [PATCH 29/37] fix:change datablock to old version for compatibility --- include/common/tdatablock.h | 2 +- source/client/src/clientMsgHandler.c | 4 +- source/client/src/clientTmq.c | 67 ++++++++++++++------- source/common/src/tdatablock.c | 12 ++-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/stream/src/streamDispatch.c | 4 +- 12 files changed, 62 insertions(+), 41 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index e305f4c2ec..292e9a3181 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -256,7 +256,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); int32_t blockGetEncodeSize(const SSDataBlock* pBlock); -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); // for debug diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 3aa46d135e..e0cedb9924 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -499,7 +499,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { @@ -611,7 +611,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 69681b9ae0..ebd4e1c7b6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1562,17 +1562,39 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } -SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { - SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - pRspObj->resType = RES_TYPE__TMQ; +void changeByteEndian(char* pData){ + char* p = pData; + + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + // version: + int32_t blockVersion = *(int32_t*)p; + ASSERT(blockVersion == BLOCK_VERSION_1); + *(int32_t*)p = BLOCK_VERSION_2; + + p += sizeof(int32_t); + p += sizeof(int32_t); + p += sizeof(int32_t); + int32_t cols = *(int32_t*)p; + p += sizeof(int32_t); + p += sizeof(int32_t); + p += sizeof(uint64_t); + // check fields + p += cols * (sizeof(int8_t) + sizeof(int32_t)); + + int32_t* colLength = (int32_t*)p; + for (int32_t i = 0; i < cols; ++i) { + colLength[i] = htonl(colLength[i]); + } +} + +static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) { (*numOfRows) = 0; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; - memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; @@ -1584,11 +1606,21 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, } // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); + void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i); + void* rawData = NULL; + int64_t rows = 0; + // deal with compatibility + if(*(int64_t*)pRetrieve == 0){ + rawData = ((SRetrieveTableRsp*)pRetrieve)->data; + rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows); + }else if(*(int64_t*)pRetrieve == 1){ + rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; + rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows); + } + pVg->numOfRows += rows; (*numOfRows) += rows; - + changeByteEndian(rawData); if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if(schema){ @@ -1596,29 +1628,22 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, } } } +} +SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { + SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); + pRspObj->resType = RES_TYPE__TMQ; + memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj); return pRspObj; } SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); pRspObj->resType = RES_TYPE__TMQ_METADATA; - tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - pRspObj->vgId = pWrapper->vgHandle->vgId; - pRspObj->resIter = -1; memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); - pRspObj->resInfo.totalRows = 0; - pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - - // extract the rows in this data packet - for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); - pVg->numOfRows += rows; - (*numOfRows) += rows; - } + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj); return pRspObj; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c6f004f83b..49f1f1eb0e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2191,12 +2191,12 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha return TSDB_CODE_SUCCESS; } -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { int32_t dataLen = 0; // todo extract method int32_t* version = (int32_t*)data; - *version = bVersion; + *version = BLOCK_VERSION_1; data += sizeof(int32_t); int32_t* actualLen = (int32_t*)data; @@ -2277,9 +2277,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, in data += colSizes[col]; } - if(bVersion == BLOCK_VERSION_1){ - colSizes[col] = htonl(colSizes[col]); - } + colSizes[col] = htonl(colSizes[col]); // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, // htonl(colSizes[col]), colSizes[col]); } @@ -2340,9 +2338,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - if(version == BLOCK_VERSION_1){ - colLen[i] = htonl(colLen[i]); - } + colLen[i] = htonl(colLen[i]); ASSERT(colLen[i] >= 0); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index b36dc71322..14853009e0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -401,7 +401,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, numOfCols, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, pStart, numOfCols); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index e46c057129..8e7e72aa0e 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -328,7 +328,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ed1192c490..1ddd2f34e6 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -3150,7 +3150,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // pStart += sizeof(SSysTableSchema); // } // -// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); +// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); // } // // pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 4e4e1481f9..5432637482 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -28,7 +28,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_2); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRspForTmq); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index e74c735497..6bae0e1022 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -40,7 +40,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index d2d14bb9f3..66b50bcb47 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1817,7 +1817,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock), BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index fb57d6b531..abe566473f 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols, BLOCK_VERSION_1); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 080c2c9556..b51845d152 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -188,7 +188,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { .streamId = pTask->id.streamId, @@ -772,7 +772,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From 5b0fc1544d08e74487deca8f5300bb4e99ba23f7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 18:00:12 +0800 Subject: [PATCH 30/37] enh: unit test and optimization for base58 --- source/util/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 5e32bd82b7..3514c578e9 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -107,4 +107,4 @@ target_link_libraries(tbaseCodecTest os util common gtest_main) add_test( NAME tbaseCodecTest COMMAND tbaseCodecTest - +) From 75caea4fbf6b47a17b87528960f69ce5d79e40d9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 19:04:21 +0800 Subject: [PATCH 31/37] enh: code optimization for tbase58 --- source/util/src/tbase58.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index e1cee72b47..2566d54431 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -145,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, inlen + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); if (!result) { if (bfree) taosMemoryFree(pbuf); terrno = TSDB_CODE_OUT_OF_MEMORY; From 7c9ae199899df75abfa10b9233cd1c9fc56776a5 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 12:31:33 +0000 Subject: [PATCH 32/37] core at show user --- source/dnode/mnode/impl/src/mndUser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5e5a3626a4..8a2313370a 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -2484,7 +2484,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); - char objName[20] = {0}; + char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); From c7a50acd632af1789f2a195ce4652c268d359a32 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 23 Feb 2024 09:05:30 +0800 Subject: [PATCH 33/37] rebuild sliding window --- source/libs/executor/src/streamcountwindowoperator.c | 3 +++ source/libs/stream/src/streamSessionState.c | 1 + 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 689ba54ca6..294c2730df 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -99,6 +99,9 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, (void**)&pCurWin->winInfo.pStatePos, &size); } } + if (ts < pCurWin->winInfo.sessionWin.win.ekey) { + pBuffInfo->rebuildWindow = true; + } } else { code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index e2aae130e5..3d0241df75 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -488,6 +488,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS void* pFileStore = getStateFileStore(pFileState); SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey); if (pCur) { + pCur->pStreamFileState = pFileState; SSessionKey key = {0}; void* pVal = NULL; int len = 0; From e80ac5a4a69e42fa6ccc48cbee67a6513c5945cb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 23 Feb 2024 10:14:59 +0800 Subject: [PATCH 34/37] fix(vnd/query): not release reader if not initialized --- source/dnode/vnode/src/vnd/vnodeQuery.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 4fc7a88494..d1c811858a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "vnd.h" #include "tsdb.h" +#include "vnd.h" #define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ do { \ @@ -49,7 +49,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // decode req if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + goto _exit4; } metaRsp.dbId = pVnode->config.dbId; @@ -59,7 +59,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { - goto _exit; + goto _exit4; } // query meta @@ -67,7 +67,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) { code = terrno; - goto _exit; + goto _exit3; } metaRsp.tableType = mer1.me.type; @@ -81,7 +81,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); - if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; strcpy(metaRsp.stbName, mer2.me.name); metaRsp.suid = mer2.me.uid; @@ -125,6 +125,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); _exit: + taosMemoryFree(metaRsp.pSchemas); +_exit2: + metaReaderClear(&mer2); +_exit3: + metaReaderClear(&mer1); +_exit4: rpcMsg.info = pMsg->info; rpcMsg.pCont = pRsp; rpcMsg.contLen = rspLen; @@ -141,9 +147,6 @@ _exit: *pMsg = rpcMsg; } - taosMemoryFree(metaRsp.pSchemas); - metaReaderClear(&mer2); - metaReaderClear(&mer1); return TSDB_CODE_SUCCESS; } @@ -706,5 +709,5 @@ void *vnodeGetIvtIdx(void *pVnode) { } int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) { - return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid); + return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid); } From 31cf877f50c6b2221daa48b3e252323b81588938 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Feb 2024 10:16:27 +0800 Subject: [PATCH 35/37] enh: code optimization for tbase58 --- source/util/src/tbase58.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index 2566d54431..5eb72879c3 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -87,6 +87,7 @@ static const signed char index_58[256] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { + const char *pb = value; const char *pe = value + inlen; uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; @@ -98,20 +99,22 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { return NULL; } - for (int32_t i = 0; i < inlen; ++i) { - if (value[i] == 0) { + while (pb != pe) { + if (*pb == 0) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } + ++pb; } - while (*value && isspace(*value)) ++value; - while (*value == '1') { + pb = value; + while (pb != pe && *pb && isspace(*pb)) ++pb; + while (pb != pe && *pb == '1') { ++nz; - ++value; + ++pb; } - size = (int32_t)(pe - value) * 733 / 1000 + 1; + size = (int32_t)(pe - pb) * 733 / 1000 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -120,8 +123,8 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { bfree = true; } - while (*value && !isspace(*value)) { - int32_t num = index_58[(uint8_t)*value]; + while (pb != pe && *pb && !isspace(*pb)) { + int32_t num = index_58[(uint8_t)*pb]; if (num == -1) { terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); @@ -134,11 +137,11 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { num >>= 8; } len = i; - ++value; + ++pb; } - while (isspace(*value)) ++value; - if (*value != 0) { + while (pb != pe && isspace(*pb)) ++pb; + if (*pb != 0) { if (bfree) taosMemoryFree(pbuf); return NULL; } From ef7b38d4f69257df739a6e602deca68482e4bd81 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Feb 2024 10:30:47 +0800 Subject: [PATCH 36/37] enh: code optimization for tbase58 --- source/util/test/tbaseCodecTest.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp index 70639e7ec9..4c56979885 100644 --- a/source/util/test/tbaseCodecTest.cpp +++ b/source/util/test/tbaseCodecTest.cpp @@ -3,10 +3,10 @@ #include #include "os.h" +#include "osTime.h" #include "taos.h" #include "taoserror.h" #include "tbase58.h" -#include "tbase64.h" #include "tglobal.h" using namespace std; @@ -23,14 +23,17 @@ int main(int argc, char **argv) { } static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { - char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + int64_t start = taosGetTimestampUs(); + char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); ASSERT_NE(nullptr, pEnc); int32_t encLen = strlen(pEnc); - std::cout << "index:" << index << ", encLen is " << encLen << std::endl; + int64_t endOfEnc = taosGetTimestampUs(); + std::cout << "index:" << index << ", encLen is " << encLen << ", cost:" << endOfEnc - start << " us" << std::endl; int32_t decLen = 0; char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); - std::cout << "index:" << index << ", decLen is " << decLen << std::endl; + std::cout << "index:" << index << ", decLen is " << decLen << ", cost:" << taosGetTimestampUs() - endOfEnc << " us" + << std::endl; ASSERT_NE(nullptr, pDec); ASSERT_EQ(rawLen, decLen); ASSERT_LE(rawLen, encLen); @@ -51,7 +54,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) { // 1. normal case // string blend with char and '\0' rawLen = TEST_LEN_MAX; - for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { checkBase58Codec(pRaw, rawLen, i); pRaw[i] = i & 127; } @@ -61,7 +64,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) { pRaw[i] = i & 127; } checkBase58Codec(pRaw, TEST_LEN_MAX, 0); - for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { rawLen = i; checkBase58Codec(pRaw, rawLen, i); } From acdc21ff6c3aaadf0d68cbbe94b2e692126dae09 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 23 Feb 2024 11:38:04 +0800 Subject: [PATCH 37/37] fix: modify retry to upload to 8 minutes --- tests/army/enterprise/s3/s3_basic.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 5d9a52ebc4..45519d925f 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -28,12 +28,12 @@ from frame import * from frame.eos import * # -# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +# 192.168.1.52 MINIO S3 # ''' s3EndPoint http://192.168.1.52:9000 -s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +s3AccessKey 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX' s3BucketName ci-bucket s3UploadDelaySec 60 ''' @@ -42,7 +42,7 @@ s3UploadDelaySec 60 class TDTestCase(TBase): updatecfgDict = { 's3EndPoint': 'http://192.168.1.52:9000', - 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', + 's3AccessKey': 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX', 's3BucketName': 'ci-bucket', 's3BlockSize': '10240', 's3BlockCacheSize': '320', @@ -78,26 +78,27 @@ class TDTestCase(TBase): self.trimDb(True) rootPath = sc.clusterRootPath() - cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" + cmd = f"ls {rootPath}/dnode1/data2*/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 rets = [] - while loop < 500: + while loop < 180: + time.sleep(3) rets = eos.runRetList(cmd) cnt = len(rets) if cnt == 0: tdLog.info("All data file upload to server over.") - break - time.sleep(3) + break self.trimDb(True) - loop += 1 - tdLog.info(f"loop={loop} data files have {cnt} wait 5s...") - if loop == 4: + tdLog.info(f"loop={loop} no upload {cnt} data files wait 3s retry ...") + if loop == 0: sc.dnodeStop(1) time.sleep(2) sc.dnodeStart(1) + loop += 1 + if len(rets) > 0: - tdLog.exit(f"s3 can not upload all data to server. data files={rets}") + tdLog.exit(f"s3 can not upload all data to server. data files cnt={len(rets)} list={rets}") def checkStreamCorrect(self): sql = f"select count(*) from {self.db}.stm1"