From 25ab03e5d3615739ca01b2913169cb2dc7ba5d37 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 15 Dec 2023 17:50:16 +0800 Subject: [PATCH 001/236] refact: improve info logging msg in mmProcessWriteMsg --- source/dnode/mnode/impl/src/mndSync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index f46f33ac22..585d8ea8e7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -180,7 +180,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { goto _OUT; } - mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 + mInfo("trans:%d, process sync proposal, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 " role:%s raw:%p sec:%d seq:%" PRId64, transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state), pRaw, pMgmt->transSec, pMgmt->transSeq); From ea6e78cbaaa8d433a79d16024a8bd1937fb03461 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Dec 2023 16:33:44 +0800 Subject: [PATCH 002/236] fix(stream): add async call restart, instead of sync wait. --- include/dnode/vnode/tqCommon.h | 1 + include/libs/stream/tstream.h | 12 ++++--- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 42 ++++++++++++++++------ source/libs/stream/src/streamMeta.c | 8 +++-- source/libs/stream/src/streamStart.c | 3 ++ 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 75dafcdbff..32ac38cee6 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -32,5 +32,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t startStreamTasks(SStreamMeta* pMeta); int32_t resetStreamTaskStatus(SStreamMeta* pMeta); +int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); #endif // TDENGINE_TQ_COMMON_H diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6730d211df..812b57b01f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -460,14 +460,18 @@ struct SStreamTask { char reserve[256]; }; +typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*); + typedef struct STaskStartInfo { int64_t startTs; int64_t readyTs; int32_t tasksWillRestart; - int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. - SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing - SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed + int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. + SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing + SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed int64_t elapsedTime; + int32_t restartCount; // restart task counter + startComplete_fn_t completeFn; // complete callback function } STaskStartInfo; typedef struct STaskUpdateInfo { @@ -827,7 +831,7 @@ int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask); // stream task meta void streamMetaInit(); void streamMetaCleanup(); -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage); +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, startComplete_fn_t fn); void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 380be1dd38..432f80824a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -127,7 +127,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { } pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); if (pSnode->pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 138c58b45f..76049d21b1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -96,7 +96,8 @@ int32_t tqInitialize(STQ* pTq) { return -1; } - pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId, -1); + int32_t vgId = TD_VID(pTq->pVnode); + pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, vgId, -1, tqStartTaskCompleteCallback); if (pTq->pStreamMeta == NULL) { return -1; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 480ed7fd38..004eb240b3 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -699,8 +699,8 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { } int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { - int32_t vgId = pMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); if (numOfTasks == 0) { @@ -723,16 +723,17 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { int32_t code = 0; int64_t st = taosGetTimestampMs(); - while(1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); + streamMetaWLock(pMeta); + if (pMeta->startInfo.taskStarting == 1) { + pMeta->startInfo.restartCount += 1; + tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, %d", vgId, pMeta->startInfo.restartCount); + streamMetaWUnLock(pMeta); + return TSDB_CODE_SUCCESS; } + pMeta->startInfo.taskStarting = 1; + streamMetaWUnLock(pMeta); + terrno = 0; tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, pMeta->updateInfo.transId); @@ -791,7 +792,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, - pTask->chkInfo.nextProcessVer); + pTask->chkInfo.nextProcessVer); streamExecTask(pTask); } else { int8_t status = streamTaskSetSchedStatusInactive(pTask); @@ -808,4 +809,23 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead } } +int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { + STaskStartInfo* pStartInfo = &pMeta->startInfo; + taosWLockLatch(&pMeta->lock); + if (pStartInfo->restartCount > 0) { + pStartInfo->restartCount -= 1; + + ASSERT(pStartInfo->taskStarting == 0); + tqDebug("vgId:%d role:%d need to restart all tasks again, restartCounter:%d", pMeta->vgId, pMeta->role, + pStartInfo->restartCount); + + taosWUnLockLatch(&pMeta->lock); + restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); + } else { + taosWUnLockLatch(&pMeta->lock); + tqDebug("vgId:%d start all tasks completed", pMeta->vgId); + } + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 23cb6f5a35..8868a5719c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -277,7 +277,9 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); return 0; } -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { + +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, + startComplete_fn_t fn) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -343,6 +345,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->stage = stage; pMeta->role = (vgId == SNODE_HANDLE) ? NODE_ROLE_LEADER : NODE_ROLE_UNINIT; + pMeta->startInfo.completeFn = fn; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); // pMeta->chkpId = streamGetLatestCheckpointId(pMeta); @@ -1258,8 +1261,9 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { taosHashClear(pStartInfo->pFailedTaskSet); pStartInfo->tasksWillRestart = 0; pStartInfo->readyTs = 0; + // reset the sentinel flag value to be 0 - atomic_store_32(&pStartInfo->taskStarting, 0); + pStartInfo->taskStarting = 0; } void streamMetaRLock(SStreamMeta* pMeta) { diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 53f87591e8..dd713290f3 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1083,6 +1083,7 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI int64_t endTs, bool ready) { STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskId id = {.streamId = streamId, .taskId = taskId}; + bool restart = true; streamMetaWLock(pMeta); SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; @@ -1106,6 +1107,8 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true); displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); streamMetaResetStartInfo(pStartInfo); + + pStartInfo->completeFn(pMeta); } else { stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } From e734569de07850ef02c7077fd95b702debb44a1b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Dec 2023 18:42:57 +0800 Subject: [PATCH 003/236] fix(stream): fix dead lock. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 6 +++--- source/libs/stream/src/streamMeta.c | 12 ------------ source/libs/stream/src/streamStart.c | 3 ++- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 004eb240b3..a42f81edab 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -811,7 +811,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { STaskStartInfo* pStartInfo = &pMeta->startInfo; - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); if (pStartInfo->restartCount > 0) { pStartInfo->restartCount -= 1; @@ -820,10 +820,10 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { tqDebug("vgId:%d role:%d need to restart all tasks again, restartCounter:%d", pMeta->vgId, pMeta->role, pStartInfo->restartCount); - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); } else { - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); tqDebug("vgId:%d start all tasks completed", pMeta->vgId); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8868a5719c..e94c1b7596 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -348,18 +348,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->startInfo.completeFn = fn; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - // pMeta->chkpId = streamGetLatestCheckpointId(pMeta); - // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); - // while (pMeta->streamBackend == NULL) { - // qError("vgId:%d failed to init stream backend", pMeta->vgId); - // taosMsleep(2 * 1000); - // qInfo("vgId:%d retry to init stream backend", pMeta->vgId); - // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); - // if (pMeta->streamBackend == NULL) { - // } - // } - // pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); - pMeta->numOfPausedTasks = 0; pMeta->numOfStreamTasks = 0; stInfo("vgId:%d open stream meta successfully, latest checkpoint:%" PRId64 ", stage:%" PRId64, vgId, pMeta->chkpId, diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index dd713290f3..8bd7984d29 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1107,12 +1107,13 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true); displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); streamMetaResetStartInfo(pStartInfo); + streamMetaWUnLock(pMeta); pStartInfo->completeFn(pMeta); } else { + streamMetaWUnLock(pMeta); stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } - streamMetaWUnLock(pMeta); return TSDB_CODE_SUCCESS; } From 4c13dee7216a24fccff8f8fa0936b7cbcab087dc Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 19 Dec 2023 12:00:36 +0800 Subject: [PATCH 004/236] enh: protect processing of snap preparation rsp with sender buf mutex --- source/libs/sync/src/syncSnapshot.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index f060e9da13..2277b70c8f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1087,7 +1087,10 @@ static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSe // sender static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + int32_t code = -1; SSnapshot snapshot = {0}; + + taosThreadMutexLock(&pSender->pSndBuf->mutex); pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); // prepare @@ -1103,20 +1106,24 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // start reader if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) { - return -1; + goto _out; } } - int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); + code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); if (code != 0) { sSError(pSender, "prepare snapshot failed since %s", terrstr()); - return -1; + goto _out; } // update next index syncIndexMgrSetIndex(pSyncNode->pNextIndex, &pMsg->srcId, snapshot.lastApplyIndex + 1); - return snapshotSend(pSender); + code = snapshotSend(pSender); + +_out: + taosThreadMutexUnlock(&pSender->pSndBuf->mutex); + return code; } static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { From fee198f9ada4cdf223810f3c28512f11867e0348 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Dec 2023 16:21:11 +0800 Subject: [PATCH 005/236] refactor: do some internal refactor. --- include/dnode/vnode/tqCommon.h | 2 +- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- source/libs/stream/src/streamMeta.c | 4 ++++ source/libs/stream/src/streamStart.c | 1 - source/os/test/osTests.cpp | 2 +- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 32ac38cee6..0ad3023cbe 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -31,7 +31,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t startStreamTasks(SStreamMeta* pMeta); -int32_t resetStreamTaskStatus(SStreamMeta* pMeta); +int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); #endif // TDENGINE_TQ_COMMON_H diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 853dc9d1aa..31d923169a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -150,7 +150,7 @@ FAIL: } int32_t sndInit(SSnode * pSnode) { - resetStreamTaskStatus(pSnode->pMeta); + tqStreamTaskResetStatus(pSnode->pMeta); startStreamTasks(pSnode->pMeta); return 0; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index bfae691dbf..011ce674ec 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -218,7 +218,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { tqInfo("vgId:%d start all stream tasks after all being updated", vgId); - resetStreamTaskStatus(pTq->pStreamMeta); + tqStreamTaskResetStatus(pTq->pStreamMeta); tqStartStreamTaskAsync(pTq, false); } else { tqInfo("vgId:%d, follower node not start stream tasks", vgId); @@ -703,7 +703,7 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { return code; } -int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { +int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); @@ -763,7 +763,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } if (isLeader && !tsDisableStream) { - resetStreamTaskStatus(pMeta); + tqStreamTaskResetStatus(pMeta); streamMetaWUnLock(pMeta); tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5871a60c9e..48811b2925 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -570,7 +570,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); - resetStreamTaskStatus(pVnode->pTq->pStreamMeta); + tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta); tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); } } else { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 59178336d8..c263968828 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1270,18 +1270,22 @@ void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); taosRLockLatch(&pMeta->lock); } + void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); taosRUnLockLatch(&pMeta->lock); } + void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); taosWLockLatch(&pMeta->lock); } + void streamMetaWUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosWUnLockLatch(&pMeta->lock); } + static void execHelper(struct SSchedMsg* pSchedMsg) { __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; int32_t code = execFn(pSchedMsg->thandle); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 8bd7984d29..0ae85024c0 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1083,7 +1083,6 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI int64_t endTs, bool ready) { STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskId id = {.streamId = streamId, .taskId = taskId}; - bool restart = true; streamMetaWLock(pMeta); SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index 16660a9477..4d9ad9b5bd 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -72,7 +72,7 @@ TEST(osTest, osSystem) { const int sysLen = 64; char osSysName[sysLen]; int ret = taosGetOsReleaseName(osSysName, NULL, NULL, sysLen); - printf("os systeme name:%s\n", osSysName); + printf("os system name:%s\n", osSysName); ASSERT_EQ(ret, 0); } From 92045f54856713bff0ef6801f8cc4bc412afce69 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Dec 2023 18:34:30 +0800 Subject: [PATCH 006/236] fix(stream): add snode check, and handle the task status reset from checkpoint when it is in the snode. --- include/dnode/vnode/tqCommon.h | 1 + source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 127 ++++++++++++++------ source/dnode/snode/src/snode.c | 2 + source/dnode/vnode/src/tq/tq.c | 21 +--- source/dnode/vnode/src/tqCommon/tqCommon.c | 22 ++++ 6 files changed, 120 insertions(+), 55 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 0ad3023cbe..da9dcf1c04 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -33,5 +33,6 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t startStreamTasks(SStreamMeta* pMeta); int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); +int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); #endif // TDENGINE_TQ_COMMON_H diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 6de29f8513..444739e461 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -90,7 +90,7 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 62840f7e1f..b8ecf66686 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -685,6 +685,40 @@ _OVER: return -1; } +static int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool* hasEpset, int32_t taskId, int32_t nodeId) { + *hasEpset = false; + + if (nodeId == SNODE_HANDLE) { + SSnodeObj *pObj = NULL; + void *pIter = NULL; + + pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter != NULL) { + addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); + sdbRelease(pMnode->pSdb, pObj); + sdbCancelFetch(pMnode->pSdb, pIter); + *hasEpset = true; + return TSDB_CODE_SUCCESS; + } else { + mError("failed to acquire snode epset"); + return TSDB_CODE_INVALID_PARA; + } + } else { + SVgObj *pVgObj = mndAcquireVgroup(pMnode, nodeId); + if (pVgObj != NULL) { + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + + epsetAssign(pEpSet, &epset); + *hasEpset = true; + return TSDB_CODE_SUCCESS; + } else { + mDebug("orphaned task:0x%x need to be dropped, nodeId:%d, no redo action", taskId, nodeId); + return TSDB_CODE_SUCCESS; + } + } +} + static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { @@ -698,28 +732,17 @@ static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask STransAction action = {0}; SEpSet epset = {0}; - if (pTask->info.nodeId == SNODE_HANDLE) { - SSnodeObj *pObj = NULL; - void *pIter = NULL; - while (1) { - pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); - if (pIter == NULL) { - break; - } + bool hasEpset = false; - addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); - sdbRelease(pMnode->pSdb, pObj); - } - } else { - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - if (pVgObj != NULL) { - epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); - } else { - mDebug("orphaned task:0x%x need to be dropped, nodeId:%d, no redo action", pTask->id.taskId, pTask->info.nodeId); - taosMemoryFree(pReq); - return 0; - } + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } + + // no valid epset, return directly without redoAction + if (!hasEpset) { + return TSDB_CODE_SUCCESS; } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. @@ -1776,9 +1799,20 @@ static int32_t mndPauseStreamTask(SMnode *pMnode, STrans *pTrans, SStreamTask *p pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + taosMemoryFree(pReq); + return -1; + } + + // no valid epset, return directly without redoAction + if (!hasEpset) { + taosMemoryFree(pReq); + return TSDB_CODE_SUCCESS; + } STransAction action = {0}; initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0); @@ -1920,17 +1954,25 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndResumeStreamTask(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) { SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); if (pReq == NULL) { + mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; pReq->igUntreated = igUntreated; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + taosMemoryFree(pReq); + return -1; + } STransAction action = {0}; initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0); @@ -2208,6 +2250,8 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP epsetAssign(&updateInfo.newEp, &pCurrent->epset); taosArrayPush(info.pUpdateNodeList, &updateInfo); } + + // todo handle the snode info if (pCurrent->nodeId != SNODE_HANDLE) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pCurrent->nodeId); taosHashPut(info.pDBMap, pVgroup->dbName, strlen(pVgroup->dbName), NULL, 0); @@ -2704,9 +2748,18 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pReq); + continue; + } + + if (!hasEpset) { + taosMemoryFree(pReq); + continue; + } STransAction action = {0}; initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0); @@ -2878,11 +2931,17 @@ static int32_t mndDropRelatedFillhistoryTask(SMnode *pMnode, STaskStatusEntry *p mDebug("build and send drop related fill-history task for task:0x%x", pTask->id.taskId); - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (hasEpset) { + tmsgSendReq(&epset, &msg); + } - tmsgSendReq(&epset, &msg); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 31d923169a..f0fe662025 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -205,6 +205,8 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return tqStreamTaskProcessDropReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen); case TDMT_VND_STREAM_TASK_UPDATE: return tqStreamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); + case TDMT_VND_STREAM_TASK_RESET: + return tqStreamTaskProcessTaskResetReq(pSnode->pMeta, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 76049d21b1..c33becf5b9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1336,26 +1336,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { - SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg->pCont; - - SStreamMeta* pMeta = pTq->pStreamMeta; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask == NULL) { - tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", - pMeta->vgId, pReq->taskId); - return TSDB_CODE_SUCCESS; - } - - tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); - - // clear flag set during do checkpoint, and open inputQ for all upstream tasks - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) { - streamTaskClearCheckInfo(pTask, true); - streamTaskSetStatusReady(pTask); - } - - streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; + return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 011ce674ec..0d932e6e72 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -832,5 +832,27 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { tqDebug("vgId:%d start all tasks completed", pMeta->vgId); } + return TSDB_CODE_SUCCESS; +} + +int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg->pCont; + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, pReq->taskId); + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); + + // clear flag set during do checkpoint, and open inputQ for all upstream tasks + if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) { + streamTaskClearCheckInfo(pTask, true); + streamTaskSetStatusReady(pTask); + } + + streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; } \ No newline at end of file From 2de0f6e9713fb56c04cfa34ba505cfe3e2cf8d3d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Dec 2023 18:56:15 +0800 Subject: [PATCH 007/236] test(stream): lock the update procedure. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 63 +++------------------- 1 file changed, 8 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 0d932e6e72..c39e653596 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -106,14 +106,15 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM return rsp.code; } - streamMetaWUnLock(pMeta); +// streamMetaWUnLock(pMeta); + // todo for test purpose // the following two functions should not be executed within the scope of meta lock to avoid deadlock streamTaskUpdateEpsetInfo(pTask, req.pNodeList); streamTaskResetStatus(pTask); // continue after lock the meta again - streamMetaWLock(pMeta); +// streamMetaWLock(pMeta); SStreamTask** ppHTask = NULL; if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { @@ -166,65 +167,17 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM streamMetaWUnLock(pMeta); } else { if (!restored) { - tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", - vgId); + tqDebug("vgId:%d vnode restore not completed, not start the tasks, clear the start after nodeUpdate flag", vgId); pMeta->startInfo.tasksWillRestart = 0; streamMetaWUnLock(pMeta); } else { tqDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); -#if 1 +#if 0 + // for test purpose, to trigger the leader election + taosMSleep(5000); +#endif tqStreamTaskStartAsync(pMeta, cb, true); streamMetaWUnLock(pMeta); -#else - streamMetaWUnLock(pMeta); - - // For debug purpose. - // the following procedure consume many CPU resource, result in the re-election of leader - // with high probability. So we employ it as a test case for the stream processing framework, with - // checkpoint/restart/nodeUpdate etc. - while (1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - while (streamMetaTaskInTimer(pMeta)) { - tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - - int32_t code = streamMetaReopen(pMeta); - if (code != 0) { - tqError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - streamMetaInitBackend(pMeta); - - if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { - tqError("vgId:%d failed to load stream tasks", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { - tqInfo("vgId:%d start all stream tasks after all being updated", vgId); - tqStreamTaskResetStatus(pTq->pStreamMeta); - tqStartStreamTaskAsync(pTq, false); - } else { - tqInfo("vgId:%d, follower node not start stream tasks", vgId); - } - streamMetaWUnLock(pMeta); -#endif } } From f7ab8e99b503f6f20285d712f2f3362a008b2a65 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 19 Dec 2023 19:51:51 +0800 Subject: [PATCH 008/236] enh: demarcate top and bottom halves of task trans execution to ensure a consistent order --- source/dnode/mnode/impl/inc/mndTrans.h | 2 +- source/dnode/mnode/impl/src/mndSync.c | 8 +- source/dnode/mnode/impl/src/mndTrans.c | 148 ++++++++++++------------- 3 files changed, 74 insertions(+), 84 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 04544da80e..1bd39a2299 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -97,7 +97,7 @@ SSdbRaw *mndTransEncode(STrans *pTrans); SSdbRow *mndTransDecode(SSdbRaw *pRaw); void mndTransDropData(STrans *pTrans); -bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans, bool topHalf); #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 585d8ea8e7..1ba6c51d4c 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -208,15 +208,11 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { } if (pTrans->stage == TRN_STAGE_PREPARE) { - bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans); + bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans, false); if (!continueExec) goto _OUT; } - if (pTrans->id != pMgmt->transId) { - mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d", - pTrans->id, pTrans->createdTime, pMgmt->transId); - mndTransRefresh(pMnode, pTrans); - } + mndTransRefresh(pMnode, pTrans); sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7749decf91..49b483114a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -36,21 +36,25 @@ static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static void mndTransDropLogs(SArray *pArray); static void mndTransDropActions(SArray *pArray); -static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray); -static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans); -static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } + +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf); +static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf); + +static bool mndCannotExecuteTransAction(SMnode *pMnode, bool topHalf) { + return (!pMnode->deploy && !mndIsLeader(pMnode)) || !topHalf; +} static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); @@ -1090,8 +1094,9 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) } } -static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { +static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; + if (topHalf) return -1; int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -1112,9 +1117,9 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi return code; } -static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { - if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode)) return -1; +static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { + if (pAction->msgSent) return 0; + if (mndCannotExecuteTransAction(pMnode, topHalf)) return -1; int64_t signature = pTrans->id; signature = (signature << 32); @@ -1159,7 +1164,8 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio return code; } -static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { +static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { + if (!topHalf) return -1; pAction->rawWritten = 0; pAction->errCode = 0; mInfo("trans:%d, %s:%d confirm action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); @@ -1168,34 +1174,34 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction return 0; } -static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { +static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->actionType == TRANS_ACTION_RAW) { - return mndTransWriteSingleLog(pMnode, pTrans, pAction); + return mndTransWriteSingleLog(pMnode, pTrans, pAction, topHalf); } else if (pAction->actionType == TRANS_ACTION_MSG) { - return mndTransSendSingleMsg(pMnode, pTrans, pAction); + return mndTransSendSingleMsg(pMnode, pTrans, pAction, topHalf); } else { - return mndTransExecNullMsg(pMnode, pTrans, pAction); + return mndTransExecNullMsg(pMnode, pTrans, pAction, topHalf); } } -static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { +static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) { int32_t numOfActions = taosArrayGetSize(pArray); int32_t code = 0; for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); - code = mndTransExecSingleAction(pMnode, pTrans, pAction); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code != 0) break; } return code; } -static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) { int32_t numOfActions = taosArrayGetSize(pArray); if (numOfActions == 0) return 0; - if (mndTransExecSingleActions(pMnode, pTrans, pArray) != 0) { + if (mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf) != 0) { return -1; } @@ -1248,31 +1254,31 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } } -static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); +static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since:%s, code:0x%x", terrstr(), terrno); } return code; } -static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); +static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; } -static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions); +static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute commitActions since %s", terrstr()); } return code; } -static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) { +static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = 0; int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; @@ -1289,7 +1295,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); - code = mndTransExecSingleAction(pMnode, pTrans, pAction); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code == 0) { if (pAction->msgSent) { if (pAction->msgReceived) { @@ -1317,14 +1323,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } mndSetTransLastAction(pTrans, pAction); - if (mndCannotExecuteTransAction(pMnode)) break; + if (mndCannotExecuteTransAction(pMnode, topHalf)) break; if (code == 0) { pTrans->code = 0; pTrans->redoActionPos++; mInfo("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), pAction->id); + taosThreadMutexUnlock(&pTrans->mutex); code = mndTransSync(pMnode, pTrans); + taosThreadMutexLock(&pTrans->mutex); if (code != 0) { pTrans->redoActionPos--; pTrans->code = terrno; @@ -1357,7 +1365,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) return code; } -bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; int32_t code = 0; @@ -1368,7 +1376,7 @@ bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); - code = mndTransExecSingleAction(pMnode, pTrans, pAction); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code != 0) { mError("trans:%d, failed to execute prepare action:%d, numOfActions:%d", pTrans->id, action, numOfActions); return false; @@ -1381,17 +1389,17 @@ _OVER: return continueExec; } -static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; int32_t code = 0; if (pTrans->exec == TRN_EXEC_SERIAL) { - code = mndTransExecuteRedoActionsSerial(pMnode, pTrans); + code = mndTransExecuteRedoActionsSerial(pMnode, pTrans, topHalf); } else { - code = mndTransExecuteRedoActions(pMnode, pTrans); + code = mndTransExecuteRedoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode)) return false; + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; terrno = code; if (code == 0) { @@ -1431,8 +1439,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { - if (mndCannotExecuteTransAction(pMnode)) return false; +static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransCommit(pMnode, pTrans); @@ -1452,9 +1460,9 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; - int32_t code = mndTransExecuteCommitActions(pMnode, pTrans); + int32_t code = mndTransExecuteCommitActions(pMnode, pTrans, topHalf); if (code == 0) { pTrans->code = 0; @@ -1471,9 +1479,9 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; - int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); + int32_t code = mndTransExecuteUndoActions(pMnode, pTrans, topHalf); if (code == 0) { pTrans->stage = TRN_STAGE_PRE_FINISH; @@ -1491,8 +1499,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { - if (mndCannotExecuteTransAction(pMnode)) return false; +static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransRollback(pMnode, pTrans); @@ -1510,8 +1518,8 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { - if (mndCannotExecuteTransAction(pMnode)) return false; +static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransPreFinish(pMnode, pTrans); @@ -1529,8 +1537,9 @@ static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = false; + if (topHalf) return continueExec; SSdbRaw *pRaw = mndTransEncode(pTrans); if (pRaw == NULL) { @@ -1558,43 +1567,28 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { case TRN_STAGE_PREPARE: - continueExec = mndTransPerformPrepareStage(pMnode, pTrans); + continueExec = mndTransPerformPrepareStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_REDO_ACTION: - continueExec = mndTransPerformRedoActionStage(pMnode, pTrans); + continueExec = mndTransPerformRedoActionStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_COMMIT: - if (topHalf) { - continueExec = mndTransPerformCommitStage(pMnode, pTrans); - } else { - mInfo("trans:%d, can not commit since not leader", pTrans->id); - continueExec = false; - } + continueExec = mndTransPerformCommitStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_COMMIT_ACTION: - continueExec = mndTransPerformCommitActionStage(pMnode, pTrans); + continueExec = mndTransPerformCommitActionStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_ROLLBACK: - if (topHalf) { - continueExec = mndTransPerformRollbackStage(pMnode, pTrans); - } else { - mInfo("trans:%d, can not rollback since not leader", pTrans->id); - continueExec = false; - } + continueExec = mndTransPerformRollbackStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_UNDO_ACTION: - continueExec = mndTransPerformUndoActionStage(pMnode, pTrans); + continueExec = mndTransPerformUndoActionStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_PRE_FINISH: - if (topHalf) { - continueExec = mndTransPerformPreFinishStage(pMnode, pTrans); - } else { - mInfo("trans:%d, can not pre-finish since not leader", pTrans->id); - continueExec = false; - } + continueExec = mndTransPerformPreFinishStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_FINISH: - continueExec = mndTransPerformFinishStage(pMnode, pTrans); + continueExec = mndTransPerformFinishStage(pMnode, pTrans, topHalf); break; default: continueExec = false; From e06f355143d5be866c226080a8b6e9122cd9c3ad Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 19 Dec 2023 20:16:33 +0800 Subject: [PATCH 009/236] enh: improve logging msg of mndPostMgmtCode --- source/dnode/mnode/impl/src/mndSync.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 1ba6c51d4c..00cbcbce6c 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -230,6 +230,7 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { goto _OUT; } + int32_t transId = pMgmt->transId; pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->transSeq = 0; @@ -237,9 +238,9 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { tsem_post(&pMgmt->syncSem); if (pMgmt->errCode != 0) { - mError("trans:%d, failed to propose since %s, post sem", pMgmt->transId, tstrerror(pMgmt->errCode)); + mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); } else { - mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, pMgmt->transId, pMgmt->transSeq); + mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, transId, pMgmt->transSeq); } _OUT: From 541967e99ebf6fd85ec60064abff3e1bc08e3acf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 10:24:03 +0800 Subject: [PATCH 010/236] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 42 +++++++++++++++---------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index fd179dba2e..773ce9e513 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1360,7 +1360,8 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SVersionRange* pVerRange) { - int32_t step = ASCENDING_TRAVERSE(pSttBlockReader->order) ? 1 : -1; + int32_t order = pSttBlockReader->order; + int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; while (1) { bool hasVal = tMergeTreeNext(&pSttBlockReader->mergeTree); @@ -1377,10 +1378,12 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc pSttBlockReader->currentKey = key; pScanInfo->sttKeyInfo.nextProcKey = key; - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pSttBlockReader->order, - pVerRange)) { - pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; - return true; + if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) { + bool dropped = hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange); + if (!dropped) { + pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; + return true; + } } } } @@ -2054,9 +2057,12 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum return false; } - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->info.order, - &pReader->info.verRange)) { - return false; + if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { + bool deleted = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, + pReader->info.order, &pReader->info.verRange); + if (deleted) { + return false; + } } return true; @@ -3219,7 +3225,7 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange) { - if (pDelList == NULL || (taosArrayGetSize(pDelList) == 0)) { + if (pDelList == NULL || (TARRAY_SIZE(pDelList) == 0)) { return false; } @@ -3327,16 +3333,18 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); TSDBKEY key = TSDBROW_KEY(pRow); - + int32_t order = pReader->info.order; if (outOfTimeWindow(key.ts, &pReader->info.window)) { pIter->hasVal = false; return NULL; } // it is a valid data version - if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && - (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { - return pRow; + if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer)) { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } while (1) { @@ -3353,9 +3361,11 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p return NULL; } - if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && - (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { - return pRow; + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } } } From 81d8b2434f8b5c8e4a2c36be00115d3e0d25672b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 10:25:31 +0800 Subject: [PATCH 011/236] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 773ce9e513..751569bf90 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -3340,7 +3340,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } // it is a valid data version - if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer)) { + if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && + pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); if (!dropped) { return pRow; @@ -3361,7 +3362,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p return NULL; } - if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && + pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); if (!dropped) { return pRow; From 94b260a1f9431d21a4776d1c7e078de13781368e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 10:29:04 +0800 Subject: [PATCH 012/236] refactor: do some internal refactor. --- source/libs/stream/src/streamStart.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 743661bb59..68380e4ba4 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -209,7 +209,11 @@ void streamTaskCheckDownstream(SStreamTask* pTask) { int32_t numOfVgs = taosArrayGetSize(vgInfo); pTask->notReadyTasks = numOfVgs; - pTask->checkReqIds = taosArrayInit(numOfVgs, sizeof(int64_t)); + if (pTask->checkReqIds == NULL) { + pTask->checkReqIds = taosArrayInit(numOfVgs, sizeof(int64_t)); + } else { + taosArrayClear(pTask->checkReqIds); + } stDebug("s-task:%s check %d downstream tasks, ver:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64, pTask->id.idStr, numOfVgs, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey); @@ -436,8 +440,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs ASSERT(left >= 0); if (left == 0) { - taosArrayDestroy(pTask->checkReqIds); - pTask->checkReqIds = NULL; + pTask->checkReqIds = taosArrayDestroy(pTask->checkReqIds);; doProcessDownstreamReadyRsp(pTask); } else { From d25a323a4c1e801999e3c197fbbf85811f622829 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 13:55:12 +0800 Subject: [PATCH 013/236] fix(stream): record the launch failure. --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tqCommon/tqCommon.c | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 866bb4dc29..e38a677c8b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -324,6 +324,7 @@ typedef struct SStreamStatus { int8_t keepTaskStatus; bool appendTranstateBlock; // has append the transfer state data block already, todo: remove it int32_t timerActive; // timer is active + int8_t allowedAddInTimer; // allowed to add into timer int32_t inScanHistorySentinel; } SStreamStatus; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c33becf5b9..7e2bd1c5e0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1071,6 +1071,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { tqScanWal(pTq); return 0; } + int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); if(code == 0 && taskId > 0){ tqScanWalAsync(pTq, false); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c39e653596..2201f0744e 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -621,6 +621,8 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, + taosGetTimestampMs(), false); continue; } @@ -637,7 +639,7 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { streamLaunchFillHistoryTask(pTask); } - streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, + streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, pTask->execInfo.start, true); streamMetaReleaseTask(pMeta, pTask); continue; From 1a8583887f471ca7e87128a7c6c4e17517d8d8a1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 15:35:43 +0800 Subject: [PATCH 014/236] fix(stream): remove clear backendcache. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 5 +++-- source/libs/stream/src/streamMeta.c | 4 ++-- source/libs/stream/src/streamStart.c | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 2201f0744e..0864afdf2c 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -686,7 +686,8 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { streamMetaWLock(pMeta); if (pMeta->startInfo.taskStarting == 1) { pMeta->startInfo.restartCount += 1; - tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, %d", vgId, pMeta->startInfo.restartCount); + tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, + pMeta->startInfo.restartCount); streamMetaWUnLock(pMeta); return TSDB_CODE_SUCCESS; } @@ -720,7 +721,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { if (isLeader && !tsDisableStream) { tqStreamTaskResetStatus(pMeta); streamMetaWUnLock(pMeta); - tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); + tqInfo("vgId:%d start all stream tasks after reload tasks from disk", vgId); startStreamTasks(pMeta); } else { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 2bbc1607b3..b67b81b611 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -266,11 +266,12 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { if (pBackend == NULL) { taosThreadMutexUnlock(&pMeta->backendMutex); taosMsleep(1000); - stDebug("backed holded by other task, restart later, path: %s, key: %s", pMeta->path, key); + stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key); } else { taosThreadMutexUnlock(&pMeta->backendMutex); break; } + taosThreadMutexLock(&pMeta->backendMutex); pBackend = taskDbOpen(pMeta->path, key, chkpId); } @@ -448,7 +449,6 @@ void streamMetaClear(SStreamMeta* pMeta) { taosRemoveRef(streamBackendId, pMeta->streamBackendRid); taosHashClear(pMeta->pTasksMap); - taosHashClear(pMeta->pTaskDbUnique); taosArrayClear(pMeta->pTaskList); taosArrayClear(pMeta->chkpSaved); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 68380e4ba4..5ce7668048 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1113,7 +1113,7 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI pStartInfo->completeFn(pMeta); } else { streamMetaWUnLock(pMeta); - stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); + stDebug("vgId:%d recv check downstream results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } return TSDB_CODE_SUCCESS; From 43fb7cf14ade07c3b4d6ed3baf4f71e538662ca1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 16:04:20 +0800 Subject: [PATCH 015/236] other: add log info. --- source/libs/stream/src/streamDispatch.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5fb7db233f..f0ee03810b 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -372,8 +372,8 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD pTask->msgInfo.pData = pReqs; } - stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64, pTask->id.idStr, pTask->execInfo.dispatch, - pTask->pMeta->stage); + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%"PRId64" %p", pTask->id.idStr, pTask->execInfo.dispatch, + pTask->pMeta->stage, pTask->msgInfo.pData); return code; } @@ -1081,6 +1081,7 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask) { // this message has been sent successfully, let's try next one. static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { + stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); From d6297e26ce089ce4ff2d97db792719b524af0c6d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 20 Dec 2023 16:56:12 +0800 Subject: [PATCH 016/236] init backend mutex --- source/libs/stream/src/streamMeta.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index ce391eeadb..6c5b1ef994 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -394,6 +394,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); pMeta->bkdChkptMgt = bkdMgtCreate(tpath); + taosThreadMutexInit(&pMeta->backendMutex, NULL); return pMeta; From 849aaf8d4fb70b73c03bbd85f9e068d0b2d74aa1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 18:07:53 +0800 Subject: [PATCH 017/236] fix(stream): avoid dead lock. --- source/libs/stream/src/streamMeta.c | 1 + source/libs/stream/src/streamStart.c | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b67b81b611..d6e1286093 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1215,6 +1215,7 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->status.timerActive >= 1) { + stDebug("s-task:%s in timer, blocking tasks in vgId:%d restart", pTask->id.idStr, pMeta->vgId); inTimer = true; } } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 5ce7668048..94f635a480 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -48,9 +48,13 @@ static void tryLaunchHistoryTask(void* param, void* tmrId); static void doProcessDownstreamReadyRsp(SStreamTask* pTask); int32_t streamTaskSetReady(SStreamTask* pTask) { - char* p = NULL; - int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); - ETaskStatus status = streamTaskGetStatus(pTask, &p); + SStreamMeta* pMeta = pTask->pMeta; + SStreamTaskId id = pTask->id; + int64_t initTs = pTask->execInfo.init; + int64_t startTs = pTask->execInfo.start; + char* p = NULL; + int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); + ETaskStatus status = streamTaskGetStatus(pTask, &p); if ((status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { @@ -67,8 +71,12 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", pTask->id.idStr, numOfDowns, el, p); - streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, - pTask->execInfo.start, true); + taosThreadMutexUnlock(&pTask->lock); + + // to avoid deadlock + streamMetaUpdateTaskDownstreamStatus(pMeta, id.streamId, id.taskId, initTs, startTs, true); + + taosThreadMutexLock(&pTask->lock); return TSDB_CODE_SUCCESS; } @@ -461,8 +469,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) { stError("s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%" PRId64 ", current stage:%" PRId64 - ", " - "not check wait for downstream task nodeUpdate, and all tasks restart", + ", not check wait for downstream task nodeUpdate, and all tasks restart", id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); } else { From 49062431150be43ac0704c00c12cbe2b36bdfc76 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 20 Dec 2023 18:32:23 +0800 Subject: [PATCH 018/236] fix: dynamic cfg resetlog --- source/common/src/tglobal.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fd88098b03..fdac3882c3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1432,6 +1432,13 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { terrno = TSDB_CODE_SUCCESS; + if (strcasecmp(name, "resetlog") == 0) { + // trigger, no item in cfg + taosResetLog(); + cfgDumpCfg(tsCfg, 0, false); + return 0; + } + SConfigItem *pItem = cfgGetItem(pCfg, name); if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { uError("failed to config:%s, not support", name); @@ -1445,12 +1452,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { return 0; } - if (strcasecmp(name, "resetlog") == 0) { - taosResetLog(); - cfgDumpCfg(tsCfg, 0, false); - return 0; - } - { // 'bool/int32_t/int64_t/float/double' variables with general modification function static OptionNameAndVar debugOptions[] = { {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag}, @@ -1774,4 +1775,4 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { uInfo("all debug flag are set to %d", flag); } -int8_t taosGranted() { return atomic_load_8(&tsGrant); } \ No newline at end of file +int8_t taosGranted() { return atomic_load_8(&tsGrant); } From 64eee4bbd2f799f1cb843d0b944fbbeef9863d77 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 19:06:32 +0800 Subject: [PATCH 019/236] fix(stream): add conflict check for nodeUpdate --- source/dnode/mnode/impl/src/mndStream.c | 30 ++++++++++++++++---- source/dnode/mnode/impl/src/mndStreamTrans.c | 4 +-- source/libs/stream/src/streamStart.c | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b8ecf66686..0e593a5109 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2330,12 +2330,28 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { } static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { - SSdb *pSdb = pMnode->pSdb; - - // check all streams that involved this vnode should update the epset info + SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void * pIter = NULL; - STrans * pTrans = NULL; + void *pIter = NULL; + STrans *pTrans = NULL; + + // conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool + while(1) { + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) { + break; + } + + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, true); + sdbRelease(pSdb, pStream); + + if (conflict) { + mWarn("nodeUpdate trans in progress, current nodeUpdate ignored"); + sdbCancelFetch(pSdb, pIter); + return -1; + } + } + while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -2351,6 +2367,8 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange sdbCancelFetch(pSdb, pIter); return terrno; } + + mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); } void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb)); @@ -2597,7 +2615,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { code = mndProcessVgroupChange(pMnode, &changeInfo); - // keep the new vnode snapshot + // keep the new vnode snapshot if success if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("create trans successfully, update cached node list"); taosArrayDestroy(execInfo.pNodeList); diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 0db3cb0a8a..a7d4703c8c 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -94,8 +94,8 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char tInfo.name); return true; } - } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || - (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0)) { + } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) || + (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) { mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 94f635a480..d94aa80369 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -73,7 +73,7 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { taosThreadMutexUnlock(&pTask->lock); - // to avoid deadlock + // todo: fix it, to avoid deadlock in: tqStreamTaskProcessUpdateReq streamMetaUpdateTaskDownstreamStatus(pMeta, id.streamId, id.taskId, initTs, startTs, true); taosThreadMutexLock(&pTask->lock); From ad5d78a6ca95d5fdb503bdbed9d0c7a392e700b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 19:12:47 +0800 Subject: [PATCH 020/236] fix(stream): fix deadlock. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0e593a5109..d5bb8b2b92 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2342,7 +2342,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange break; } - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, true); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, false); sdbRelease(pSdb, pStream); if (conflict) { From 32ce4b6a4ce9452f6fd4b14bfb1fc59295850932 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 23:12:59 +0800 Subject: [PATCH 021/236] fix(stream): fix race condition. --- source/libs/stream/src/streamStart.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index d94aa80369..d62990f181 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -48,10 +48,6 @@ static void tryLaunchHistoryTask(void* param, void* tmrId); static void doProcessDownstreamReadyRsp(SStreamTask* pTask); int32_t streamTaskSetReady(SStreamTask* pTask) { - SStreamMeta* pMeta = pTask->pMeta; - SStreamTaskId id = pTask->id; - int64_t initTs = pTask->execInfo.init; - int64_t startTs = pTask->execInfo.start; char* p = NULL; int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); ETaskStatus status = streamTaskGetStatus(pTask, &p); @@ -70,13 +66,6 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { int64_t el = (pTask->execInfo.start - pTask->execInfo.init); stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", pTask->id.idStr, numOfDowns, el, p); - - taosThreadMutexUnlock(&pTask->lock); - - // todo: fix it, to avoid deadlock in: tqStreamTaskProcessUpdateReq - streamMetaUpdateTaskDownstreamStatus(pMeta, id.streamId, id.taskId, initTs, startTs, true); - - taosThreadMutexLock(&pTask->lock); return TSDB_CODE_SUCCESS; } @@ -392,6 +381,10 @@ void doProcessDownstreamReadyRsp(SStreamTask* pTask) { } streamTaskOnHandleEventSuccess(pTask->status.pSM, event); + + int64_t initTs = pTask->execInfo.init; + int64_t startTs = pTask->execInfo.start; + streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, initTs, startTs, true); } static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { From 572368703dc0108090c572ebee03a13ec5f0e0e3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Dec 2023 09:33:56 +0800 Subject: [PATCH 022/236] enh: ensure mndTransProcessRsp invoked in mnode-write thread --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index d6de406987..0fb246e945 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -108,11 +108,11 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; From 8a8ddf56997ccfc5db8133217b3ebb241a689ee8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 10:30:44 +0800 Subject: [PATCH 023/236] fix(stream): record the checkpoint failure when nodeEp changed. --- source/libs/stream/inc/streamInt.h | 12 +----------- source/libs/stream/src/streamCheckpoint.c | 5 ++--- source/libs/stream/src/streamStart.c | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index f709741b57..341706ff3b 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -116,6 +116,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId); int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask); int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask); +void streamTaskSetCheckpointFailedId(SStreamTask* pTask); int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask); int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*); STaskId streamTaskExtractKey(const SStreamTask* pTask); @@ -137,17 +138,6 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); int32_t streamTransferStateToStreamTask(SStreamTask* pTask); -// <<<<<<< HEAD -// void streamClearChkptReadyMsg(SStreamTask* pTask); - -// int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const -// char*); STaskId streamTaskExtractKey(const SStreamTask* pTask); void streamTaskInitForLaunchHTask(SHistoryTaskInfo* -// pInfo); void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo); - -// void streamMetaResetStartInfo(STaskStartInfo* pMeta); - -// ======= -// >>>>>>> 3.0 SStreamQueue* streamQueueOpen(int64_t cap); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueProcessSuccess(SStreamQueue* queue); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index cf0682f037..4a99d54de1 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -341,9 +341,8 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { return code; } -void streamTaskSetFailedId(SStreamTask* pTask) { +void streamTaskSetCheckpointFailedId(SStreamTask* pTask) { pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; - pTask->chkInfo.checkpointId = pTask->chkInfo.checkpointingId; } int32_t getChkpMeta(char* id, char* path, SArray* list) { @@ -485,7 +484,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); taosThreadMutexUnlock(&pTask->lock); - streamTaskSetFailedId(pTask); + streamTaskSetCheckpointFailedId(pTask); stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, pTask->id.idStr, ckId); } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index d62990f181..3bef6adf57 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -313,9 +313,24 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64 ", prev:%" PRId64, id, upstreamTaskId, vgId, stage, pInfo->stage); + // record the checkpoint failure id and sent to mnode + taosThreadMutexLock(&pTask->lock); + ETaskStatus status = streamTaskGetStatus(pTask, NULL); + if (status == TASK_STATUS__CK) { + streamTaskSetCheckpointFailedId(pTask); + } + taosThreadMutexUnlock(&pTask->lock); } if (pInfo->stage != stage) { + + taosThreadMutexLock(&pTask->lock); + ETaskStatus status = streamTaskGetStatus(pTask, NULL); + if (status == TASK_STATUS__CK) { + streamTaskSetCheckpointFailedId(pTask); + } + taosThreadMutexUnlock(&pTask->lock); + return TASK_UPSTREAM_NEW_STAGE; } else if (pTask->status.downstreamReady != 1) { stDebug("s-task:%s vgId:%d leader:%d, downstream not ready", id, vgId, (pTask->pMeta->role == NODE_ROLE_LEADER)); From 71f6d9f06fd57b95ce938f5796d1d5698779f611 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 11:52:45 +0800 Subject: [PATCH 024/236] fix(stream): handle the case that recvs drop msg twice. --- source/dnode/vnode/src/tq/tq.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7e2bd1c5e0..02c9b61432 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1340,6 +1340,7 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg); } +// NOTE: here we may receive this message more than once, so need to handle this case int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { SVDropHTaskReq* pReq = (SVDropHTaskReq*)pMsg->pCont; @@ -1358,14 +1359,17 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { return TSDB_CODE_SUCCESS; } + taosThreadMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask, NULL); - ASSERT(status == TASK_STATUS__STREAM_SCAN_HISTORY); - - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); + if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { + streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); + } SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id); + taosThreadMutexUnlock(&pTask->lock); + // clear the scheduler status streamTaskSetSchedStatusInactive(pTask); tqDebug("s-task:%s set scheduler status:%d after drop fill-history task", pTask->id.idStr, pTask->status.schedStatus); From 2acc8423c5263fe57fc205a9e2f26447926e56e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 16:00:50 +0800 Subject: [PATCH 025/236] fix(stream): send msg to mnode before closing all tasks. --- include/libs/stream/tstream.h | 3 + source/dnode/vnode/src/tq/tqStreamTask.c | 13 ++- source/dnode/vnode/src/tq/tqUtil.c | 6 ++ source/libs/stream/src/streamMeta.c | 128 +++++++++++++++-------- 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e38a677c8b..9b96a41516 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -494,6 +494,7 @@ typedef struct SStreamMeta { int32_t vgId; int64_t stage; int32_t role; + bool sendMsgBeforeClosing; // send hb to mnode before close all tasks when switch to follower. STaskStartInfo startInfo; SRWLatch lock; int32_t walScanCounter; @@ -782,6 +783,8 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ int64_t* oldStage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); +SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); + bool streamTaskAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 1b0a76e81c..99cc3a36ea 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -126,17 +126,16 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t tqStopStreamTasks(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = TD_VID(pTq->pVnode); - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + int32_t num = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d stop all %d stream task(s)", vgId, numOfTasks); - if (numOfTasks == 0) { + tqDebug("vgId:%d stop all %d stream task(s)", vgId, num); + if (num == 0) { return TSDB_CODE_SUCCESS; } - SArray* pTaskList = NULL; - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - streamMetaWUnLock(pMeta); + // send hb msg to mnode before closing all tasks. + SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); + int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 110cf79b4e..d3ea1f19e5 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -40,6 +40,12 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { int64_t stage = pMeta->stage; pMeta->stage = state.term; + + // mark the sign to send msg before close all tasks + if ((!isLeader) && (pMeta->role == NODE_ROLE_LEADER)) { + pMeta->sendMsgBeforeClosing = true; + } + pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; if (isLeader) { tqInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d6e1286093..355e17db9a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1051,43 +1051,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { taosThreadMutexUnlock(&pTask->lock); } -void metaHbToMnode(void* param, void* tmrId) { - int64_t rid = *(int64_t*)param; - - SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid); - if (pMeta == NULL) { - return; - } - - // need to stop, stop now - if (pMeta->pHbInfo->stopFlag == STREAM_META_WILL_STOP) { - pMeta->pHbInfo->stopFlag = STREAM_META_OK_TO_STOP; - stDebug("vgId:%d jump out of meta timer", pMeta->vgId); - taosReleaseRef(streamMetaId, rid); - return; - } - - // not leader not send msg - if (pMeta->role != NODE_ROLE_LEADER) { - stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); - taosReleaseRef(streamMetaId, rid); - pMeta->pHbInfo->hbStart = 0; - return; - } - - // set the hb start time - if (pMeta->pHbInfo->hbStart == 0) { - pMeta->pHbInfo->hbStart = taosGetTimestampMs(); - } - - if (!waitForEnoughDuration(pMeta->pHbInfo)) { - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); - taosReleaseRef(streamMetaId, rid); - return; - } - - stDebug("vgId:%d build stream task hb, leader:%d", pMeta->vgId, (pMeta->role == NODE_ROLE_LEADER)); - +static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { SStreamHbMsg hbMsg = {0}; SEpSet epset = {0}; bool hasMnodeEpset = false; @@ -1181,23 +1145,62 @@ void metaHbToMnode(void* param, void* tmrId) { } tEncoderClear(&encoder); - SRpcMsg msg = { - .info.noResp = 1, - }; + SRpcMsg msg = {.info.noResp = 1}; initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); pMeta->pHbInfo->hbCount += 1; - stDebug("vgId:%d build and send hb to mnode, numOfTasks:%d total:%d", pMeta->vgId, hbMsg.numOfTasks, pMeta->pHbInfo->hbCount); + tmsgSendReq(&epset, &msg); } else { stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId); } -_end: + _end: streamMetaClearHbMsg(&hbMsg); taosArrayDestroy(pIdList); + return TSDB_CODE_SUCCESS; +} + +void metaHbToMnode(void* param, void* tmrId) { + int64_t rid = *(int64_t*)param; + + SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid); + if (pMeta == NULL) { + return; + } + + // need to stop, stop now + if (pMeta->pHbInfo->stopFlag == STREAM_META_WILL_STOP) { + pMeta->pHbInfo->stopFlag = STREAM_META_OK_TO_STOP; + stDebug("vgId:%d jump out of meta timer", pMeta->vgId); + taosReleaseRef(streamMetaId, rid); + return; + } + + // not leader not send msg + if (pMeta->role != NODE_ROLE_LEADER) { + stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); + taosReleaseRef(streamMetaId, rid); + pMeta->pHbInfo->hbStart = 0; + return; + } + + // set the hb start time + if (pMeta->pHbInfo->hbStart == 0) { + pMeta->pHbInfo->hbStart = taosGetTimestampMs(); + } + + if (!waitForEnoughDuration(pMeta->pHbInfo)) { + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); + taosReleaseRef(streamMetaId, rid); + return; + } + + stDebug("vgId:%d build stream task hb, leader:%d", pMeta->vgId, (pMeta->role == NODE_ROLE_LEADER)); + metaHeartbeatToMnodeImpl(pMeta); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); } @@ -1320,3 +1323,44 @@ int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, voi schedMsg.msg = code; return taosScheduleTask(pMeta->qHandle, &schedMsg); } + +SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { + SArray* pTaskList = NULL; + bool sendMsg = false; + + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + sendMsg = pMeta->sendMsgBeforeClosing; + streamMetaWUnLock(pMeta); + + if (!sendMsg) { + stDebug("vgId:%d no need to send msg to mnode before closing tasks", pMeta->vgId); + return pTaskList; + } + + stDebug("vgId:%d send msg to mnode before closing all tasks", pMeta->vgId); + + // send hb msg to mnode before closing all tasks. + int32_t numOfTasks = taosArrayGetSize(pTaskList); + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + taosThreadMutexLock(&pTask->lock); + ETaskStatus s = streamTaskGetStatus(pTask, NULL); + if (s == TASK_STATUS__CK) { + streamTaskSetCheckpointFailedId(pTask); + stDebug("s-task:%s mark the checkpoint:%"PRId64" failed", pTask->id.idStr, pTask->chkInfo.checkpointingId); + } + + taosThreadMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pMeta, pTask); + } + + metaHeartbeatToMnodeImpl(pMeta); + pMeta->sendMsgBeforeClosing = false; + return pTaskList; +} From 094c6cdfb4e4c2eb7f9f4f1b8ee767cff3f761fc Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Dec 2023 16:31:35 +0800 Subject: [PATCH 026/236] fix: the pre-allocated column has no data to copy --- source/libs/executor/src/tsort.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 1ebc7ad3c6..205cd7d3ef 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -475,6 +475,7 @@ static void appendOneRowToDataBlock(SSDataBlock* pBlock, const SSDataBlock* pSou if (isNull) { colDataSetVal(pColInfo, pBlock->info.rows, NULL, true); } else { + if (!pSrcColInfo->pData) continue; char* pData = colDataGetData(pSrcColInfo, *rowIndex); colDataSetVal(pColInfo, pBlock->info.rows, pData, false); } @@ -900,7 +901,7 @@ static int32_t getPageBufIncForRow(SSDataBlock* blk, int32_t row, int32_t rowIdx for (int32_t i = 0; i < numCols; ++i) { SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(blk->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - if (pColInfoData->varmeta.offset[row] != -1) { + if ((pColInfoData->varmeta.offset[row] != -1) && (pColInfoData->pData)) { char* p = colDataGetData(pColInfoData, row); sz += varDataTLen(p); } @@ -970,7 +971,6 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); nMergedRows += pHandle->pDataBlock->info.rows; - blockDataCleanup(pHandle->pDataBlock); blkPgSz = pgHeaderSz; bufInc = getPageBufIncForRow(minBlk, minRow, 0); From c5e39b8a81d7fd1db1fd3e6dac3fbe056d2d37a1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Dec 2023 16:52:27 +0800 Subject: [PATCH 027/236] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/sort-pre-cols.sim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/script/tsim/query/sort-pre-cols.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8edfb352ab..bcdd143cfc 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1069,6 +1069,7 @@ e ,,y,script,./test.sh -f tsim/query/unionall_as_table.sim ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim +,,y,script,./test.sh -f tsim/query/sort-pre-cols.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim diff --git a/tests/script/tsim/query/sort-pre-cols.sim b/tests/script/tsim/query/sort-pre-cols.sim new file mode 100644 index 0000000000..ef69725d87 --- /dev/null +++ b/tests/script/tsim/query/sort-pre-cols.sim @@ -0,0 +1,17 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database d +sql use d +sql create table st(ts timestamp, v int) tags(lj json) +sql insert into ct1 using st tags('{"instance":"200"}') values(now, 1)(now+1s, 2); +sql insert into ct2 using st tags('{"instance":"200"}') values(now+2s, 3)(now+3s, 4); +sql select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(v) from st group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), lj->'instance' order by time; +print $data01 +if $data01 != 0.000000000 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT From bcb3518622059fb915e716dcbc1227eb517193d8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Dec 2023 17:46:38 +0800 Subject: [PATCH 028/236] fix: shell autotab extend to save 95 visible character --- tools/shell/inc/shellTire.h | 4 ++-- tools/shell/src/shellTire.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/shell/inc/shellTire.h b/tools/shell/inc/shellTire.h index e87c3ee4f3..472f604a2c 100644 --- a/tools/shell/inc/shellTire.h +++ b/tools/shell/inc/shellTire.h @@ -19,8 +19,8 @@ // // The prefix search tree is a efficient storage words and search words tree, it support 95 visible ascii code character // -#define FIRST_ASCII 40 // first visible char is '0' -#define LAST_ASCII 122 // last visilbe char is 'z' +#define FIRST_ASCII 32 // first visible char is '0' +#define LAST_ASCII 126 // last visilbe char is 'z' // capacity save char is 95 #define CHAR_CNT (LAST_ASCII - FIRST_ASCII + 1) diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index a8726f3126..8aa67457f2 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -95,7 +95,7 @@ bool insertToTree(STire* tire, char* word, int len) { STireNode** nodes = tire->root.d; for (int i = 0; i < len; i++) { m = word[i] - FIRST_ASCII; - if (m < 0 || m > CHAR_CNT) { + if (m < 0 || m >= CHAR_CNT) { return false; } From 4137a44fd0f8df409bbcebe1360377a425de0df2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Dec 2023 17:49:20 +0800 Subject: [PATCH 029/236] enh: protect syncNodeRestore with log buffer mutex --- source/libs/sync/src/syncMain.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b8740a2858..6f3b3fdf98 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1162,9 +1162,12 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { ASSERTS(pSyncNode->pLogStore != NULL, "log store not created"); ASSERTS(pSyncNode->pLogBuf != NULL, "ring log buffer not created"); + taosThreadMutexLock(&pSyncNode->pLogBuf->mutex); SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore); SyncIndex endIndex = pSyncNode->pLogBuf->endIndex; + taosThreadMutexUnlock(&pSyncNode->pLogBuf->mutex); + if (lastVer != -1 && endIndex != lastVer + 1) { terrno = TSDB_CODE_WAL_LOG_INCOMPLETE; sError("vgId:%d, failed to restore sync node since %s. expected lastLogIndex:%" PRId64 ", lastVer:%" PRId64 "", From 9d3eb02f3c8faca8787dd2448d4474b1f659768b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 21 Dec 2023 17:53:44 +0800 Subject: [PATCH 030/236] fix(tsdb/cache): plus 1 to nCols to load stt data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3ca2846d07..0f5511d658 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1875,7 +1875,7 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb .backward = 1, .pSttFileBlockIterArray = pr->pLDataIterArray, .pCols = aCols, - .numOfCols = nCols, + .numOfCols = nCols + 1, .loadTombFn = loadSttTomb, .pReader = pr, .idstr = pr->idstr, From f564e3f7bfb77ac7ab63416afce20c647b10bf18 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 21 Dec 2023 18:43:40 +0800 Subject: [PATCH 031/236] test: modify test of last_cache_scan.py --- tests/pytest/util/sql.py | 96 ++++++++++++++++---- tests/system-test/2-query/last_cache_scan.py | 38 +------- 2 files changed, 80 insertions(+), 54 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 99d166ee33..5ed985187a 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -78,6 +78,26 @@ class TDSql: self.cursor.execute(s) time.sleep(2) + def execute(self, sql, queryTimes=30, show=False): + self.sql = sql + if show: + tdLog.info(sql) + i=1 + while i <= queryTimes: + try: + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + except Exception as e: + tdLog.notice("Try to execute sql again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 + time.sleep(1) + pass + def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True): caller = inspect.getframeinfo(inspect.stack()[1][0]) expectErrNotOccured = True @@ -158,6 +178,62 @@ class TDSql: time.sleep(1) pass + def query_success_failed(self, sql, row_tag=None, queryTimes=10, count_expected_res=None, expectErrInfo = None, fullMatched = True): + self.sql = sql + i=1 + while i <= queryTimes: + try: + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + + if count_expected_res is not None: + counter = 0 + while count_expected_res != self.queryResult[0][0]: + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + if counter < queryTimes: + counter += 0.5 + time.sleep(0.5) + else: + return False + + tdLog.info("query is success") + time.sleep(1) + continue + except Exception as e: + tdLog.notice("Try to query again, query times: %d "%i) + caller = inspect.getframeinfo(inspect.stack()[1][0]) + if i < queryTimes: + error_info = repr(e) + self.error_info = ','.join(error_info[error_info.index('(')+1:-1].split(",")[:-1]).replace("'","") + self.queryRows = 0 + self.queryCols = 0 + self.queryResult = None + + if fullMatched: + if expectErrInfo != None: + if expectErrInfo == self.error_info: + tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) + else: + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + else: + if expectErrInfo != None: + if expectErrInfo in self.error_info: + tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) + else: + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + + return self.error_info + elif i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 + time.sleep(1) + pass def is_err_sql(self, sql): err_flag = True @@ -471,25 +547,7 @@ class TDSql: time.sleep(1) continue - def execute(self, sql, queryTimes=30, show=False): - self.sql = sql - if show: - tdLog.info(sql) - i=1 - while i <= queryTimes: - try: - self.affectedRows = self.cursor.execute(sql) - return self.affectedRows - except Exception as e: - tdLog.notice("Try to execute sql again, query times: %d "%i) - if i == queryTimes: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - i+=1 - time.sleep(1) - pass + def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 01795f6eef..0457475ee7 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -339,44 +339,12 @@ class TDTestCase: tdSql.checkData(0, 0, '999') p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) p.check_returncode() - tdSql.query("select last(c1) from meters", queryTimes=1) - tdSql.checkData(0, 0, None) - tdSql.query('select last(*) from meters', queryTimes=1) - print(str(tdSql.queryResult)) - tdSql.checkData(0, 1, None) - tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) - tdSql.checkRows(1) - tdSql.checkData(0, 0, None) - tdSql.checkData(0, 1, None) - tdSql.checkData(0, 2, None) - - try: - tdSql.query('select ts, last(c1), c1, ts, c1 from meters', queryTimes=1) - except Exception as e: - if str(e).count('Invalid column name') == 1: - print('column has been dropped, the cache has been updated: %s' % (str(e))) - return - else: - raise - tdSql.checkRows(1) - tdSql.checkCols(5) - tdSql.checkData(0, 0, None) - tdSql.checkData(0, 1, None) - tdSql.checkData(0, 2, None) - tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, None) - - try: - tdSql.query('select last(c1), last(c2), last(c3) from meters', queryTimes=1) - except Exception as e: - if str(e).count('Invalid column name') == 1: - print('column has been dropped, the cache has been updated: %s' % (str(e))) - return - else: - raise + tdSql.query_success_failed("select ts, last(c1), c1, ts, c1 from meters", queryTimes=10, expectErrInfo="Invalid column name: c1") + tdSql.query('select last(c12), c12, ts from meters', queryTimes=1) tdSql.checkRows(1) tdSql.checkCols(3) tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) def test_cache_scan_with_drop_column(self): tdSql.query('select last(*) from meters') From 913fda4a36c29c86d03578f45ee7046ece7e2db6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Dec 2023 18:52:25 +0800 Subject: [PATCH 032/236] fix snode crash --- source/dnode/snode/src/snode.c | 77 ++++++++++++++++------------------ 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 21d813c7c0..79a89cb56e 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -19,26 +19,24 @@ #include "tqCommon.h" #include "tuuid.h" -#define sndError(...) \ - do { \ - if (sndDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("SND ERROR ", DEBUG_ERROR, sndDebugFlag, __VA_ARGS__); \ - } \ - } while (0) +// clang-format off +#define sndError(...) do { if (sndDebugFlag & DEBUG_ERROR) {taosPrintLog("SND ERROR ", DEBUG_ERROR, sndDebugFlag, __VA_ARGS__);}} while (0) +#define sndInfo(...) do { if (sndDebugFlag & DEBUG_INFO) { taosPrintLog("SND INFO ", DEBUG_INFO, sndDebugFlag, __VA_ARGS__);}} while (0) +#define sndDebug(...) do { if (sndDebugFlag & DEBUG_DEBUG) { taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__);}} while (0) +// clang-format on -#define sndInfo(...) \ - do { \ - if (sndDebugFlag & DEBUG_INFO) { \ - taosPrintLog("SND INFO ", DEBUG_INFO, sndDebugFlag, __VA_ARGS__); \ - } \ - } while (0) - -#define sndDebug(...) \ - do { \ - if (sndDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__); \ - } \ - } while (0) +static STaskId replaceStreamTaskId(SStreamTask *pTask) { + ASSERT(pTask->info.fillHistory); + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + pTask->id.streamId = pTask->streamTaskId.streamId; + pTask->id.taskId = pTask->streamTaskId.taskId; + return id; +} +static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) { + ASSERT(pTask->info.fillHistory); + pTask->id.taskId = pId->taskId; + pTask->id.streamId = pId->streamId; +} int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0); @@ -50,23 +48,22 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer streamTaskOpenAllUpstreamInput(pTask); - SStreamTask *pSateTask = pTask; - SStreamTask task = {0}; + STaskId taskId = {0}; if (pTask->info.fillHistory) { - task.id.streamId = pTask->streamTaskId.streamId; - task.id.taskId = pTask->streamTaskId.taskId; - task.pMeta = pTask->pMeta; - pSateTask = &task; + taskId = replaceStreamTaskId(pTask); } - pTask->pState = streamStateOpen(pSnode->path, pSateTask, false, -1, -1); + pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); if (pTask->pState == NULL) { sndError("s-task:%s failed to open state for task", pTask->id.idStr); return -1; } else { sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } - + + if (pTask->info.fillHistory) { + restoreStreamTaskId(pTask, &taskId); + } int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); SReadHandle handle = { @@ -90,8 +87,8 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; - sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, - pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); + sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, + pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } char *p = NULL; @@ -99,18 +96,18 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer if (pTask->info.fillHistory) { sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " nextProcessVer:%" PRId64 - " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", - SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, - pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, - (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam); + " nextProcessVer:%" PRId64 + " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", + SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, + pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, + (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam); } else { sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " nextProcessVer:%" PRId64 - " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", - SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, - pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, - (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam); + " nextProcessVer:%" PRId64 + " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", + SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, + pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, + (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam); } return 0; } @@ -149,7 +146,7 @@ FAIL: return NULL; } -int32_t sndInit(SSnode * pSnode) { +int32_t sndInit(SSnode *pSnode) { resetStreamTaskStatus(pSnode->pMeta); startStreamTasks(pSnode->pMeta); return 0; From 84583580e25dd55d787bd524b14ee521b8da7d0b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 19:12:49 +0800 Subject: [PATCH 033/236] fix(stream): transfer the checkpoint trans id. --- include/libs/stream/tstream.h | 4 +- source/dnode/mnode/impl/inc/mndStream.h | 7 ++- source/dnode/mnode/impl/src/mndStream.c | 61 ++++++++++++----------- source/libs/stream/src/streamCheckpoint.c | 7 ++- source/libs/stream/src/streamMeta.c | 8 ++- source/libs/stream/src/streamTask.c | 1 + 6 files changed, 51 insertions(+), 37 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9b96a41516..9137965849 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -314,6 +314,7 @@ typedef struct SCheckpointInfo { int32_t checkpointNotReadyTasks; bool dispatchCheckpointTrigger; int64_t msgVer; + int32_t transId; } SCheckpointInfo; typedef struct SStreamStatus { @@ -635,6 +636,7 @@ typedef struct { int32_t nodeId; SEpSet mgmtEps; int32_t mnodeId; + int32_t transId; int64_t expireTime; } SStreamCheckpointSourceReq; @@ -677,8 +679,8 @@ typedef struct STaskStatusEntry { int64_t verStart; // start version in WAL, only valid for source task int64_t verEnd; // end version in WAL, only valid for source task int64_t processedVer; // only valid for source task - int32_t relatedHTask; // has related fill-history task int64_t activeCheckpointId; // current active checkpoint id + int32_t chkpointTransId; // checkpoint trans id bool checkpointFailed; // denote if the checkpoint is failed or not bool inputQChanging; // inputQ is changing or not int64_t inputQUnchangeCounter; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index b0f6273acc..4d06b16297 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -44,12 +44,11 @@ typedef struct SStreamTransMgmt { } SStreamTransMgmt; typedef struct SStreamExecInfo { - SArray * pNodeList; + SArray *pNodeList; int64_t ts; // snapshot ts SStreamTransMgmt transMgmt; - int64_t activeCheckpoint; // active check point id - SHashObj * pTaskMap; - SArray * pTaskList; + SHashObj *pTaskMap; + SArray *pTaskList; TdThreadMutex lock; } SStreamExecInfo; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d5bb8b2b92..b04a707728 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -62,7 +62,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter); static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq); static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq); static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, - int64_t streamId, int32_t taskId); + int64_t streamId, int32_t taskId, int32_t transId); static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static SArray *extractNodeListFromStream(SMnode *pMnode); @@ -997,13 +997,14 @@ static int32_t mndProcessStreamRemainChkptTmr(SRpcMsg *pReq) { } static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, - int64_t streamId, int32_t taskId) { + int64_t streamId, int32_t taskId, int32_t transId) { SStreamCheckpointSourceReq req = {0}; req.checkpointId = checkpointId; req.nodeId = nodeId; req.expireTime = -1; req.streamId = streamId; // pTask->id.streamId; req.taskId = taskId; // pTask->id.taskId; + req.transId = transId; int32_t code; int32_t blen; @@ -1093,7 +1094,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre void * buf; int32_t tlen; if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, - pTask->id.taskId) < 0) { + pTask->id.taskId, pTrans->id) < 0) { mndReleaseVgroup(pMnode, pVgObj); taosWUnLockLatch(&pStream->lock); goto _ERR; @@ -1162,7 +1163,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream void * buf; int32_t tlen; if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, chkptId, pTask->id.streamId, - pTask->id.taskId) < 0) { + pTask->id.taskId, pTrans->id) < 0) { mndReleaseVgroup(pMnode, pVgObj); taosWUnLockLatch(&pStream->lock); return -1; @@ -2837,39 +2838,36 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t le return TSDB_CODE_SUCCESS; } -static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { +static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, int32_t transId) { + int32_t code = TSDB_CODE_SUCCESS; + STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { mInfo("kill checkpoint transId:%d to reset task status", transId); mndKillTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); + } else { + mError("failed to acquire checkpoint trans:%d", transId); } - // set all tasks status to be normal, refactor later to be stream level, instead of vnode level. - SSdb * pSdb = pMnode->pSdb; - SStreamObj *pStream = NULL; - void * pIter = NULL; - while (1) { - pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); - if (pIter == NULL) { - break; - } - + SStreamObj *pStream = mndGetStreamObj(pMnode, streamId); + if (pStream == NULL) { + code = TSDB_CODE_STREAM_TASK_NOT_EXIST; + mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid); + } else { bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_RESET_NAME, false); if (conflict) { - mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name, - pStream->sourceDb, pStream->targetDb); - continue; - } - - mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid); - int32_t code = createStreamResetStatusTrans(pMnode, pStream); - if (code != TSDB_CODE_SUCCESS) { - sdbCancelFetch(pSdb, pIter); - return code; + mError("stream:%s other trans exists in DB:%s, dstTable:%s failed to start reset-status trans", pStream->name, + pStream->sourceDb, pStream->targetSTbName); + } else { + mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name, + pStream->uid, transId); + code = createStreamResetStatusTrans(pMnode, pStream); + mndReleaseStream(pMnode, pStream); } } - return 0; + + return code; } static SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { @@ -3007,6 +3005,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { bool checkpointFailed = false; int64_t activeCheckpointId = 0; + int64_t streamId = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, pReq->pCont, pReq->contLen); @@ -3049,7 +3048,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) { updateStageInfo(pTaskEntry, p->stage); - if (pTaskEntry->nodeId == SNODE_HANDLE) snodeChanged = true; + if (pTaskEntry->nodeId == SNODE_HANDLE) { + snodeChanged = true; + } } else { // task is idle for more than 50 sec. if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) { @@ -3073,6 +3074,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->checkpointFailed) { checkpointFailed = p->checkpointFailed; + streamId = p->id.streamId; } } } @@ -3111,9 +3113,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal - mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", - execInfo.activeCheckpoint); - mndResetStatusFromCheckpoint(pMnode, activeCheckpointId); + mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", activeCheckpointId); + mndResetStatusFromCheckpoint(pMnode, streamId, activeCheckpointId); } else { mInfo("not all vgroups are ready, wait for next HB from stream tasks"); } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 4a99d54de1..2a766f21ec 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -25,6 +25,7 @@ typedef struct { SStreamTask* pTask; } SAsyncUploadArg; + int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -34,6 +35,7 @@ int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckp if (tEncodeSEpSet(pEncoder, &pReq->mgmtEps) < 0) return -1; if (tEncodeI32(pEncoder, pReq->mnodeId) < 0) return -1; if (tEncodeI64(pEncoder, pReq->expireTime) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->transId) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; } @@ -47,6 +49,7 @@ int32_t tDecodeStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSo if (tDecodeSEpSet(pDecoder, &pReq->mgmtEps) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->mnodeId) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->expireTime) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->transId) < 0) return -1; tEndDecode(pDecoder); return 0; } @@ -149,6 +152,7 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo // 1. set task status to be prepared for check point, no data are allowed to put into inputQ. streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); + pTask->chkInfo.transId = pReq->transId; pTask->chkInfo.checkpointingId = pReq->checkpointId; pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); pTask->chkInfo.startTs = taosGetTimestampMs(); @@ -273,8 +277,9 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { pTask->chkInfo.failedId = 0; pTask->chkInfo.startTs = 0; // clear the recorded start time pTask->chkInfo.checkpointNotReadyTasks = 0; - // pTask->chkInfo.checkpointAlignCnt = 0; + pTask->chkInfo.transId = 0; pTask->chkInfo.dispatchCheckpointTrigger = false; + streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks if (clearChkpReadyMsg) { streamClearChkptReadyMsg(pTask); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 355e17db9a..d1df74b8d3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -940,6 +940,7 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { if (tEncodeI64(pEncoder, ps->verEnd) < 0) return -1; if (tEncodeI64(pEncoder, ps->activeCheckpointId) < 0) return -1; if (tEncodeI8(pEncoder, ps->checkpointFailed) < 0) return -1; + if (tEncodeI32(pEncoder, ps->chkpointTransId) < 0) return -1; } int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes); @@ -978,6 +979,7 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { if (tDecodeI64(pDecoder, &entry.verEnd) < 0) return -1; if (tDecodeI64(pDecoder, &entry.activeCheckpointId) < 0) return -1; if (tDecodeI8(pDecoder, (int8_t*)&entry.checkpointFailed) < 0) return -1; + if (tDecodeI32(pDecoder, &entry.chkpointTransId) < 0) return -1; entry.id.taskId = taskId; taosArrayPush(pReq->pTaskStatus, &entry); @@ -1103,6 +1105,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { if ((*pTask)->chkInfo.checkpointingId != 0) { entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; + entry.chkpointTransId = (*pTask)->chkInfo.transId; } if ((*pTask)->exec.pWalReader != NULL) { @@ -1350,10 +1353,13 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { } taosThreadMutexLock(&pTask->lock); - ETaskStatus s = streamTaskGetStatus(pTask, NULL); + char* p = NULL; + ETaskStatus s = streamTaskGetStatus(pTask, &p); if (s == TASK_STATUS__CK) { streamTaskSetCheckpointFailedId(pTask); stDebug("s-task:%s mark the checkpoint:%"PRId64" failed", pTask->id.idStr, pTask->chkInfo.checkpointingId); + } else { + stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, p); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 335f9d27d5..4d343a484d 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -796,5 +796,6 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->sinkDataSize = pSrc->sinkDataSize; pDst->activeCheckpointId = pSrc->activeCheckpointId; pDst->checkpointFailed = pSrc->checkpointFailed; + pDst->chkpointTransId = pSrc->chkpointTransId; } From 153221e1d607aa59733f15a561e8b466fa24e836 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 21 Dec 2023 19:46:33 +0800 Subject: [PATCH 034/236] fix: cache scan heap overflow and forbid DDD for to_char in windows --- include/common/ttime.h | 2 +- include/util/taoserror.h | 1 + source/common/src/ttime.c | 10 +++++++--- source/libs/executor/src/cachescanoperator.c | 21 ++++++++++++++++---- source/libs/scalar/src/sclfunc.c | 6 ++++-- source/util/src/terror.c | 1 + tests/system-test/2-query/last_cache_scan.py | 2 ++ 7 files changed, 33 insertions(+), 10 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 1dfa609064..ed4d1a9290 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -105,7 +105,7 @@ int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision); /// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again. /// @param out output buffer, should be initialized by memset /// @notes remember to free the generated formats -void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen); +int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen); /// @brief convert a formatted timestamp string to a timestamp /// @param format must null terminated /// @param [in, out] formats, see taosTs2Char diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 06e6b8d041..b49eb916f0 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -760,6 +760,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808) +#define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809) //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index ec00fce4dd..023a425df2 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1297,7 +1297,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) { } } -static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { +static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { int32_t size = taosArrayGetSize(formats); const char* start = s; for (int32_t i = 0; i < size; ++i) { @@ -1332,6 +1332,9 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_ s += 4; break; case TSFKW_DDD: +#ifdef WINDOWS + return TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED; +#endif sprintf(s, "%03d", tm->tm.tm_yday + 1); s += strlen(s); break; @@ -1486,6 +1489,7 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_ break; } } + return TSDB_CODE_SUCCESS; } /// @brief find s in arr case insensitively @@ -1889,14 +1893,14 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec return ret; } -void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { +int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { if (!*formats) { *formats = taosArrayInit(8, sizeof(TSFormatNode)); parseTsFormat(format, *formats); } struct STm tm; taosTs2Tm(ts, precision, &tm); - tm2char(*formats, &tm, out, outLen); + return tm2char(*formats, &tm, out, outLen); } int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 75fe3c51dd..9d4c20493a 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -54,10 +54,10 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM #define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) -static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) { +static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode* pScan) { SNode* pNode; int32_t idx = 0; - FOREACH(pNode, pTargets) { + FOREACH(pNode, pScan->pTargets) { if (nodeType(pNode) == QUERY_NODE_COLUMN) { SColumnNode* pCol = (SColumnNode*)pNode; SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx); @@ -65,6 +65,19 @@ static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) } idx++; } + + for (; idx < pBlock->pDataBlock->size; ++idx) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx); + if (pScan->scan.pScanPseudoCols) { + FOREACH(pNode, pScan->scan.pScanPseudoCols) { + STargetNode* pTarget = (STargetNode*)pNode; + if (pColInfo->info.slotId == pTarget->slotId) { + pColInfo->info.colId = 0; + break; + } + } + } + } } SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, @@ -127,12 +140,12 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe capacity = TMIN(totalTables, 4096); pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false); - setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode->pTargets); + setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode); blockDataEnsureCapacity(pInfo->pBufferredRes, capacity); } else { // by tags pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull); capacity = 1; // only one row output - setColIdForCacheReadBlock(pInfo->pRes, pScanNode->pTargets); + setColIdForCacheReadBlock(pInfo->pRes, pScanNode); } initResultSizeInfo(&pOperator->resultInfo, capacity); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 734d1e7d17..e713043b80 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1254,6 +1254,7 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu char * out = taosMemoryCalloc(1, TS_FORMAT_MAX_LEN + VARSTR_HEADER_SIZE); int32_t len; SArray *formats = NULL; + int32_t code = 0; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) { colDataSetNULL(pOutput->columnData, i); @@ -1272,14 +1273,15 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu } } int32_t precision = pInput[0].columnData->info.precision; - taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN); + code = taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN); + if (code) break; varDataSetLen(out, strlen(varDataVal(out))); colDataSetVal(pOutput->columnData, i, out, false); } if (formats) taosArrayDestroy(formats); taosMemoryFree(format); taosMemoryFree(out); - return TSDB_CODE_SUCCESS; + return code; } /** Time functions **/ diff --git a/source/util/src/terror.c b/source/util/src/terror.c index dc5f44cf43..6a32fed147 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -622,6 +622,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed for format mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED, "Func to_char failed for unsupported format") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 01795f6eef..2ef2e50a88 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -284,6 +284,8 @@ class TDTestCase: tdSql.checkData(0, 3, 1001) tdSql.checkData(0, 4, "2018-11-25 19:30:00.000") + tdSql.query("select last(ts) from meters partition by tbname") + tdSql.query("select last(ts) from meters partition by t1") sql_template = 'select %s from meters partition by tbname' select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts"] has_last_row_scan_res = [1,1,1] From 7466fcc39838935ef1a3782260b297774ac971f0 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 21 Dec 2023 19:52:40 +0800 Subject: [PATCH 035/236] fix: no more rm old log in resetlog --- source/util/src/tlog.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 184e18fc67..e113a95fcc 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -337,14 +337,10 @@ static int32_t taosOpenNewLogFile() { } void taosResetLog() { - char lastName[LOG_FILE_NAME_LEN + 20]; - sprintf(lastName, "%s.%d", tsLogObj.logName, tsLogObj.flag); - // force create a new log file tsLogObj.lines = tsNumOfLogLines + 10; taosOpenNewLogFile(); - (void)taosRemoveFile(lastName); uInfo("=================================="); uInfo(" reset log file "); From 0fab9a1827da20da21d12856c89329f7868e1f4a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 00:29:49 +0800 Subject: [PATCH 036/236] other: fix syntax error. --- source/dnode/mnode/impl/src/mndStream.c | 2 ++ source/dnode/mnode/impl/src/mndStreamTrans.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b04a707728..3ba6280f59 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -3006,6 +3006,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { bool checkpointFailed = false; int64_t activeCheckpointId = 0; int64_t streamId = 0; + int32_t transId = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, pReq->pCont, pReq->contLen); @@ -3075,6 +3076,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->checkpointFailed) { checkpointFailed = p->checkpointFailed; streamId = p->id.streamId; + transId = p->chkpointTransId; } } } diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index a7d4703c8c..b8a1c3d4ae 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -93,6 +93,8 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; + } else { + mDebug("not conflict with checkpoint trans, name:%s, continue create trans", pTransName); } } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) { @@ -100,6 +102,8 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char tInfo.name); return true; } + } else { + mDebug("stream:0x%"PRIx64" no conflict trans existed, continue create trans", streamUid); } if (lock) { From c9475060de3d834be1153e4dc6584b73441c070f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 09:34:49 +0800 Subject: [PATCH 037/236] fix(stream): fix race condition. --- source/dnode/mnode/impl/src/mndStreamTrans.c | 4 ++-- source/dnode/vnode/src/tq/tqStreamTask.c | 7 ++++++- source/dnode/vnode/src/tq/tqUtil.c | 8 ++++++-- source/libs/stream/src/streamMeta.c | 9 ++------- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index b8a1c3d4ae..8f94843500 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -90,7 +90,7 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) { if (strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) { - mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, + mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; } else { @@ -98,7 +98,7 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char } } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) { - mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, + mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 99cc3a36ea..4fc1c6cc01 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -126,10 +126,13 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t tqStopStreamTasks(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = TD_VID(pTq->pVnode); - int32_t num = taosArrayGetSize(pMeta->pTaskList); + streamMetaRLock(pMeta); + + int32_t num = taosArrayGetSize(pMeta->pTaskList); tqDebug("vgId:%d stop all %d stream task(s)", vgId, num); if (num == 0) { + streamMetaRUnLock(pMeta); return TSDB_CODE_SUCCESS; } @@ -149,6 +152,8 @@ int32_t tqStopStreamTasks(STQ* pTq) { } taosArrayDestroy(pTaskList); + + streamMetaRUnLock(pMeta); return 0; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index d3ea1f19e5..64c733d63d 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -39,6 +39,8 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { SStreamMeta* pMeta = pTq->pStreamMeta; int64_t stage = pMeta->stage; + streamMetaWLock(pMeta); + pMeta->stage = state.term; // mark the sign to send msg before close all tasks @@ -52,9 +54,11 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { state.term, stage, isLeader); streamMetaStartHb(pMeta); } else { - tqInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d", pMeta->vgId, state.term, stage, - isLeader); + tqInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, + state.term, stage, isLeader, pMeta->sendMsgBeforeClosing); } + + streamMetaWUnLock(pMeta); } static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d1df74b8d3..3504c97398 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1328,14 +1328,9 @@ int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, voi } SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { - SArray* pTaskList = NULL; - bool sendMsg = false; - - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - sendMsg = pMeta->sendMsgBeforeClosing; - streamMetaWUnLock(pMeta); + SArray* pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + bool sendMsg = pMeta->sendMsgBeforeClosing; if (!sendMsg) { stDebug("vgId:%d no need to send msg to mnode before closing tasks", pMeta->vgId); return pTaskList; From 16ad59fb18976b468b3de6b3a1ecf93ff642926d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 22 Dec 2023 12:12:15 +0800 Subject: [PATCH 038/236] fix(tsdb/cache): prepend ts col to load stt --- source/dnode/vnode/src/tsdb/tsdbCache.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 0f5511d658..d7fea4956d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -885,9 +885,17 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr int32_t code = 0; rocksdb_writebatch_t *wb = NULL; SArray *pTmpColArray = NULL; - int num_keys = TARRAY_SIZE(remainCols); - int16_t *aCols = taosMemoryMalloc(num_keys * sizeof(int16_t)); - int16_t *slotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); + + SIdxKey *idxKey = taosArrayGet(remainCols, 0); + if (idxKey->key.cid != PRIMARYKEY_TIMESTAMP_COL_ID) { + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = PRIMARYKEY_TIMESTAMP_COL_ID}; + + taosArrayInsert(remainCols, 0, &(SIdxKey){0, *key}); + } + + int num_keys = TARRAY_SIZE(remainCols); + int16_t *aCols = taosMemoryMalloc(num_keys * sizeof(int16_t)); + int16_t *slotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); for (int i = 0; i < num_keys; ++i) { SIdxKey *idxKey = taosArrayGet(remainCols, i); @@ -1875,7 +1883,7 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb .backward = 1, .pSttFileBlockIterArray = pr->pLDataIterArray, .pCols = aCols, - .numOfCols = nCols + 1, + .numOfCols = nCols, .loadTombFn = loadSttTomb, .pReader = pr, .idstr = pr->idstr, From 7dd6cb422774719c692ec477febd99fac147b27d Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Dec 2023 13:17:25 +0800 Subject: [PATCH 039/236] fix: move bi to community version --- source/libs/parser/CMakeLists.txt | 3 - source/libs/parser/src/parTranslater.c | 194 +++++++++++++++++++++++-- 2 files changed, 183 insertions(+), 14 deletions(-) diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 4e4a4def1d..c5ee1a00c4 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -4,9 +4,6 @@ IF (TD_ENTERPRISE) LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/parserView.c) ENDIF () -IF (TD_BI_SUPPORT) - LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/bi/src/biRewriteQuery.c) -ENDIF () add_library(parser STATIC ${PARSER_SRC}) target_include_directories( parser diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..0c41bc48b5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1104,11 +1104,192 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p return DEAL_RES_CONTINUE; } -#ifndef TD_ENTERPRISE +static void biMakeAliasNameInMD5(char* pExprStr, int32_t len, char* pAlias) { + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, pExprStr, len); + tMD5Final(&ctx); + char* p = pAlias; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } +} + +static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) { + SValueNode* valNode = NULL; + if (tableAlias != NULL) { + SValueNode* n = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + n->literal = tstrdup(tableAlias); + n->node.resType.type = TSDB_DATA_TYPE_BINARY; + n->node.resType.bytes = strlen(n->literal); + n->isDuration = false; + n->translate = false; + valNode = n; + } + + SFunctionNode* tbNameFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + tstrncpy(tbNameFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); + if (valNode != NULL) { + nodesListMakeAppend(&tbNameFunc->pParameterList, (SNode*)valNode); + } + snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), + (tableAlias)? "%s.tbname" : "%stbname", + (tableAlias)? tableAlias : ""); + strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); + + if (funcName == NULL) { + return (SNode*)tbNameFunc; + } else { + SFunctionNode* multiResFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN); + nodesListMakeAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); + + if (tsKeepColumnName) { + snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), + (tableAlias)? "%s.tbname" : "%stbname", + (tableAlias)? tableAlias : ""); + strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName); + } else { + snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), + tableAlias? "%s(%s.tbname)" : "%s(%stbname)", funcName, + tableAlias? tableAlias: ""); + biMakeAliasNameInMD5(multiResFunc->node.userAlias, strlen(multiResFunc->node.userAlias), multiResFunc->node.aliasName); + } + + return (SNode*)multiResFunc; + } +} + +static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, SListCell* pSelectListCell) { + SNodeList* pTbnameNodeList = nodesMakeList(); + + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (strcasecmp(pFunc->functionName, "last") == 0 || + strcasecmp(pFunc->functionName, "last_row") == 0 || + strcasecmp(pFunc->functionName, "first") == 0) { + SNodeList* pParams = pFunc->pParameterList; + SNode* pPara = NULL; + FOREACH(pPara, pParams) { + if (nodesIsStar(pPara)) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t n = taosArrayGetSize(pTables); + for (int32_t i = 0; i < n; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, NULL); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); + } + } else if (nodesIsTableStar(pPara)) { + char* pTableAlias = ((SColumnNode*)pPara)->tableAlias; + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, pTableAlias, &pTable); + if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, pTableAlias); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); + } + } + } + } + return TSDB_CODE_SUCCESS; +} + +// after translate from +// before translate select list +int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { + SNode* pNode = NULL; + SNodeList* pTbnameNodeList = nodesMakeList(); + WHERE_EACH(pNode, pSelect->pProjectionList) { + if (nodesIsStar(pNode)) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t n = taosArrayGetSize(pTables); + for (int32_t i = 0; i < n; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, NULL); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); + } + } else if (nodesIsTableStar(pNode)) { + char* pTableAlias = ((SColumnNode*)pNode)->tableAlias; + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, pTableAlias, &pTable); + if (TSDB_CODE_SUCCESS == code && + nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, pTableAlias); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); + } + } else if (nodeType(pNode) == QUERY_NODE_FUNCTION) { + biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell); + } + WHERE_NEXT; + } + + return TSDB_CODE_SUCCESS; +} + bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { + SColumnNode* pCol = (SColumnNode*)(*ppNode); + if ((strcasecmp(pCol->colName, "tbname") == 0) && + ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable && + QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + SFunctionNode* tbnameFuncNode = NULL; + tbnameFuncNode = (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0]!='\0') ? pCol->tableAlias : NULL); + tbnameFuncNode->node.resType = pCol->node.resType; + strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName); + strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias); + + nodesDestroyNode(*ppNode); + *ppNode = (SNode*)tbnameFuncNode; + return true; + } + return false; } -#endif + +int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + if (pStmt->pTags) { + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pTags) { + SColumnDefNode* pTag = (SColumnDefNode*)pNode; + if (strcasecmp(pTag->colName, "tbname") == 0) { + int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, "tbname can not used for tags in BI mode"); + return code; + } + } + } + if (pStmt->pCols) { + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pCols) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if (strcasecmp(pCol->colName, "tbname") == 0) { + int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "tbname can not used for columns in BI mode"); + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { if (NULL == pCxt->pCurrStmt || @@ -3178,10 +3359,6 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { return TSDB_CODE_SUCCESS; } -#ifndef TD_ENTERPRISE -int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } -#endif - static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { SNode* pNode = NULL; WHERE_EACH(pNode, pSelect->pProjectionList) { @@ -5743,11 +5920,6 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions return code; } -#ifndef TD_ENTERPRISE -int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { - return TSDB_CODE_SUCCESS; -} -#endif static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { if (NULL != strchr(pStmt->tableName, '.')) { From 90898a16e0393da66938541560585d8e3679d6ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 13:57:15 +0800 Subject: [PATCH 040/236] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqStreamTask.c | 4 ++-- source/libs/stream/src/streamMeta.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 4fc1c6cc01..fde5082212 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -141,8 +141,8 @@ int32_t tqStopStreamTasks(STQ* pTq) { int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { continue; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 3504c97398..151605493a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1210,7 +1210,7 @@ void metaHbToMnode(void* param, void* tmrId) { bool streamMetaTaskInTimer(SStreamMeta* pMeta) { bool inTimer = false; - streamMetaWLock(pMeta); + streamMetaRLock(pMeta); void* pIter = NULL; while (1) { @@ -1226,7 +1226,7 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { } } - streamMetaWUnLock(pMeta); + streamMetaRUnLock(pMeta); return inTimer; } @@ -1303,6 +1303,7 @@ void streamMetaRUnLock(SStreamMeta* pMeta) { void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); taosWLockLatch(&pMeta->lock); + stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { From e91afa7df2b948cfa3a69edf5a75e9fa3c1ded19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 14:51:04 +0800 Subject: [PATCH 041/236] fix(stream): fix deadlock. --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tq/tqStreamTask.c | 3 ++- source/libs/stream/src/streamMeta.c | 25 +++++++++++++----------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9137965849..9772c7ecf3 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -845,6 +845,7 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); +SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaClear(SStreamMeta* pMeta); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index fde5082212..63e71f8ee8 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -123,6 +123,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { return 0; } +// todo: createMsg to invoke this function in stream threads, to avoid blocking the syn thread int32_t tqStopStreamTasks(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = TD_VID(pTq->pVnode); @@ -142,7 +143,7 @@ int32_t tqStopStreamTasks(STQ* pTq) { for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { continue; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 151605493a..686cf841e1 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -607,22 +607,23 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { return (int32_t)size; } -SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { - streamMetaRLock(pMeta); - +SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { STaskId id = {.streamId = streamId, .taskId = taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - if (ppTask != NULL) { - if (!streamTaskShouldStop(*ppTask)) { - int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); - streamMetaRUnLock(pMeta); - stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); - return *ppTask; - } + if (ppTask == NULL || streamTaskShouldStop(*ppTask)) { + return NULL; } + int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); + return *ppTask; +} + +SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { + streamMetaRLock(pMeta); + SStreamTask* p = streamMetaAcquireTaskNoLock(pMeta, streamId, taskId); streamMetaRUnLock(pMeta); - return NULL; + return p; } void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) { @@ -1293,6 +1294,7 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); taosRLockLatch(&pMeta->lock); + ASSERT(pMeta->lock != 0x40000001); } void streamMetaRUnLock(SStreamMeta* pMeta) { @@ -1303,6 +1305,7 @@ void streamMetaRUnLock(SStreamMeta* pMeta) { void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); taosWLockLatch(&pMeta->lock); + ASSERT(pMeta->lock != 0x40000001); stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } From dce0a6c74a3218a98f0c3146ebd43e47b3ef92e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 15:24:37 +0800 Subject: [PATCH 042/236] fix(stream): remove invalid assert --- source/libs/stream/src/streamMeta.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 686cf841e1..b82021abf6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1294,7 +1294,6 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); taosRLockLatch(&pMeta->lock); - ASSERT(pMeta->lock != 0x40000001); } void streamMetaRUnLock(SStreamMeta* pMeta) { From 2de6d6c81d996ad8663721a93bbdd2c8d49339f5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 22 Dec 2023 16:06:59 +0800 Subject: [PATCH 043/236] enh: add TSDB_CODE_MND_TRANS_CTX_SWITCH --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndSync.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 15 ++++++++++----- source/util/src/terror.c | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 06e6b8d041..a4f56cf066 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -355,6 +355,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5) #define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal #define TSDB_CODE_MND_TRANS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7) +#define TSDB_CODE_MND_TRANS_CTX_SWITCH TAOS_DEF_ERROR_CODE(0, 0x03D8) #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) // mnode-mq diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 00cbcbce6c..0fc8dad420 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -539,7 +539,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { taosThreadMutexLock(&pMgmt->lock); pMgmt->errCode = 0; - if (pMgmt->transId != 0 /* && pMgmt->transId != transId*/) { + if (pMgmt->transId != 0) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosThreadMutexUnlock(&pMgmt->lock); rpcFreeCont(req.pCont); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 49b483114a..9e478f3aa5 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1096,7 +1096,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; - if (topHalf) return -1; + if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH; int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -1119,7 +1119,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode, topHalf)) return -1; + if (mndCannotExecuteTransAction(pMnode, topHalf)) return TSDB_CODE_MND_TRANS_CTX_SWITCH; int64_t signature = pTrans->id; signature = (signature << 32); @@ -1165,7 +1165,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio } static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { - if (!topHalf) return -1; + if (!topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH; pAction->rawWritten = 0; pAction->errCode = 0; mInfo("trans:%d, %s:%d confirm action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); @@ -1191,7 +1191,11 @@ static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); - if (code != 0) break; + if (code != 0) { + mInfo("trans:%d, action:%d not executed since %s. numOfActions:%d", pTrans->id, action, tstrerror(code), + numOfActions); + break; + } } return code; @@ -1199,9 +1203,10 @@ static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) { int32_t numOfActions = taosArrayGetSize(pArray); + int32_t code = 0; if (numOfActions == 0) return 0; - if (mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf) != 0) { + if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) { return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index dc5f44cf43..91e4c09334 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -286,6 +286,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Transaction context switch") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq From 7ce545bcd417b4d63531fc3a54b76e9bffeb2aeb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 16:14:49 +0800 Subject: [PATCH 044/236] refactor: refactor the lock type. --- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 20 ++++++++++---------- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 12 ++++++------ source/libs/stream/src/streamMeta.c | 9 ++++----- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9772c7ecf3..e7ac0e99c5 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -497,7 +497,7 @@ typedef struct SStreamMeta { int32_t role; bool sendMsgBeforeClosing; // send hb to mnode before close all tasks when switch to follower. STaskStartInfo startInfo; - SRWLatch lock; + TdThreadRwlock lock; int32_t walScanCounter; void* streamBackend; int64_t streamBackendRid; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index abe4c3f2fc..4963a0aa3b 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -240,23 +240,23 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTaskId *pId) { STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; - taosRLockLatch(&pMeta->lock); + streamMetaRLock(pMeta); SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask && *ppTask) { pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer; pItem->fetchResultVer = (*ppTask)->info.triggerParam; } - taosRUnLockLatch(&pMeta->lock); + streamMetaRUnLock(pMeta); } static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskId) { streamMetaUnregisterTask(pMeta, streamId, taskId); - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); if (streamMetaCommit(pMeta) < 0) { // persist to disk } - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); smaDebug("vgId:%d, rsma task:%" PRIi64 ",%d dropped, remain tasks:%d", pMeta->vgId, streamId, taskId, numOfTasks); } @@ -1301,14 +1301,14 @@ _checkpoint: checkpointBuilt = true; } - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); if (streamMetaSaveTask(pMeta, pTask)) { - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; taosHashCancelIterate(pInfoHash, infoHash); TSDB_CHECK_CODE(code, lino, _exit); } - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); smaDebug("vgId:%d, rsma commit, succeed to commit task:%p, submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64 ", table:%" PRIi64 ", level:%d", TD_VID(pVnode), pTask, pItem->submitReqVer, pItem->fetchResultVer, pRSmaInfo->suid, i + 1); @@ -1316,13 +1316,13 @@ _checkpoint: } } if (pMeta) { - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); if (streamMetaCommit(pMeta)) { - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); } if (checkpointBuilt) { smaInfo("vgId:%d, rsma commit, succeed to commit checkpoint:%" PRIi64, TD_VID(pVnode), checkpointId); diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index f966e90b9a..dda5173ad9 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -196,7 +196,7 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { int32_t code = 0; STQ* pTq = pWriter->pTq; - taosWLockLatch(&pTq->pStreamMeta->lock); + streamMetaWLock(pTq->pStreamMeta); tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode)); if (rollback) { tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); @@ -212,14 +212,14 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { taosMemoryFree(pWriter); goto _err; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); taosMemoryFree(pWriter); return code; _err: tqError("vgId:%d, vnode stream-task snapshot writer failed to close since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code)); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); return code; } @@ -240,13 +240,13 @@ int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t tDecoderClear(&decoder); int64_t key[2] = {taskId.streamId, taskId.taskId}; - taosWLockLatch(&pTq->pStreamMeta->lock); + streamMetaWLock(pTq->pStreamMeta); if (tdbTbUpsert(pTq->pStreamMeta->pTaskDb, key, sizeof(int64_t) << 1, (uint8_t*)pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr), pTq->pStreamMeta->txn) < 0) { - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); return -1; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); } else if (pHdr->type == SNAP_DATA_STREAM_TASK_CHECKPOINT) { // do nothing } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b82021abf6..665288cf7b 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1293,24 +1293,23 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); - taosRLockLatch(&pMeta->lock); + taosThreadRwlockRdlock(&pMeta->lock); } void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); - taosRUnLockLatch(&pMeta->lock); + taosThreadRwlockUnlock(&pMeta->lock); } void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); - taosWLockLatch(&pMeta->lock); - ASSERT(pMeta->lock != 0x40000001); + taosThreadRwlockWrlock(&pMeta->lock); stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wunlock", pMeta->vgId); - taosWUnLockLatch(&pMeta->lock); + taosThreadRwlockUnlock(&pMeta->lock); } static void execHelper(struct SSchedMsg* pSchedMsg) { From 212c1ebb3c139a10881c15cd3e6bb8af7375b3f8 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 21 Dec 2023 19:52:40 +0800 Subject: [PATCH 045/236] udpate test case test_hot_refresh_configurations --- source/util/src/tlog.c | 4 ---- .../system-test/0-others/test_hot_refresh_configurations.py | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 184e18fc67..e113a95fcc 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -337,14 +337,10 @@ static int32_t taosOpenNewLogFile() { } void taosResetLog() { - char lastName[LOG_FILE_NAME_LEN + 20]; - sprintf(lastName, "%s.%d", tsLogObj.logName, tsLogObj.flag); - // force create a new log file tsLogObj.lines = tsNumOfLogLines + 10; taosOpenNewLogFile(); - (void)taosRemoveFile(lastName); uInfo("=================================="); uInfo(" reset log file "); diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index cbde8c060e..59222281ae 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -215,6 +215,12 @@ class TDTestCase: self.svr_check(item["name"], item["alias"], item["except_values"], True) else: raise Exception(f"unknown key: {key}") + # reset log + path = os.sep.join([tdDnodes.getDnodesRootDir(), "dnode1", "log", "taosdlog.*"]) + tdSql.execute("alter all dnodes 'resetlog';") + r = subprocess.Popen("cat {} | grep 'reset log file'".format(path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = r.communicate() + assert('reset log file' in stdout.decode()) def stop(self): tdSql.close() From 536c002ae42a1f8254bbb0928f9559e9708766d9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 22 Dec 2023 16:48:57 +0800 Subject: [PATCH 046/236] test: modify test of last_cache_scan.py --- tests/pytest/util/sql.py | 9 ++-- tests/system-test/2-query/last_cache_scan.py | 53 ++++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 5ed985187a..f8a49983e1 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -128,7 +128,7 @@ class TDSql: if expectErrInfo == self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) else: if expectedErrno != None: if expectedErrno in self.errno: @@ -140,7 +140,7 @@ class TDSql: if expectErrInfo in self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) return self.error_info @@ -207,6 +207,7 @@ class TDSql: caller = inspect.getframeinfo(inspect.stack()[1][0]) if i < queryTimes: error_info = repr(e) + print(error_info) self.error_info = ','.join(error_info[error_info.index('(')+1:-1].split(",")[:-1]).replace("'","") self.queryRows = 0 self.queryCols = 0 @@ -217,13 +218,13 @@ class TDSql: if expectErrInfo == self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) else: if expectErrInfo != None: if expectErrInfo in self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) return self.error_info elif i == queryTimes: diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 0457475ee7..3504ca62d3 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: @@ -369,49 +369,48 @@ class TDTestCase: tdSql.checkData(0, 10, None) def test_cache_scan_last_row_with_drop_column2(self): - tdSql.query('select last_row(c1) from meters') + tdSql.query('select last_row(c2) from meters') print(str(tdSql.queryResult)) tdSql.checkCols(1) - p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"]) p.check_returncode() - tdSql.query('select last_row(c1) from meters', queryTimes=1) - print(str(tdSql.queryResult)) - tdSql.checkCols(1) + tdSql.query_success_failed("select ts, last_row(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") + tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) + tdSql.checkRows(1) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) def test_cache_scan_last_row_with_partition_by(self): tdSql.query('select last(c1) from meters partition by t1') print(str(tdSql.queryResult)) tdSql.checkCols(1) - tdSql.checkRows(5) - p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + tdSql.checkRows(2) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c2 int"]) p.check_returncode() - tdSql.query('select last_row(c1) from meters partition by t1', queryTimes=1) + tdSql.query_success_failed('select last(c1) from meters partition by t1', queryTimes=10, expectErrInfo="Invalid column name: c1") + tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) - tdSql.checkCols(1) - tdSql.checkRows(5) + tdSql.checkRows(1) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) - tdSql.checkData(1, 0, None) - tdSql.checkData(2, 0, None) - tdSql.checkData(3, 0, None) - tdSql.checkData(4, 0, None) + tdSql.checkData(0, 1, None) + def test_cache_scan_last_row_with_partition_by_tbname(self): - tdSql.query('select last(c1) from meters partition by tbname', queryTimes=1) + tdSql.query('select last(c2) from meters partition by tbname') print(str(tdSql.queryResult)) tdSql.checkCols(1) tdSql.checkRows(10) - p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"]) p.check_returncode() - tdSql.query('select last_row(c1) from meters partition by tbname', queryTimes=1) + tdSql.query_success_failed('select last_row(c2) from meters partition by tbname', queryTimes=10, expectErrInfo="Invalid column name: c2") + tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) - tdSql.checkCols(1) - tdSql.checkRows(10) + tdSql.checkRows(1) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) - tdSql.checkData(1, 0, None) - tdSql.checkData(2, 0, None) - tdSql.checkData(3, 0, None) - tdSql.checkData(4, 0, None) + tdSql.checkData(0, 1, None) @@ -423,9 +422,9 @@ class TDTestCase: self.test_cache_scan_with_drop_and_add_column2() #self.test_cache_scan_with_drop_column() #self.test_cache_scan_last_row_with_drop_column() - #self.test_cache_scan_last_row_with_drop_column2() - #self.test_cache_scan_last_row_with_partition_by() - #self.test_cache_scan_last_row_with_partition_by_tbname() + self.test_cache_scan_last_row_with_drop_column2() + self.test_cache_scan_last_row_with_partition_by() + self.test_cache_scan_last_row_with_partition_by_tbname() def stop(self): tdSql.close() From a71fabb24fdcdfc105d04ee05a1a8e376fe9b292 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 22 Dec 2023 17:50:55 +0800 Subject: [PATCH 047/236] coverage: remove tfunctionInt.c --- source/libs/function/inc/tfunctionInt.h | 2 -- source/libs/function/src/tfunctionInt.c | 30 ------------------------- 2 files changed, 32 deletions(-) diff --git a/source/libs/function/inc/tfunctionInt.h b/source/libs/function/inc/tfunctionInt.h index 821544106f..b4c48abf37 100644 --- a/source/libs/function/inc/tfunctionInt.h +++ b/source/libs/function/inc/tfunctionInt.h @@ -28,8 +28,6 @@ extern "C" { #include "tudf.h" #include "tvariant.h" -bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); - /** * the numOfRes should be kept, since it may be used later * and allow the ResultInfo to be re initialized diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index 3da5d63fa1..80b869b2d2 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -27,33 +27,3 @@ #include "tpercentile.h" #include "ttszip.h" #include "tudf.h" - -int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { - int32_t maxRows = 0; - - for (int32_t j = 0; j < num; ++j) { - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { - maxRows = pResInfo->numOfRes; - } - } - - blockDataEnsureCapacity(pResBlock, maxRows); - for (int32_t i = 0; i < num; ++i) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[i]); - if (pResInfo->numOfRes == 0) { - for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataSetVal(pCol, j, NULL, true); // TODO add set null data api - } - } else { - for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataSetVal(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); - } - } - } - - pResBlock->info.rows = maxRows; - return maxRows; -} From e13b3b9ceb39da59e60742988447c5c0bd0215c2 Mon Sep 17 00:00:00 2001 From: fullhonest Date: Fri, 22 Dec 2023 17:51:16 +0800 Subject: [PATCH 048/236] Fix TD-27989: percentile crash --- source/libs/function/src/tpercentile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 8101b342a4..93008b565a 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -142,13 +142,13 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { } // divide the value range into 1024 buckets - uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; + uint64_t span = (uint64_t)(pBucket->range.i64MaxVal - pBucket->range.i64MinVal); if (span < pBucket->numOfSlots) { int64_t delta = v - pBucket->range.i64MinVal; index = (delta % pBucket->numOfSlots); } else { double slotSpan = ((double)span) / pBucket->numOfSlots; - uint64_t delta = v - pBucket->range.i64MinVal; + uint64_t delta = (uint64_t)(v - pBucket->range.i64MinVal); index = (int32_t)(delta / slotSpan); if (v == pBucket->range.i64MaxVal || index == pBucket->numOfSlots) { From b175a4b7b3cc7db8400e00547d421d898e486d0f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 17:56:16 +0800 Subject: [PATCH 049/236] refactor(stream): async stop tasks, and do some internal refactor. --- include/dnode/vnode/tqCommon.h | 1 - include/libs/stream/tstream.h | 11 +- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/inc/tq.h | 3 - source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tq/tqStreamTask.c | 43 +++----- source/dnode/vnode/src/tq/tqUtil.c | 26 +---- source/dnode/vnode/src/tqCommon/tqCommon.c | 67 +----------- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- source/libs/stream/src/streamMeta.c | 116 +++++++++++++++++++++ 12 files changed, 150 insertions(+), 125 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index da9dcf1c04..da2325f006 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -30,7 +30,6 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); -int32_t startStreamTasks(SStreamMeta* pMeta); int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e7ac0e99c5..2d2db5c1dc 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -53,6 +53,7 @@ extern "C" { #define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) #define STREAM_EXEC_START_ALL_TASKS_ID (-2) #define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) +#define STREAM_EXEC_STOP_ALL_TASKS_ID (-4) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -785,8 +786,6 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ int64_t* oldStage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); -SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); - bool streamTaskAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); @@ -821,6 +820,7 @@ int32_t streamTaskReleaseState(SStreamTask* pTask); int32_t streamTaskReloadState(SStreamTask* pTask); void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId); void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key); void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask); void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); @@ -851,10 +851,8 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaClear(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key); void streamMetaStartHb(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta); int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, @@ -864,6 +862,11 @@ void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta); void streamMetaResetStartInfo(STaskStartInfo* pMeta); +SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); +void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); +int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); +int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); +int32_t streamMetaStopAllTasks(SStreamMeta* pMeta); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ba6280f59..25f51b85df 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -3116,7 +3116,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", activeCheckpointId); - mndResetStatusFromCheckpoint(pMnode, streamId, activeCheckpointId); + mndResetStatusFromCheckpoint(pMnode, streamId, transId); } else { mInfo("not all vgroups are ready, wait for next HB from stream tasks"); } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index f0fe662025..2225a07d3a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -151,7 +151,7 @@ FAIL: int32_t sndInit(SSnode * pSnode) { tqStreamTaskResetStatus(pSnode->pMeta); - startStreamTasks(pSnode->pMeta); + streamMetaStartAllTasks(pSnode->pMeta); return 0; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index cf57623a43..0ef29fcb3a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -152,9 +152,6 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data); char* tqOffsetBuildFName(const char* path, int32_t fVer); int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); -// tqStream -int32_t tqStopStreamTasks(STQ* pTq); - // tq util int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index f3b495675d..84f51827ba 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -232,6 +232,7 @@ int tqPushMsg(STQ*, tmsg_t msgType); int tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg); int tqUnregisterPushHandle(STQ* pTq, void* pHandle); int tqScanWalAsync(STQ* pTq, bool ckPause); +int32_t tqStopStreamTasksAsync(STQ* pTq); int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp); int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02c9b61432..e8e88e52f7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1076,6 +1076,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { if(code == 0 && taskId > 0){ tqScanWalAsync(pTq, false); } + return code; } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 63e71f8ee8..2bb2609ae9 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -109,8 +109,8 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { return -1; } - tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, restored:%d", vgId, numOfTasks, - alreadyRestored); + tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, vnd restored:%d", vgId, + numOfTasks, alreadyRestored); pRunReq->head.vgId = vgId; pRunReq->streamId = 0; @@ -123,38 +123,25 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { return 0; } -// todo: createMsg to invoke this function in stream threads, to avoid blocking the syn thread -int32_t tqStopStreamTasks(STQ* pTq) { +int32_t tqStopStreamTasksAsync(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; - int32_t vgId = TD_VID(pTq->pVnode); + int32_t vgId = pMeta->vgId; - streamMetaRLock(pMeta); - - int32_t num = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d stop all %d stream task(s)", vgId, num); - if (num == 0) { - streamMetaRUnLock(pMeta); - return TSDB_CODE_SUCCESS; + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to stop tasks, code:%s", vgId, terrstr()); + return -1; } - // send hb msg to mnode before closing all tasks. - SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); - int32_t numOfTasks = taosArrayGetSize(pTaskList); + tqDebug("vgId:%d create msg to stop tasks", vgId); - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { - continue; - } + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = STREAM_EXEC_STOP_ALL_TASKS_ID; - streamTaskStop(pTask); - streamMetaReleaseTask(pMeta, pTask); - } - - taosArrayDestroy(pTaskList); - - streamMetaRUnLock(pMeta); + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); return 0; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 64c733d63d..d18455d221 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -35,30 +35,8 @@ int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { } void tqUpdateNodeStage(STQ* pTq, bool isLeader) { - SSyncState state = syncGetState(pTq->pVnode->sync); - SStreamMeta* pMeta = pTq->pStreamMeta; - int64_t stage = pMeta->stage; - - streamMetaWLock(pMeta); - - pMeta->stage = state.term; - - // mark the sign to send msg before close all tasks - if ((!isLeader) && (pMeta->role == NODE_ROLE_LEADER)) { - pMeta->sendMsgBeforeClosing = true; - } - - pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; - if (isLeader) { - tqInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, - state.term, stage, isLeader); - streamMetaStartHb(pMeta); - } else { - tqInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, - state.term, stage, isLeader, pMeta->sendMsgBeforeClosing); - } - - streamMetaWUnLock(pMeta); + SSyncState state = syncGetState(pTq->pVnode->sync); + streamMetaUpdateStageRole(pTq->pStreamMeta, state.term, isLeader); } static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 0864afdf2c..af9363e363 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -598,66 +598,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen return 0; } -int32_t startStreamTasks(SStreamMeta* pMeta) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = pMeta->vgId; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - SArray* pTaskList = NULL; - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - taosHashClear(pMeta->startInfo.pReadyTaskSet); - taosHashClear(pMeta->startInfo.pFailedTaskSet); - pMeta->startInfo.startTs = taosGetTimestampMs(); - streamMetaWUnLock(pMeta); - - // broadcast the check downstream tasks msg - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { - streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, - taosGetTimestampMs(), false); - continue; - } - - // fill-history task can only be launched by related stream tasks. - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - if (pTask->status.downstreamReady == 1) { - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", - pTask->id.idStr); - streamLaunchFillHistoryTask(pTask); - } - - streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, - pTask->execInfo.start, true); - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); - if (ret != TSDB_CODE_SUCCESS) { - code = ret; - } - - streamMetaReleaseTask(pMeta, pTask); - } - - taosArrayDestroy(pTaskList); - return code; -} - int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); @@ -723,7 +663,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { streamMetaWUnLock(pMeta); tqInfo("vgId:%d start all stream tasks after reload tasks from disk", vgId); - startStreamTasks(pMeta); + streamMetaStartAllTasks(pMeta); } else { streamMetaResetStartInfo(&pMeta->startInfo); streamMetaWUnLock(pMeta); @@ -741,11 +681,14 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t vgId = pMeta->vgId; if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { - startStreamTasks(pMeta); + streamMetaStartAllTasks(pMeta); return 0; } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { restartStreamTasks(pMeta, isLeader); return 0; + } else if (taskId == STREAM_EXEC_STOP_ALL_TASKS_ID) { + streamMetaStopAllTasks(pMeta); + return 0; } SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 48811b2925..048092131d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -594,7 +594,7 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { if (pVnode->pTq) { tqUpdateNodeStage(pVnode->pTq, false); - tqStopStreamTasks(pVnode->pTq); + tqStopStreamTasksAsync(pVnode->pTq); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 665288cf7b..0702a908ab 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1367,3 +1367,119 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { pMeta->sendMsgBeforeClosing = false; return pTaskList; } + +void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) { + streamMetaWLock(pMeta); + + int64_t prevStage = pMeta->stage; + pMeta->stage = stage; + + // mark the sign to send msg before close all tasks + if ((!isLeader) && (pMeta->role == NODE_ROLE_LEADER)) { + pMeta->sendMsgBeforeClosing = true; + } + + pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; + + if (isLeader) { + stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, + prevStage, stage, isLeader); + streamMetaStartHb(pMeta); + } else { + stInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, + prevStage, stage, isLeader, pMeta->sendMsgBeforeClosing); + } + + streamMetaWUnLock(pMeta); +} + +int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { + streamMetaRLock(pMeta); + + int32_t num = taosArrayGetSize(pMeta->pTaskList); + stDebug("vgId:%d stop all %d stream task(s)", pMeta->vgId, num); + if (num == 0) { + streamMetaRUnLock(pMeta); + return TSDB_CODE_SUCCESS; + } + + // send hb msg to mnode before closing all tasks. + SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); + int32_t numOfTasks = taosArrayGetSize(pTaskList); + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + streamTaskStop(pTask); + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + + streamMetaRUnLock(pMeta); + return 0; +} + +int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + stDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); + pMeta->startInfo.startTs = taosGetTimestampMs(); + streamMetaWUnLock(pMeta); + + // broadcast the check downstream tasks msg + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, + taosGetTimestampMs(), false); + continue; + } + + // fill-history task can only be launched by related stream tasks. + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + if (pTask->status.downstreamReady == 1) { + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", + pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } + + streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, + pTask->execInfo.start, true); + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); + if (ret != TSDB_CODE_SUCCESS) { + code = ret; + } + + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return code; +} \ No newline at end of file From e3098accc4c7dc222670311e887d01437d63907c Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 13 Dec 2023 23:25:37 +0800 Subject: [PATCH 050/236] support count empty table --- .gitignore | 4 +- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 1 + source/dnode/vnode/src/sma/smaRollup.c | 1 + source/libs/executor/inc/executorInt.h | 4 +- source/libs/executor/src/scanoperator.c | 102 +++++++++- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 9 +- source/libs/nodes/src/nodesMsgFuncs.c | 7 +- source/libs/parser/src/parTranslater.c | 1 + source/libs/planner/src/planPhysiCreater.c | 22 +++ tests/parallel_test/cases.task | 1 + tests/system-test/2-query/count.py | 33 ++-- tests/system-test/2-query/group_partition.py | 183 ++++++++++++++++++ .../2-query/nestedQueryInterval.py | 26 +-- 15 files changed, 359 insertions(+), 37 deletions(-) create mode 100644 tests/system-test/2-query/group_partition.py diff --git a/.gitignore b/.gitignore index 08e3d57717..f8b42f9176 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -build/ +*build/ compile_commands.json CMakeSettings.json .cache @@ -132,3 +132,5 @@ tools/taos-tools tools/taosws-rs tags .clangd +*CMakeCache* +*CMakeFiles* diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b99a97a194..4b3c846389 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -434,6 +434,7 @@ typedef struct STableScanPhysiNode { bool assignBlockUid; int8_t igCheckUpdate; bool filesetDelimited; + bool needCountEmptyTable; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5c5172b9cd..fdf598153f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -363,6 +363,7 @@ typedef struct SSelectStmt { bool hasLastRowFunc; bool hasLastFunc; bool hasTimeLineFunc; + bool hasCountFunc; bool hasUdaf; bool hasStateKey; bool onlyHasKeepOrderFunc; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index abe4c3f2fc..5b33451be5 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -750,6 +750,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); + pReq = NULL; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e3e504cdbc..2523b87cfb 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -262,6 +262,7 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; + SHashObj* pValuedTables; // non empty table uids SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; @@ -269,8 +270,9 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - // TsdReader readerAPI; bool filesetDelimited; + bool needCountEmptyTable; + bool processingEmptyTable; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..1889492aa0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,6 +655,50 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } + +// record processed (non empty) table +static int32_t insertTableToProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { + if (!pTableScanInfo->needCountEmptyTable) { + return TSDB_CODE_SUCCESS; + } + if (NULL == pTableScanInfo->pValuedTables) { + int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); + pTableScanInfo->pValuedTables = + taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (NULL == pTableScanInfo->pValuedTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + taosHashPut(pTableScanInfo->pValuedTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, + sizeof(pTableScanInfo->scanTimes)); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { + STableScanInfo* pTableScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSDataBlock* pBlock = pTableScanInfo->pResBlock; + + blockDataEmpty(pBlock); + pBlock->info.rows = 1; + pBlock->info.id.uid = tbInfo->uid; + pBlock->info.id.groupId = pOperator->dynamicTask ? tbInfo->uid : tbInfo->groupId; + + // only one row: set all col data to null & hasNull + int32_t col_num = blockDataGetNumOfCols(pBlock); + for (int32_t i = 0; i < col_num; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + colDataSetNULL(pColInfoData, 0); + } + + // set tag/tbname + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); + + pOperator->resultInfo.totalRows++; + return pBlock; +} + static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -722,7 +766,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { +static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKeyInfo* pList, int32_t num) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -736,6 +780,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { + insertTableToProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -764,6 +809,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { + insertTableToProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -780,6 +826,39 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } } + if (pTableScanInfo->needCountEmptyTable) { + if (num == 0 && 0 == taosHashGetSize(pTableScanInfo->pValuedTables)) { + // table by table, num is 0 + if (!pTableScanInfo->processingEmptyTable) { + pTableScanInfo->processingEmptyTable = true; + // current table is empty, fill result block info & return + const STableKeyInfo* info = tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->currentTable); + return getBlockForEmptyTable(pOperator, info); + } + + } else if (num > taosHashGetSize(pTableScanInfo->pValuedTables)) { + // group by group, num >= 1 + if (!pTableScanInfo->processingEmptyTable) { + pTableScanInfo->processingEmptyTable = true; + pTableScanInfo->currentTable = 0; + } + if (pTableScanInfo->currentTable < num) { + // loop: get empty table uid & process + while (pTableScanInfo->currentTable < num) { + const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; + if (pTableScanInfo->pValuedTables && + NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { + } else { + return getBlockForEmptyTable(pOperator, info); + } + } + } + } + + pTableScanInfo->processingEmptyTable = false; + } + taosHashClear(pTableScanInfo->pValuedTables); + return NULL; } @@ -861,7 +940,7 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -876,15 +955,16 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int32_t num = 0; + STableKeyInfo* pList = NULL; if (pInfo->currentGroupId == -1) { - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo) || numOfTables == 0) { setOperatorCompleted(pOperator); return NULL; } - int32_t num = 0; - STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); @@ -899,9 +979,11 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } + } else { + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); } - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -923,7 +1005,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - + if (pOperator->pOperatorGetParam) { pOperator->dynamicTask = true; int32_t code = createTableListInfoFromParam(pOperator); @@ -952,7 +1034,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo tInfo = {0}; while (1) { - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } @@ -1012,6 +1094,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); + taosHashCleanup(pTableScanInfo->pValuedTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1075,6 +1158,9 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; + pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; + pInfo->processingEmptyTable = false; + pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 97438b84a6..8a154dcf00 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -652,6 +652,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(igExpired); COPY_SCALAR_FIELD(filesetDelimited); + COPY_SCALAR_FIELD(needCountEmptyTable); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c445af61cc..402a6c6e3d 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1841,6 +1841,7 @@ static const char* jkTableScanPhysiPlanSubtable = "Subtable"; static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; +static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1912,7 +1913,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanFilesetDelimited, pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, pNode->needCountEmptyTable); + } return code; } @@ -1986,7 +1989,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanFilesetDelimited, &pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, &pNode->needCountEmptyTable); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index d6eb3360aa..b36e2695f6 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2170,6 +2170,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueBool(pEncoder, pNode->filesetDelimited); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->needCountEmptyTable); + } return code; } @@ -2251,7 +2254,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj) if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueBool(pDecoder, &pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->needCountEmptyTable); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..98799d3d1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1902,6 +1902,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { if (NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt)) { SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt; pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); + pSelect->hasCountFunc = pSelect->hasCountFunc ? true : (FUNCTION_TYPE_COUNT == pFunc->funcType); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); if (fmIsIndefiniteRowsFunc(pFunc->funcId)) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index d1fbd0681d..901927b1d1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,6 +587,27 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } +static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + // refuse interval + if (pScanLogicNode->interval > 0) { + return false; + } + SNode* pRoot = pCxt->pPlanCxt->pAstRoot; + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + SSelectStmt* pSelect = (SSelectStmt*)pRoot; + // select & count + if (pSelect->hasCountFunc) { + // key only accept tag/tbname + if (NULL != pSelect->pGroupByList) { + return !keysHasCol(pSelect->pGroupByList); + } else if (NULL != pSelect->pPartitionByList) { + return !keysHasCol(pSelect->pPartitionByList); + } + } + } + return false; +} + static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, @@ -623,6 +644,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; + pTableScan->needCountEmptyTable = calcNeedCountEmpty(pCxt, pScanLogicNode); int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..e63f86743b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -347,6 +347,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 40d9b3ff8b..7e99f9e4dd 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -57,8 +57,11 @@ class TDTestCase: tdSql.query(f'select count(*) from {self.stbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + rows = [2, 0] function_names = ['count', 'hyperloglog'] - for function_name in function_names: + for i in range(2): + function_name = function_names[i] + row = rows[i] tdSql.query(f'select {function_name}(tbname) from {self.stbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -93,17 +96,17 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.stbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by t0') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by t0') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.stbname} group by c1)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -113,17 +116,24 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1 interval(1s)') tdSql.checkRows(0) - tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname)') + tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + if 'count' == function_name: + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + elif 'hyperloglog' == function_name: + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 0, 0) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + rows = [1, 0] function_names = ['count', 'hyperloglog'] - for function_name in function_names: + for i in range(2): + function_name = function_names[i] + row = rows[i] tdSql.query(f'select {function_name}(tbname) from {self.ntbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -158,7 +168,7 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.ntbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.ntbname} group by c1)') @@ -170,10 +180,11 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by c1 interval(1s)') tdSql.checkRows(0) - tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) tdSql.checkData(0, 1, None) + def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') tdSql.checkEqual(tdSql.queryResult[0][0],tbnum*rownum) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py new file mode 100644 index 0000000000..eb5f069b3d --- /dev/null +++ b/tests/system-test/2-query/group_partition.py @@ -0,0 +1,183 @@ +# author : bobliu +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.row_nums = 10 + self.tb_nums = 10 + self.ts = 1537146000000 + self.dbname = "db" + self.stable = "stb" + + def prepare_db(self): + tdSql.execute(f" use {self.dbname} ") + tdSql.execute(f" create stable {self.dbname}.{self.stable} (ts timestamp , c1 int , c2 bigint , c3 float , c4 double , c5 smallint , c6 tinyint , c7 bool , c8 binary(36) , c9 nchar(36) , uc1 int unsigned,\ + uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags(t1 timestamp , t2 int , t3 bigint , t4 float , t5 double , t6 smallint , t7 tinyint , t8 bool , t9 binary(36)\ + , t10 nchar(36) , t11 int unsigned , t12 bigint unsigned ,t13 smallint unsigned , t14 tinyint unsigned ) ") + + for i in range(self.tb_nums): + tbname = f"{self.dbname}.sub_{self.stable}_{i}" + ts = self.ts + i*10000 + tdSql.execute(f"create table {tbname} using {self.dbname}.{self.stable} tags ({ts} , {i} , {i}*10 ,{i}*1.0,{i}*1.0 , 1 , 2, 'true', 'binary_{i}' ,'nchar_{i}',{i},{i},10,20 )") + + def insert_db(self, tb_nums, row_nums): + for i in range(tb_nums): + tbname = f"{self.dbname}.sub_{self.stable}_{i}" + ts_base = self.ts + i*10000 + for row in range(row_nums): + ts = ts_base + row*1000 + tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") + + + def test_groupby(self, check_num, real_num): + # tbname + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + # having filter out empty + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # tag + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by t2 ") + tdSql.checkRows(check_num) + + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") + tdSql.checkRows(check_num) + + # having + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # col where filter nothing + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + # tdSql.checkRows(check_num) + + ############### same with old ############### + # col where filter all + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + # tdSql.checkRows(0) + + # col where filter part + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") + tdSql.checkRows(real_num) + + # col + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(real_num) + + # count + sum(col) + tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(real_num) + + tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") + num = 0 + if real_num > 0: + num = self.row_nums + tdSql.checkRows(num) + + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} group by ts ") + tdSql.checkRows(real_num * self.row_nums) + + # col + tag + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") + tdSql.checkRows(real_num * self.row_nums) + + + def test_partitionby(self, check_num, real_num): + tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + # having filter out empty + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + #tag + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} partition by t2 ") + tdSql.checkRows(check_num) + + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") + tdSql.checkRows(check_num) + + # having + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # col where filter nothing + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") + # tdSql.checkRows(check_num) + + ############### same with old ############### + # col where filter all + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") + # tdSql.checkRows(0) + + # col where filter part + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") + tdSql.checkRows(real_num) + + #col + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(real_num) + + tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") + num = 0 + if real_num > 0: + num = self.row_nums + tdSql.checkRows(num) + + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} partition by ts ") + tdSql.checkRows(real_num * self.row_nums) + + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") + tdSql.checkRows(real_num * self.row_nums) + + def test_error(self): + tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") + tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") + + + def run(self): + tdSql.prepare() + self.prepare_db() + check_num = self.tb_nums + self.test_groupby(check_num, 0) + self.test_partitionby(check_num, 0) + # insert into half of tables + real_num = 5 + self.insert_db(real_num, self.row_nums) + self.test_groupby(check_num, real_num) + self.test_partitionby(check_num, real_num) + + # test old version before changed + # self.test_groupby(0, 0) + # self.test_partitionby(0, 0) + # self.insert_db(5, self.row_nums) + # self.test_groupby(5, 5) + # self.test_partitionby(5, 5) + + self.test_error() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index c16fc03c27..b6ef50dcda 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -1112,13 +1112,13 @@ class TDTestCase: def TS_3932(self): tdLog.debug("test insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 100) tdSql.checkData(1, 1, 200) tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 101) tdSql.checkData(1, 1, 200) @@ -1127,7 +1127,7 @@ class TDTestCase: coulmn_name = qlist[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 111) tdSql.checkData(1, 1, 200) @@ -1136,7 +1136,7 @@ class TDTestCase: coulmn_name = q_null_list[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 121) tdSql.checkData(1, 1, 200) @@ -1184,7 +1184,7 @@ class TDTestCase: def TS_3932_flushdb(self): tdLog.debug("test flush db and insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 121) tdSql.checkData(1, 1, 200) @@ -1192,7 +1192,7 @@ class TDTestCase: q_null_list = ['q_int_null', 'q_bigint_null', 'q_smallint_null', 'q_tinyint_null', 'q_float_null', 'q_double_null', 'q_bool_null', 'q_binary_null', 'q_nchar_null', 'q_ts_null'] tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 122) tdSql.checkData(1, 1, 200) @@ -1200,7 +1200,7 @@ class TDTestCase: coulmn_name = qlist[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 132) tdSql.checkData(1, 1, 200) @@ -1208,7 +1208,7 @@ class TDTestCase: coulmn_name = q_null_list[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 142) tdSql.checkData(1, 1, 200) @@ -1223,7 +1223,7 @@ class TDTestCase: nested.stable_1 (ts,tbname,q_nchar) values(now+8a,'stable_1_1',1)\ nested.stable_1 (ts,tbname,q_ts) values(now+9a,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 152); tdSql.checkData(1, 1, 200); @@ -1330,7 +1330,7 @@ class TDTestCase: nested.stable_null_childtable (ts,tbname,q_ts) values(now+9a,'stable_null_childtable_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 162); tdSql.checkData(1, 1, 200); @@ -1349,7 +1349,7 @@ class TDTestCase: nested.stable_null_childtable (ts,tbname,q_int) values(now,'$^%$%^&',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); @@ -1387,7 +1387,7 @@ class TDTestCase: nested.stable_null_childtable(tbname,ts,q_int,q_binary) file '{self.testcasePath}/stable_null_childtable.csv';") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); @@ -1423,7 +1423,7 @@ class TDTestCase: tdSql.query(f"insert into nested.stable_null_childtable(tbname,ts,q_int,q_binary) file '{self.testcasePath}/stable_null_childtable.csv';") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); From 3bfc7b34862cbea233e33b9b5617f154bf28f295 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 00:47:15 +0800 Subject: [PATCH 051/236] adjust case --- tests/system-test/2-query/group_partition.py | 44 +++++++++++++------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index eb5f069b3d..58be8563fa 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -46,6 +46,12 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname order by count(*)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname having count(*)>=0") + tdSql.checkRows(check_num) + # having filter out empty tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -62,26 +68,26 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # col where filter nothing - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") - # tdSql.checkRows(check_num) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + tdSql.checkRows(check_num) - ############### same with old ############### # col where filter all - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") - # tdSql.checkRows(0) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + tdSql.checkRows(check_num) # col where filter part tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) # col tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) # count + sum(col) tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") num = 0 if real_num > 0: @@ -103,6 +109,12 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname having count(*)>=0") + tdSql.checkRows(check_num) + # having filter out empty tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -119,22 +131,25 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # col where filter nothing - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") - # tdSql.checkRows(check_num) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") + tdSql.checkRows(check_num) - ############### same with old ############### # col where filter all - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") - # tdSql.checkRows(0) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") + tdSql.checkRows(check_num) # col where filter part tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) #col tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") tdSql.checkRows(real_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") num = 0 if real_num > 0: @@ -150,6 +165,7 @@ class TDTestCase: def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") + tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 interval(1d)") def run(self): From 3b0d1480f7783d05b4cab34a4548fd95c436c476 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 11:28:26 +0800 Subject: [PATCH 052/236] adjust case --- tests/script/tsim/field/3.sim | 14 ++++---- tests/script/tsim/field/4.sim | 18 +++++----- tests/script/tsim/field/6.sim | 38 ++++++++++---------- tests/script/tsim/field/bigint.sim | 4 +-- tests/script/tsim/field/double.sim | 4 +-- tests/script/tsim/field/smallint.sim | 4 +-- tests/script/tsim/field/unsigined_bigint.sim | 4 +-- tests/script/tsim/parser/mixed_blocks.sim | 3 +- tests/system-test/2-query/count_partition.py | 4 +-- tests/system-test/2-query/last_row.py | 2 +- 10 files changed, 48 insertions(+), 47 deletions(-) diff --git a/tests/script/tsim/field/3.sim b/tests/script/tsim/field/3.sim index 8b428febcd..4d4801c54a 100644 --- a/tests/script/tsim/field/3.sim +++ b/tests/script/tsim/field/3.sim @@ -447,44 +447,44 @@ if $data00 != 100 then endi print =============== step17 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step18 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/4.sim b/tests/script/tsim/field/4.sim index 361ca4c326..d83de81c88 100644 --- a/tests/script/tsim/field/4.sim +++ b/tests/script/tsim/field/4.sim @@ -619,56 +619,56 @@ if $data00 != 100 then endi print =============== step22 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step23 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/6.sim b/tests/script/tsim/field/6.sim index 52fd0b3780..2d2b5623c7 100644 --- a/tests/script/tsim/field/6.sim +++ b/tests/script/tsim/field/6.sim @@ -824,117 +824,117 @@ if $data00 != 25 then endi print =============== step28 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol3 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol3 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol4 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol4 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol5 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol5 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol6 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol6 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step29 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step30 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/bigint.sim b/tests/script/tsim/field/bigint.sim index ce35cacd84..a791d6a1cb 100644 --- a/tests/script/tsim/field/bigint.sim +++ b/tests/script/tsim/field/bigint.sim @@ -127,7 +127,7 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 if $data00 != 100 then @@ -136,7 +136,7 @@ if $data00 != 100 then endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/double.sim b/tests/script/tsim/field/double.sim index 1f0cea4be8..2daa78eb7a 100644 --- a/tests/script/tsim/field/double.sim +++ b/tests/script/tsim/field/double.sim @@ -127,14 +127,14 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/smallint.sim b/tests/script/tsim/field/smallint.sim index 66bfee5838..980d69297a 100644 --- a/tests/script/tsim/field/smallint.sim +++ b/tests/script/tsim/field/smallint.sim @@ -128,14 +128,14 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/unsigined_bigint.sim b/tests/script/tsim/field/unsigined_bigint.sim index baa57ce1f6..de8098c297 100644 --- a/tests/script/tsim/field/unsigined_bigint.sim +++ b/tests/script/tsim/field/unsigined_bigint.sim @@ -131,7 +131,7 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 if $data00 != 100 then @@ -140,7 +140,7 @@ if $data00 != 100 then endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index 04e9df3ff4..8716192858 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -100,7 +100,8 @@ if $data05 != 1 then endi sql select max(c1), min(c1), sum(c1), avg(c1), count(c1), t1 from $stb where c1 > 0 group by t1 -if $rows != 1 then +if $rows != 2 then + print === rows $rows return -1 endi if $data00 != 319 then diff --git a/tests/system-test/2-query/count_partition.py b/tests/system-test/2-query/count_partition.py index f59376a979..e970b00cec 100644 --- a/tests/system-test/2-query/count_partition.py +++ b/tests/system-test/2-query/count_partition.py @@ -66,9 +66,9 @@ class TDTestCase: tdSql.checkRows(self.row_nums+1) tdSql.query(f"select count(c1) , count(t2) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") - tdSql.checkRows(2) + tdSql.checkRows(10) tdSql.query(f"select count(c1) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") - tdSql.checkRows(2) + tdSql.checkRows(10) tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname order by tbname") tdSql.checkRows(self.tb_nums) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index a6bcc2c5f1..5b989eb456 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -436,7 +436,7 @@ class TDTestCase: tdSql.checkData(1,1,None) tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 ") - tdSql.checkRows(2) + tdSql.checkRows(4) # filter by tbname tdSql.query(f"select last_row(c1) from {dbname}.stb1 where tbname = 'ct1' ") From ef5b5786b38e3d14465ae16780b47ed4f5935265 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 16:03:09 +0800 Subject: [PATCH 053/236] adjust --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1889492aa0..6df46ef4d0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -683,7 +683,7 @@ static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STable blockDataEmpty(pBlock); pBlock->info.rows = 1; pBlock->info.id.uid = tbInfo->uid; - pBlock->info.id.groupId = pOperator->dynamicTask ? tbInfo->uid : tbInfo->groupId; + pBlock->info.id.groupId = tbInfo->groupId; // only one row: set all col data to null & hasNull int32_t col_num = blockDataGetNumOfCols(pBlock); From 6e4703e5b43af47d83aca6acf4e5d8fe1dc16d7d Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 22:58:40 +0800 Subject: [PATCH 054/236] adjust ts --- .../2-query/nestedQueryInterval.py | 100 ++++++++++-------- 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index b6ef50dcda..185848f6e0 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -5,6 +5,8 @@ import socket import os import threading +from datetime import timezone, datetime + from util.log import * from util.sql import * from util.cases import * @@ -1196,9 +1198,14 @@ class TDTestCase: tdSql.checkData(0, 1, 122) tdSql.checkData(1, 1, 200) + pd = datetime.datetime.now() + ts = int(datetime.datetime.timestamp(pd)*1000 - 10000) + print(f"start time {ts}") + for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values({ts+i},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 132) @@ -1206,92 +1213,101 @@ class TDTestCase: for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values({ts+i},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 142) tdSql.checkData(1, 1, 200) - tdSql.execute(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'stable_1_1',1) \ - nested.stable_1 (ts,tbname,q_bigint) values(now+1a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_smallint) values(now+2a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_tinyint) values(now+3a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_float) values(now+4a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_double) values(now+5a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_bool) values(now+6a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_binary) values(now+7a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_nchar) values(now+8a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_ts) values(now+9a,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts,tbname,q_int) values({ts},'stable_1_1',1) \ + nested.stable_1 (ts,tbname,q_bigint) values({ts+1},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_smallint) values({ts+2},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_tinyint) values({ts+3},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_float) values({ts+4},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_double) values({ts+5},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_bool) values({ts+6},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_binary) values({ts+7},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_nchar) values({ts+8},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_ts) values({ts+9},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 152); tdSql.checkData(1, 1, 200); - tdSql.query(f"insert into nested.stable_null_data (ts,tbname) values(now,'stable_null_data_1');") + tdSql.query(f"insert into nested.stable_null_data (ts,tbname) values({ts},'stable_null_data_1');") + ts = ts + 1 tdSql.query(f"select tbname,count(*) from nested.stable_null_data_1 group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 22); for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 32) for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") - + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 42) - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values(now,'stable_null_data_1',1) \ - nested.stable_null_data (ts,tbname,q_bigint) values(now+1a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_smallint) values(now+2a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_tinyint) values(now+3a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_float) values(now+4a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_double) values(now+5a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_bool) values(now+6a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_binary) values(now+7a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_nchar) values(now+8a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_ts) values(now+9a,'stable_null_data_1',1);") + tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values({ts},'stable_null_data_1',1) \ + nested.stable_null_data (ts,tbname,q_bigint) values({ts+1},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_smallint) values({ts+2},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_tinyint) values({ts+3},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_float) values({ts+4},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_double) values({ts+5},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_bool) values({ts+6},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_binary) values({ts+7},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_nchar) values({ts+8},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_ts) values({ts+9},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 52); - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values(now,'stable_null_childtable_1');") + tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values({ts},'stable_null_childtable_1');") + ts = ts + 1 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 22) for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 32) for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 42); - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values(now,'stable_null_childtable_1',1) \ - nested.stable_null_childtable (ts,tbname,q_bigint) values(now+1a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_smallint) values(now+2a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_tinyint) values(now+3a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_float) values(now+4a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_double) values(now+5a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_bool) values(now+6a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_binary) values(now+7a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_nchar) values(now+8a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_ts) values(now+9a,'stable_null_childtable_1',1);") + tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values({ts},'stable_null_childtable_1',1) \ + nested.stable_null_childtable (ts,tbname,q_bigint) values({ts+1},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_smallint) values({ts+2},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_tinyint) values({ts+3},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_float) values({ts+4},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_double) values({ts+5},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_bool) values({ts+6},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_binary) values({ts+7},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_nchar) values({ts+8},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_ts) values({ts+9},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 52); @@ -1344,9 +1360,9 @@ class TDTestCase: #test special character - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'!@!@$$^$',1) \ - nested.stable_null_data (ts,tbname,q_int) values(now,'%^$^&^&',1) \ - nested.stable_null_childtable (ts,tbname,q_int) values(now,'$^%$%^&',1);") + tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now+10a,'!@!@$$^$',1) \ + nested.stable_null_data (ts,tbname,q_int) values(now+10a,'%^$^&^&',1) \ + nested.stable_null_childtable (ts,tbname,q_int) values(now+10a,'$^%$%^&',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(7) From 80604ec8e29c83313da061d9b2d868d558158aaa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 23:31:51 +0800 Subject: [PATCH 055/236] fix(stream): init rwlock. --- source/libs/stream/src/streamMeta.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0702a908ab..292bd8cb38 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -375,6 +375,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF stage); pMeta->rid = taosAddRef(streamMetaId, pMeta); + taosThreadRwlockInit(&pMeta->lock, NULL); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); From 66bdace9eb70058bd80476a9185d090edb1ee980 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 00:09:55 +0800 Subject: [PATCH 056/236] fix(stream): check rsp. --- source/libs/stream/src/streamMeta.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 292bd8cb38..eebde87b28 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1299,7 +1299,12 @@ void streamMetaRLock(SStreamMeta* pMeta) { void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); - taosThreadRwlockUnlock(&pMeta->lock); + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code != TSDB_CODE_SUCCESS) { + stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code); + } else { + stDebug("vgId:%d meta-runlock completed", pMeta->vgId); + } } void streamMetaWLock(SStreamMeta* pMeta) { From 5c1ff76082d393fae04374e09e679004a07dc7ad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 00:58:52 +0800 Subject: [PATCH 057/236] fix(stream): set rwlock attr. --- source/libs/stream/src/streamMeta.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index eebde87b28..25f0fbf991 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -375,7 +375,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF stage); pMeta->rid = taosAddRef(streamMetaId, pMeta); - taosThreadRwlockInit(&pMeta->lock, NULL); + + TdThreadRwlockAttr attr; + taosThreadRwlockAttrInit(&attr); + + pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + taosThreadRwlockInit(&pMeta->lock, &attr); + taosThreadRwlockAttrDestroy(&attr); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); @@ -1386,6 +1392,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) } pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; + streamMetaWUnLock(pMeta); if (isLeader) { stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, @@ -1395,8 +1402,6 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) stInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, prevStage, stage, isLeader, pMeta->sendMsgBeforeClosing); } - - streamMetaWUnLock(pMeta); } int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { From dc451984b5ae4acc77cf4c21479df75d3c054023 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 12:59:46 +0800 Subject: [PATCH 058/236] feat: add level and disk param on test.py --- tests/army/frame/caseBase.py | 25 ++ tests/army/frame/common.py | 2 +- tests/army/frame/dnodes-default.py | 502 ---------------------- tests/army/frame/dnodes-no-random-fail.py | 500 --------------------- tests/army/frame/dnodes-random-fail.py | 497 --------------------- tests/army/frame/{ => server}/cluster.py | 10 +- tests/army/frame/{ => server}/dnodes.py | 23 +- tests/army/frame/sql.py | 3 + tests/army/frame/srvCtl.py | 27 ++ tests/army/test.py | 21 +- 10 files changed, 96 insertions(+), 1514 deletions(-) create mode 100644 tests/army/frame/caseBase.py delete mode 100644 tests/army/frame/dnodes-default.py delete mode 100644 tests/army/frame/dnodes-no-random-fail.py delete mode 100644 tests/army/frame/dnodes-random-fail.py rename tests/army/frame/{ => server}/cluster.py (91%) rename tests/army/frame/{ => server}/dnodes.py (98%) create mode 100644 tests/army/frame/srvCtl.py diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py new file mode 100644 index 0000000000..abaa92a51c --- /dev/null +++ b/tests/army/frame/caseBase.py @@ -0,0 +1,25 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + +class caseBase: + def __init__(self): + # record server information + self.dnodeNum = 0 + self.mnodeNum = 0 + self.mLevel = 0 + self.mLevelDisk = 0 diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index 93059ff078..b8704080c0 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -23,7 +23,7 @@ import taos from frame.log import * from frame.sql import * from frame.cases import * -from frame.dnodes import * +from TDengine.tests.army.frame.server.dnodes import * from frame.common import * from frame.constant import * from dataclasses import dataclass,field diff --git a/tests/army/frame/dnodes-default.py b/tests/army/frame/dnodes-default.py deleted file mode 100644 index a1d1698b00..0000000000 --- a/tests/army/frame/dnodes-default.py +++ /dev/null @@ -1,502 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "minTablesPerVnode": "4", - "maxTablesPerVnode": "1000", - "tableIncStepPerVnode": "10000", - "maxVgroupsPerDb": "1000", - "sdbDebugFlag": "143", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/dnodes-no-random-fail.py b/tests/army/frame/dnodes-no-random-fail.py deleted file mode 100644 index 3b3083396b..0000000000 --- a/tests/army/frame/dnodes-no-random-fail.py +++ /dev/null @@ -1,500 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "anyIp": "0", - "sdbDebugFlag": "135", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s --random-file-fail-factor 0 > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/dnodes-random-fail.py b/tests/army/frame/dnodes-random-fail.py deleted file mode 100644 index 794adfd7ed..0000000000 --- a/tests/army/frame/dnodes-random-fail.py +++ /dev/null @@ -1,497 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s --alloc-random-fail --random-file-fail-factor 5 > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/cluster.py b/tests/army/frame/server/cluster.py similarity index 91% rename from tests/army/frame/cluster.py rename to tests/army/frame/server/cluster.py index 4da53840c0..ade8ac39a2 100644 --- a/tests/army/frame/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -8,7 +8,7 @@ import socket from frame.log import * from frame.sql import * from frame.cases import * -from frame.dnodes import * +from frame.server.dnodes import * from frame.common import * class ClusterDnodes(TDDnodes): @@ -35,17 +35,21 @@ class ConfigureyCluster: self.startPort = 6030 self.portStep = 100 self.mnodeNums = 0 + self.level = 0 + self.disk = 0 - def configure_cluster(self ,dnodeNums=5,mnodeNums=0,independentMnode=True,startPort=6030,portStep=100,hostname="%s"%hostname): + def configure_cluster(self ,dnodeNums=5, mnodeNums=0, independentMnode=True, startPort=6030, portStep=100, hostname="%s"%hostname, level=1, disk=1): self.startPort=int(startPort) self.portStep=int(portStep) self.hostname=hostname self.dnodeNums = int(dnodeNums) self.mnodeNums = int(mnodeNums) + self.level = int(level) + self.disk = int(disk) self.dnodes = [] startPort_sec = int(startPort+portStep) for num in range(1, (self.dnodeNums+1)): - dnode = TDDnode(num) + dnode = TDDnode(num, self.level, self.disk) dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}") dnode.addExtraCfg("fqdn", f"{hostname}") dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}") diff --git a/tests/army/frame/dnodes.py b/tests/army/frame/server/dnodes.py similarity index 98% rename from tests/army/frame/dnodes.py rename to tests/army/frame/server/dnodes.py index 5581f29a57..7eee6f4bb6 100644 --- a/tests/army/frame/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -115,8 +115,11 @@ class TDSimClient: class TDDnode: - def __init__(self, index): + def __init__(self, index, level, disk): self.index = index + self.level = level + self.disk = disk + self.dataDir = [] self.running = 0 self.deployed = 0 self.testCluster = False @@ -210,13 +213,18 @@ class TDDnode: def deploy(self, *updatecfgDict): self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") + simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) + for i in range(1, self.level): + for j in range(1, self.disk): + eDir = os.path.join(simPath, f"data{i}{j}") + self.dataDir.append(eDir) self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + + for eDir in self.dataDir: + cmd = "rm -rf " + eDir + if os.system(cmd) != 0: + tdLog.exit(cmd) cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -229,7 +237,8 @@ class TDDnode: # cmd = "mkdir -p " + self.dataDir # if os.system(cmd) != 0: # tdLog.exit(cmd) - os.makedirs(self.dataDir) + for eDir in self.dataDir: + os.makedirs(self.dataDir) # cmd = "mkdir -p " + self.logDir # if os.system(cmd) != 0: diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index d7dce2bc3e..409dc6c96c 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -630,6 +630,7 @@ class TDSql: tdLog.exit("taosd state is %d != expect:%d" %args) pass + ''' def haveFile(self, dir, state): if os.path.exists(dir) and os.path.isdir(dir): if not os.listdir(dir): @@ -644,6 +645,7 @@ class TDSql: tdLog.exit("dir: %s is not empty, expect: empty" %dir) else: tdLog.exit("dir: %s doesn't exist" %dir) + def createDir(self, dir): if os.path.exists(dir): shrmtree(dir) @@ -651,5 +653,6 @@ class TDSql: os.makedirs( dir, 755 ) tdLog.info("dir: %s is created" %dir) pass +''' tdSql = TDSql() diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py new file mode 100644 index 0000000000..e7ef08cde9 --- /dev/null +++ b/tests/army/frame/srvCtl.py @@ -0,0 +1,27 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + +class srvCtl: + def __init__(self): + # record server information + self.dnodeNum = 0 + self.mnodeNum = 0 + self.mLevel = 0 + self.mLevelDisk = 0 + +sc = srvCtl() \ No newline at end of file diff --git a/tests/army/test.py b/tests/army/test.py index cc114e0d16..ae873fb9d6 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -28,9 +28,9 @@ import importlib import toml from frame.log import * -from frame.dnodes import * +from frame.server.dnodes import * +from frame.server.cluster import * from frame.cases import * -from frame.cluster import * from frame.taosadapter import * import taos @@ -111,6 +111,9 @@ if __name__ == "__main__": asan = False independentMnode = False previousCluster = False + level = 1 + disk = 1 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) for key, value in opts: @@ -139,6 +142,8 @@ if __name__ == "__main__": tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.') + tdLog.printNoPrefix('-L set multiple level number. range 1 ~ 3') + tdLog.printNoPrefix('-D set disk number on each level. range 1 ~ 10') sys.exit(0) @@ -226,6 +231,12 @@ if __name__ == "__main__": if key in ['-P', '--previous']: previousCluster = True + if key in ['-L', '--level']: + level = value + + if key in ['-D', '--disk']: + disk = value + # # do exeCmd command # @@ -391,7 +402,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -498,6 +509,7 @@ if __name__ == "__main__": else: tdLog.info("not need to query") else: + # except windows tdDnodes.setKillValgrind(killValgrind) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -574,7 +586,8 @@ if __name__ == "__main__": # dnode > 1 cluster tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) print(independentMnode,"independentMnode valuse") - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) + # create dnode list + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) From 9e7a4b8a868df69614f3f7d474cf3a09f39b7799 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 13:09:02 +0800 Subject: [PATCH 059/236] feat: test.py add -L --level -D --disk options --- tests/army/test.py | 2 +- tests/parallel_test/cases.task | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/test.py b/tests/army/test.py index ae873fb9d6..25cb9b383d 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -218,7 +218,7 @@ if __name__ == "__main__": if key in ['-a', '--asan']: asan = True - if key in ['-D', '--adaptercfgupdate']: + if key in ['-U', '--adaptercfgupdate']: try: adaptercfgupdate = eval(base64.b64decode(value.encode()).decode()) except: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..19a44767a7 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,7 +6,7 @@ ,,y,unit-test,bash test.sh #army-test -,,y,army,./pytest.sh python3 ./test.py -f empty.py +,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 #system test ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py From 8a714db17df323425fb4ca85b5edefda625a3cc4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 15:33:50 +0800 Subject: [PATCH 060/236] fix: fix deadlock. --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 25f0fbf991..7217a49ff8 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1356,7 +1356,7 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { continue; } From 266ac30d96336204f4ebe040faf624d01dcebb37 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 16:28:39 +0800 Subject: [PATCH 061/236] feat: support -L -D for test.py submit --- tests/army/frame/common.py | 2 +- tests/army/frame/server/dnodes.py | 23 +++++++++++++++++------ tests/army/test.py | 7 ++++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index b8704080c0..5cdb3f9f46 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -23,7 +23,7 @@ import taos from frame.log import * from frame.sql import * from frame.cases import * -from TDengine.tests.army.frame.server.dnodes import * +from frame.server.dnodes import * from frame.common import * from frame.constant import * from dataclasses import dataclass,field diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 7eee6f4bb6..8368a1573e 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -115,7 +115,7 @@ class TDSimClient: class TDDnode: - def __init__(self, index, level, disk): + def __init__(self, index=1, level=1, disk=1): self.index = index self.level = level self.disk = disk @@ -212,12 +212,19 @@ class TDDnode: self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) def deploy(self, *updatecfgDict): + # logDir self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") + # dataDir simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) - for i in range(1, self.level): - for j in range(1, self.disk): + primary = 1 + for i in range(self.level): + for j in range(self.disk): eDir = os.path.join(simPath, f"data{i}{j}") - self.dataDir.append(eDir) + self.dataDir.append(f"{eDir} {i} {primary}") + if primary == 1: + primary = 0 + + # taos.cfg self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") @@ -238,7 +245,7 @@ class TDDnode: # if os.system(cmd) != 0: # tdLog.exit(cmd) for eDir in self.dataDir: - os.makedirs(self.dataDir) + os.makedirs(eDir.split(' ')[0]) # cmd = "mkdir -p " + self.logDir # if os.system(cmd) != 0: @@ -284,7 +291,11 @@ class TDDnode: self.addExtraCfg(key, value) if (self.remoteIP == ""): for key, value in self.cfgDict.items(): - self.cfg(key, value) + if type(value) == list: + for v in value: + self.cfg(key, v) + else: + self.cfg(key, value) else: self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)"%self.index) diff --git a/tests/army/test.py b/tests/army/test.py index 25cb9b383d..83d065b24b 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -114,8 +114,9 @@ if __name__ == "__main__": level = 1 disk = 1 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWU:n:i:aP:L:D:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums', + 'queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode',"asan",'previous','level','disk']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -137,7 +138,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') tdLog.printNoPrefix('-R restful realization form') tdLog.printNoPrefix('-W websocket connection') - tdLog.printNoPrefix('-D taosadapter update cfg dict ') + tdLog.printNoPrefix('-U taosadapter update cfg dict ') tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') From c61203b55a98707c12675358c2e6e56b175395e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 16:37:21 +0800 Subject: [PATCH 062/236] fix: fix deadlock. --- source/libs/stream/src/streamMeta.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7217a49ff8..46e3dded11 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1065,28 +1065,16 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { SStreamHbMsg hbMsg = {0}; SEpSet epset = {0}; bool hasMnodeEpset = false; - int64_t stage = 0; + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - streamMetaRLock(pMeta); - - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); hbMsg.vgId = pMeta->vgId; - stage = pMeta->stage; - - SArray* pIdList = taosArrayDup(pMeta->pTaskList, NULL); - - streamMetaRUnLock(pMeta); - hbMsg.pTaskStatus = taosArrayInit(numOfTasks, sizeof(STaskStatusEntry)); hbMsg.pUpdateNodes = taosArrayInit(numOfTasks, sizeof(int32_t)); for (int32_t i = 0; i < numOfTasks; ++i) { - STaskId* pId = taosArrayGet(pIdList, i); + STaskId* pId = taosArrayGet(pMeta->pTaskList, i); - streamMetaRLock(pMeta); SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); - streamMetaRUnLock(pMeta); - if (pTask == NULL) { continue; } @@ -1100,7 +1088,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { .id = *pId, .status = streamTaskGetStatus(*pTask, NULL), .nodeId = hbMsg.vgId, - .stage = stage, + .stage = pMeta->stage, .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)), }; @@ -1170,7 +1158,6 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { _end: streamMetaClearHbMsg(&hbMsg); - taosArrayDestroy(pIdList); return TSDB_CODE_SUCCESS; } @@ -1210,7 +1197,9 @@ void metaHbToMnode(void* param, void* tmrId) { } stDebug("vgId:%d build stream task hb, leader:%d", pMeta->vgId, (pMeta->role == NODE_ROLE_LEADER)); + streamMetaRLock(pMeta); metaHeartbeatToMnodeImpl(pMeta); + streamMetaRUnLock(pMeta); taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); From 4b6bd14295954abf828efd2a78bcdd04512b1d17 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 16:39:13 +0800 Subject: [PATCH 063/236] fix: if level=1 disk=1 save old data folder --- tests/army/frame/server/dnodes.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 8368a1573e..1e51fbc3be 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -217,12 +217,16 @@ class TDDnode: # dataDir simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) primary = 1 - for i in range(self.level): - for j in range(self.disk): - eDir = os.path.join(simPath, f"data{i}{j}") - self.dataDir.append(f"{eDir} {i} {primary}") - if primary == 1: - primary = 0 + if self.level == 1 and self.disk == 1: + eDir = os.path.join(simPath, "data") + self.dataDir.append(eDir) + else: + for i in range(self.level): + for j in range(self.disk): + eDir = os.path.join(simPath, f"data{i}{j}") + self.dataDir.append(f"{eDir} {i} {primary}") + if primary == 1: + primary = 0 # taos.cfg self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") From 25f798e42f24c90917192ce30c705053d156d43b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 23:29:30 +0800 Subject: [PATCH 064/236] fix(stream): record the start failure tasks --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 -- source/libs/stream/src/streamMeta.c | 20 ++++++++++++++------ source/libs/stream/src/streamStart.c | 7 ++++--- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index af9363e363..f576345f64 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -661,8 +661,6 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { if (isLeader && !tsDisableStream) { tqStreamTaskResetStatus(pMeta); streamMetaWUnLock(pMeta); - tqInfo("vgId:%d start all stream tasks after reload tasks from disk", vgId); - streamMetaStartAllTasks(pMeta); } else { streamMetaResetStartInfo(&pMeta->startInfo); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 46e3dded11..a46853fc26 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1429,8 +1429,9 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - stDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + stInfo("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); if (numOfTasks == 0) { + stInfo("vgId:%d start tasks completed", pMeta->vgId); return TSDB_CODE_SUCCESS; } @@ -1443,16 +1444,20 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { streamMetaWUnLock(pMeta); // broadcast the check downstream tasks msg + int64_t now = taosGetTimestampMs(); + for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { - streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, - taosGetTimestampMs(), false); + stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, now, false); continue; } // fill-history task can only be launched by related stream tasks. + STaskExecStatisInfo* pInfo = &pTask->execInfo; if (pTask->info.fillHistory == 1) { streamMetaReleaseTask(pMeta, pTask); continue; @@ -1465,8 +1470,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { streamLaunchFillHistoryTask(pTask); } - streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, - pTask->execInfo.start, true); + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->init, pInfo->start, true); streamMetaReleaseTask(pMeta, pTask); continue; } @@ -1474,12 +1478,16 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); if (ret != TSDB_CODE_SUCCESS) { + stError("vgId:%d failed to handle event:%d", pMeta->vgId, event); code = ret; + + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->init, pInfo->start, false); } streamMetaReleaseTask(pMeta, pTask); } + stInfo("vgId:%d start tasks completed", pMeta->vgId); taosArrayDestroy(pTaskList); return code; } \ No newline at end of file diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 3bef6adf57..267dadf109 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1114,9 +1114,9 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI pStartInfo->readyTs = taosGetTimestampMs(); pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; - stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x startTs:%" PRId64 + stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64 ", readyTs:%" PRId64 " total elapsed time:%.2fs", - pMeta->vgId, numOfTotal, taskId, pStartInfo->startTs, pStartInfo->readyTs, + pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs, pStartInfo->elapsedTime / 1000.0); // print the initialization elapsed time and info @@ -1128,7 +1128,8 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI pStartInfo->completeFn(pMeta); } else { streamMetaWUnLock(pMeta); - stDebug("vgId:%d recv check downstream results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); + stDebug("vgId:%d recv check downstream results, s-task:0x%x succ:%d, received:%d, total:%d", pMeta->vgId, taskId, + ready, numOfRecv, numOfTotal); } return TSDB_CODE_SUCCESS; From c9b69316c1f8f71281d1506ed681989b5f4e4990 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 01:19:16 +0800 Subject: [PATCH 065/236] calcNeedCountEmpty --- include/libs/nodes/plannodes.h | 1 + source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/planner/src/planLogicCreater.c | 9 +++++++++ source/libs/planner/src/planOptimizer.c | 1 + source/libs/planner/src/planPhysiCreater.c | 22 +++++++++++----------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 4b3c846389..55ac3b86bf 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -147,6 +147,7 @@ typedef struct SAggLogicNode { bool isGroupTb; bool isPartTb; // true if partition keys has tbname bool hasGroup; + bool isCountByTag; // true if hasCountFunc & part by tag/tbname } SAggLogicNode; typedef struct SProjectLogicNode { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 8a154dcf00..2af31a8c7a 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -453,6 +453,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { COPY_SCALAR_FIELD(isGroupTb); COPY_SCALAR_FIELD(isPartTb); COPY_SCALAR_FIELD(hasGroup); + COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2adc5b3072..17b1b272ba 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -748,6 +748,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; + bool isCountByTag = false; + if (pSelect->hasCountFunc) { + if (pSelect->pGroupByList) { + isCountByTag = !keysHasCol(pSelect->pGroupByList); + } else if (pSelect->pPartitionByList) { + isCountByTag = !keysHasCol(pSelect->pPartitionByList); + } + } + pAgg->isCountByTag = isCountByTag; if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index aa3181e166..9ecbd3ae9f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2783,6 +2783,7 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, pNew->isGroupTb = pAgg->isGroupTb; pNew->isPartTb = pAgg->isPartTb; pNew->hasGroup = pAgg->hasGroup; + pNew->isCountByTag = pAgg->isCountByTag; pNew->node.pChildren = nodesCloneList(pAgg->node.pChildren); int32_t code = 0; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 901927b1d1..9c4e9f97ab 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -37,6 +37,7 @@ typedef struct SPhysiPlanContext { SArray* pLocationHelper; bool hasScan; bool hasSysScan; + SLogicNode* pNode; // tmp record LogicSubplan->pNode } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey, int32_t keyBufSize) { @@ -592,17 +593,13 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog if (pScanLogicNode->interval > 0) { return false; } + // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { - SSelectStmt* pSelect = (SSelectStmt*)pRoot; - // select & count - if (pSelect->hasCountFunc) { - // key only accept tag/tbname - if (NULL != pSelect->pGroupByList) { - return !keysHasCol(pSelect->pGroupByList); - } else if (NULL != pSelect->pPartitionByList) { - return !keysHasCol(pSelect->pPartitionByList); - } + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) + && pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { + SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; + if (pNode->isCountByTag) { + return true; } } return false; @@ -2302,7 +2299,9 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } + pCxt->pNode = pLogicSubplan->pNode; code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); + pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); } @@ -2454,7 +2453,8 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, - .hasSysScan = false}; + .hasSysScan = false, + .pNode = NULL}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } From 30498a40285565dfdfa7f4afcfac29906162619c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 24 Dec 2023 02:24:46 +0800 Subject: [PATCH 066/236] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 5 +++-- source/libs/stream/src/streamMeta.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e8e88e52f7..c8cc4f9d54 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1259,10 +1259,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (pTask->status.downstreamReady != 1) { pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id pTask->chkInfo.checkpointingId = req.checkpointId; + pTask->chkInfo.transId = req.transId; tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 - ", set it failure", - pTask->id.idStr, req.checkpointId); + ", transId:%d set it failed", + pTask->id.idStr, req.checkpointId, req.transId); streamMetaReleaseTask(pMeta, pTask); SRpcMsg rsp = {0}; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a46853fc26..9e20a2997c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1102,6 +1102,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; entry.chkpointTransId = (*pTask)->chkInfo.transId; + stInfo("s-task:%s send kill checkpoint trans info, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId); } if ((*pTask)->exec.pWalReader != NULL) { From bf895fadcb144cb31951cb54f3e0874290f8c8d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 24 Dec 2023 03:05:29 +0800 Subject: [PATCH 067/236] fix(stream): fix invalid read. --- source/libs/stream/src/streamDispatch.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index f0ee03810b..bfa269b0d5 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -352,6 +352,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD return code; } + // it's a new vnode to receive dispatch msg, so add one if (pReqs[j].blockNum == 0) { atomic_add_fetch_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 1); } @@ -372,8 +373,15 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD pTask->msgInfo.pData = pReqs; } - stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%"PRId64" %p", pTask->id.idStr, pTask->execInfo.dispatch, - pTask->pMeta->stage, pTask->msgInfo.pData); + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64 " %p", pTask->id.idStr, + pTask->execInfo.dispatch, pTask->pMeta->stage, pTask->msgInfo.pData); + } else { + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64 " dstVgNum:%d %p", pTask->id.idStr, + pTask->execInfo.dispatch, pTask->pMeta->stage, pTask->outputInfo.shuffleDispatcher.waitingRspCnt, + pTask->msgInfo.pData); + } + return code; } @@ -395,9 +403,11 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgroups = taosArrayGetSize(vgInfo); - stDebug("s-task:%s (child taskId:%d) start to shuffle-dispatch blocks to %d vgroup(s), msgId:%d", id, - pTask->info.selfChildId, numOfVgroups, msgId); + int32_t actualVgroups = pTask->outputInfo.shuffleDispatcher.waitingRspCnt; + stDebug("s-task:%s (child taskId:%d) start to shuffle-dispatch blocks to %d/%d vgroup(s), msgId:%d", id, + pTask->info.selfChildId, actualVgroups, numOfVgroups, msgId); + int32_t numOfSend = 0; for (int32_t i = 0; i < numOfVgroups; i++) { if (pDispatchMsg[i].blockNum > 0) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); @@ -408,6 +418,11 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch if (code < 0) { break; } + + // no need to try remain, all already send. + if (++numOfSend == actualVgroups) { + break; + } } } From 651b681d4c95879c5a20668c8b85c45583383e98 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 18:45:51 +0800 Subject: [PATCH 068/236] adjust case --- tests/system-test/2-query/count.py | 4 ++-- tests/system-test/2-query/last_row.py | 4 ++-- tests/system-test/2-query/nestedQueryInterval.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 7e99f9e4dd..0e1d332d89 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -119,8 +119,8 @@ class TDTestCase: tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname order by tbname)') tdSql.checkRows(1) if 'count' == function_name: - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) elif 'hyperloglog' == function_name: tdSql.checkData(0, 0, 0) tdSql.checkData(0, 0, 0) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 5b989eb456..6469b3ea54 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -435,8 +435,8 @@ class TDTestCase: tdSql.checkData(1,0,'ct4') tdSql.checkData(1,1,None) - tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 ") - tdSql.checkRows(4) + tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 having count(c1)>0") + tdSql.checkRows(2) # filter by tbname tdSql.query(f"select last_row(c1) from {dbname}.stb1 where tbname = 'ct1' ") diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index 185848f6e0..07b5519432 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -1113,8 +1113,8 @@ class TDTestCase: def TS_3932(self): tdLog.debug("test insert data into stable") - tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(6) + tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname having count(*)>0 order by tbname;") + tdSql.checkRows(2) tdSql.checkData(0, 1, 100) tdSql.checkData(1, 1, 200) From 287c36185223c5276d00eeb42d614b2ec150b72c Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 18:49:48 +0800 Subject: [PATCH 069/236] adjust test case --- tests/system-test/1-insert/ts-4272.py | 54 ++++++++++----------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/tests/system-test/1-insert/ts-4272.py b/tests/system-test/1-insert/ts-4272.py index bb81305eb3..496433664b 100644 --- a/tests/system-test/1-insert/ts-4272.py +++ b/tests/system-test/1-insert/ts-4272.py @@ -26,12 +26,10 @@ class TDTestCase: self.file1 = f"{self.testcasePath}/b.csv" self.file2 = f"{self.testcasePath}/c.csv" - #os.system("rm -rf %s/b.csv" %self.testcasePath) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) def check_count(self, rows, records): - tdSql.execute(f"use {self.db};") tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") tdSql.checkRows(rows) for i in range(rows): @@ -39,13 +37,6 @@ class TDTestCase: def reset_tb(self): # create database and tables - # os.system("taos -s 'drop database if exists d1;'") - # os.system("taos -s 'create database d1;use d1;create stable meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int);'") - # os.system(f"taos -s 'use d1;create table d2001 using meters(groupId) tags(5);'") - # res = os.system(f"taos -s 'use d1;create table d2002 using meters(groupId) tags(6);'") - # if (0 != res): - # tdLog.exit(f"create tb error") - tdSql.execute(f"drop database if exists {self.db};") tdSql.execute(f"create database {self.db};") tdSql.execute(f"use {self.db};") @@ -54,13 +45,14 @@ class TDTestCase: tdSql.execute(f"create table {self.tb2} {self.tag2};") tdSql.execute(f"create stable {self.stable1} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") tdSql.execute(f"create stable {self.stable2} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") + tdSql.execute(f"create table {self.stable1}_1 using {self.stable1}(t_int) tags(1);") + tdSql.execute(f"create table {self.stable2}_1 using {self.stable2}(t_int) tags(2);") def test(self, sql): sql = f"use {self.db};" + sql - res = os.system(f'taos -s "{sql}"') - # if (0 != res): - # tdLog.exit(f"taos sql error") - + os.system(f'taos -s "{sql}"') + # print(f'{sql}\n') + # tdSql.execute(sql, 1) def check(self): # same table, auto create + create @@ -95,13 +87,8 @@ class TDTestCase: sql = f"insert into {self.tb1} file '{self.file1}' {self.tb2} file '{self.file2}';" self.test(sql) - # bigNum = 1010000 - # self.check_count(5, [2100, 2100, bigNum, bigNum, bigNum]) + self.check_count(2, [2010000, 1000000]) - result = os.popen("taos -s 'select count(*) from %s.%s'" %(self.db, self.tb1)) - res = result.read() - if (f"OK" in res): - tdLog.info(f"check count success") def make_csv(self, filepath, once, qtime, startts): f = open(filepath, 'w') @@ -118,10 +105,8 @@ class TDTestCase: def test_mix(self): #forbid use both value and file in one insert - result = os.popen(f"insert into {self.tb1} file '{self.file2}' {self.tb2} values('2021-07-13 14:06:34.630', 10.2, 219, 0.32);") - res = result.read() - if (f"error" in res): - tdLog.info(f"forbid success") + self.make_csv(self.file2, 10, 10, self.ts) + tdSql.error(f"insert into {self.tb1} file '{self.file2}' {self.tb2} values('2021-07-13 14:06:34.630', 10.2, 219, 0.32);") def test_bigcsv(self): # prepare csv @@ -144,7 +129,6 @@ class TDTestCase: self.test(sql) print("end insert to table") - #tdSql.execute(f"use d1;") tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") tdSql.checkRows(2) tdSql.checkData(0, 1, rowNum1) @@ -160,7 +144,7 @@ class TDTestCase: ts = startts + offset rows = [] for i in range(once): - rows.append([table_name, ts + i, offset + i, 'NULL']) + rows.append([f"\'{table_name}\'", ts + i, offset + i, 'NULL']) writer.writerows(rows) f.close() print(datetime.now(), filepath, " ready!") @@ -171,22 +155,22 @@ class TDTestCase: once = 10000 qtime1 = 101 qtime2 = 100 - # rowNum1 = qtime1 * once - # rowNum2 = qtime2 * once child_1 = f"{self.stable1}_1" child_2 = f"{self.stable2}_1" self.make_stable_csv(self.file1, once, qtime1, self.ts - 86400000, child_1) self.make_stable_csv(self.file2, once, qtime2, self.ts, child_2) print("end stable_csv data prepare") - - # insert create child table of stable + sql = f"insert into {self.db}.{self.stable1}(tbname,ts,q_int,q_binary) file '{self.file1}' {self.db}.{self.stable2}(tbname,ts,q_int,q_binary) file '{self.file2}';" self.test(sql) print("end insert to stable") - #tdSql.execute(f"insert into {self.db}.{child_1}(ts, q_int) values(now, 1);") - tdSql.query(f"select tbname,count(*) from {self.stable1} group by tbname order by tbname;") - tdSql.checkRows(0) + tdSql.query(f"select tbname,count(*) from {self.stable1} group by tbname;") + tdSql.checkRows(1) + tdSql.checkData(0, 1, qtime1 * once) + tdSql.query(f"select tbname,count(*) from {self.stable2} group by tbname;") + tdSql.checkRows(1) + tdSql.checkData(0, 1, qtime2 * once) print("check stable success") def run(self): @@ -194,12 +178,14 @@ class TDTestCase: self.reset_tb() self.test_stable_csv() self.test_bigcsv() - self.test_mix() self.check() + self.test_mix() + os.system(f"rm -rf {self.file1}") + os.system(f"rm -rf {self.file2}") tdSql.close() def stop(self): tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 29818803174c08f2e620969ea5c6cc7d3a1f6826 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 18:56:17 +0800 Subject: [PATCH 070/236] adjust --- tests/system-test/1-insert/ts-4272.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/ts-4272.py b/tests/system-test/1-insert/ts-4272.py index 496433664b..f64f0556bf 100644 --- a/tests/system-test/1-insert/ts-4272.py +++ b/tests/system-test/1-insert/ts-4272.py @@ -188,4 +188,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From cd85dae3fddc45f58eb29c85832013462c6c806b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 24 Dec 2023 23:30:00 +0800 Subject: [PATCH 071/236] fix(stream): fix the dead lock. --- source/libs/stream/inc/streamInt.h | 2 -- source/libs/stream/src/streamCheckpoint.c | 1 + source/libs/stream/src/streamMeta.c | 5 ++++- source/libs/stream/src/streamStart.c | 16 ++++++++++++---- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 341706ff3b..9b5134d449 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -109,8 +109,6 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId); -int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); -int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 2a766f21ec..b54adb0f96 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -114,6 +114,7 @@ static int32_t streamAlignCheckpoint(SStreamTask* pTask) { return atomic_sub_fetch_32(&pTask->chkInfo.downstreamAlignNum, 1); } +// todo handle down the transId of checkpoint to sink/agg tasks. static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) { SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); if (pChkpoint == NULL) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9e20a2997c..8b2330fda0 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1102,7 +1102,10 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; entry.chkpointTransId = (*pTask)->chkInfo.transId; - stInfo("s-task:%s send kill checkpoint trans info, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId); + + if (entry.checkpointFailed) { + stInfo("s-task:%s send kill checkpoint trans info, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId); + } } if ((*pTask)->exec.pWalReader != NULL) { diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 267dadf109..ea5e2edc09 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -379,10 +379,11 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { stDebug("s-task:%s enter into scan-history data stage, status:%s", id, p); streamTaskStartScanHistory(pTask); - // start the related fill-history task, when current task is ready - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - streamLaunchFillHistoryTask(pTask); - } + // NOTE: there will be an deadlock if launch fill history here. +// // start the related fill-history task, when current task is ready +// if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { +// streamLaunchFillHistoryTask(pTask); +// } return TSDB_CODE_SUCCESS; } @@ -400,6 +401,13 @@ void doProcessDownstreamReadyRsp(SStreamTask* pTask) { int64_t initTs = pTask->execInfo.init; int64_t startTs = pTask->execInfo.start; streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, initTs, startTs, true); + + // start the related fill-history task, when current task is ready + // not invoke in success callback due to the deadlock. + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } } static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { From 52c244d9ec0a9ce286a25379b3516159f0380222 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 23:48:37 +0800 Subject: [PATCH 072/236] adjust --- source/libs/planner/src/planPhysiCreater.c | 22 +++++++++++++++----- tests/system-test/2-query/count.py | 2 +- tests/system-test/2-query/group_partition.py | 10 ++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9c4e9f97ab..5c78ff4432 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -595,13 +595,25 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog } // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) - && pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { - SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; - if (pNode->isCountByTag) { - return true; + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + // case1 root select + SSelectStmt* pSelect = (SSelectStmt*)pRoot; + if (pSelect->hasCountFunc) { + if (NULL != pSelect->pGroupByList) { + return !keysHasCol(pSelect->pGroupByList); + } else if (NULL != pSelect->pPartitionByList) { + return !keysHasCol(pSelect->pPartitionByList); + } + } + // case2 inner agg + if (pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { + SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; + if (pNode->isCountByTag) { + return true; + } } } + return false; } diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 0e1d332d89..a80c6f07e9 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -123,7 +123,7 @@ class TDTestCase: tdSql.checkData(0, 1, 2) elif 'hyperloglog' == function_name: tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 58be8563fa..c08ea98ef6 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -83,6 +83,10 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + #inner select + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") + tdSql.checkRows(check_num) + # count + sum(col) tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) @@ -146,6 +150,10 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + #inner select + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") + tdSql.checkRows(check_num) + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") tdSql.checkRows(real_num) @@ -196,4 +204,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From da0ecc18b95af94abff05cece0acd002dfd38bd0 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 02:51:52 +0800 Subject: [PATCH 073/236] test --- source/libs/planner/src/planPhysiCreater.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 5c78ff4432..397258d6ca 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2311,9 +2311,11 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - pCxt->pNode = pLogicSubplan->pNode; + + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pLogicSubplan->pNode)) { + pCxt->pNode = pLogicSubplan->pNode; + } code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); - pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); } @@ -2414,6 +2416,7 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi break; } } + pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code) { *pPhysiPlan = pPlan; From 18dc6ff695bd1b05ac9573b6cd860fc415459dd3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 25 Dec 2023 09:07:47 +0800 Subject: [PATCH 074/236] enh(cmake/curl): download from github instead of curl.se --- cmake/curl_CMakeLists.txt.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index af7f005dda..197d978fb7 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -1,6 +1,7 @@ # curl ExternalProject_Add(curl2 - URL https://curl.se/download/curl-8.2.1.tar.gz + URL https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.gz + #URL https://curl.se/download/curl-8.2.1.tar.gz URL_HASH MD5=b25588a43556068be05e1624e0e74d41 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" From 4f29500f74509577dea1fb472c12fd34396cf2b0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 22 Dec 2023 16:26:45 +0800 Subject: [PATCH 075/236] fix: set commit log for mndDropSubByTopic --- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 44ee804d22..01d4d1029c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -1215,7 +1215,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) return -1; } - if (mndSetDropSubRedoLogs(pMnode, pTrans, pSub) < 0) { + if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) { sdbRelease(pSdb, pSub); sdbCancelFetch(pSdb, pIter); return -1; From 7edfbf8023ece6def03bfe97526012b49f3cd3b5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 25 Dec 2023 09:44:55 +0800 Subject: [PATCH 076/236] fix: set commit log for mndAddCompactDetailToTran --- source/dnode/mnode/impl/src/mndCompactDetail.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCompactDetail.c b/source/dnode/mnode/impl/src/mndCompactDetail.c index a1c0e95c20..6b1dd78093 100644 --- a/source/dnode/mnode/impl/src/mndCompactDetail.c +++ b/source/dnode/mnode/impl/src/mndCompactDetail.c @@ -290,11 +290,11 @@ int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* p SSdbRaw *pVgRaw = mndCompactDetailActionEncode(&compactDetail); if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { sdbFreeRaw(pVgRaw); return -1; } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); return 0; -} \ No newline at end of file +} From 00fac852e8e22cb5c05d7e6405f1f5040af3be7d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 09:55:24 +0800 Subject: [PATCH 077/236] fix(tsdb): fix invalid delete check. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 26 +++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 751569bf90..5dafe91d3d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2058,9 +2058,9 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum } if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { - bool deleted = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, + bool dropped = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->info.order, &pReader->info.verRange); - if (deleted) { + if (dropped) { return false; } } @@ -3340,11 +3340,14 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } // it is a valid data version - if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && - pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); - if (!dropped) { + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { return pRow; + } else { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } } @@ -3362,11 +3365,14 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p return NULL; } - if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && - pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); - if (!dropped) { + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { return pRow; + } else { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } } } From 1beb3347faa5c9e348887de01178f708687bffab Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 25 Dec 2023 10:18:19 +0800 Subject: [PATCH 078/236] fix: change compact rsp first column name --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..16e0332ee9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8769,7 +8769,7 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; - strcpy((*pSchema)[0].name, "name"); + strcpy((*pSchema)[0].name, "result"); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; From 12bbf50c8f48022bef32088fd9c67a75c60ccac0 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 10:29:15 +0800 Subject: [PATCH 079/236] adjust --- tests/system-test/1-insert/ts-4272.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/1-insert/ts-4272.py b/tests/system-test/1-insert/ts-4272.py index f64f0556bf..f2bacd0c2b 100644 --- a/tests/system-test/1-insert/ts-4272.py +++ b/tests/system-test/1-insert/ts-4272.py @@ -49,10 +49,10 @@ class TDTestCase: tdSql.execute(f"create table {self.stable2}_1 using {self.stable2}(t_int) tags(2);") def test(self, sql): - sql = f"use {self.db};" + sql - os.system(f'taos -s "{sql}"') - # print(f'{sql}\n') - # tdSql.execute(sql, 1) + # sql = f"use {self.db};" + sql + # os.system(f'taos -s "{sql}"') + print(f'{sql}\n') + tdSql.execute(sql, 1) def check(self): # same table, auto create + create From e5087f6e23ae34c69c9df9f942b5bc8acc490c07 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 10:40:36 +0800 Subject: [PATCH 080/236] fix(tsdb): fix invalid delete check. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 5dafe91d3d..5ad067c113 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1379,11 +1379,13 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc pScanInfo->sttKeyInfo.nextProcKey = key; if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) { - bool dropped = hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange); - if (!dropped) { + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange)) { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; return true; } + } else { + pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; + return true; } } } From 15694df001da02ab54ced68b014d6e48fe4a1cff Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 11:54:10 +0800 Subject: [PATCH 081/236] adjust --- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/planner/src/planLogicCreater.c | 20 ++++++++-------- source/libs/planner/src/planOptimizer.c | 1 - source/libs/planner/src/planPhysiCreater.c | 27 +++------------------- tests/parallel_test/cases.task | 4 ++++ 5 files changed, 19 insertions(+), 35 deletions(-) diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 2af31a8c7a..c68fd81d22 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -424,6 +424,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(groupOrderScan); COPY_SCALAR_FIELD(onlyMetaCtbIdx); COPY_SCALAR_FIELD(filesetDelimited); + COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } @@ -453,7 +454,6 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { COPY_SCALAR_FIELD(isGroupTb); COPY_SCALAR_FIELD(isPartTb); COPY_SCALAR_FIELD(hasGroup); - COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 17b1b272ba..f55689c472 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -486,6 +486,16 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = tagScanSetExecutionMode(pScan); } + bool isCountByTag = false; + if (pSelect->hasCountFunc) { + if (pSelect->pGroupByList) { + isCountByTag = !keysHasCol(pSelect->pGroupByList); + } else if (pSelect->pPartitionByList) { + isCountByTag = !keysHasCol(pSelect->pPartitionByList); + } + } + pScan->isCountByTag = isCountByTag; + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pScan; } else { @@ -748,15 +758,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; - bool isCountByTag = false; - if (pSelect->hasCountFunc) { - if (pSelect->pGroupByList) { - isCountByTag = !keysHasCol(pSelect->pGroupByList); - } else if (pSelect->pPartitionByList) { - isCountByTag = !keysHasCol(pSelect->pPartitionByList); - } - } - pAgg->isCountByTag = isCountByTag; + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 9ecbd3ae9f..aa3181e166 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2783,7 +2783,6 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, pNew->isGroupTb = pAgg->isGroupTb; pNew->isPartTb = pAgg->isPartTb; pNew->hasGroup = pAgg->hasGroup; - pNew->isCountByTag = pAgg->isCountByTag; pNew->node.pChildren = nodesCloneList(pAgg->node.pChildren); int32_t code = 0; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 397258d6ca..0db0fd77fb 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -37,7 +37,6 @@ typedef struct SPhysiPlanContext { SArray* pLocationHelper; bool hasScan; bool hasSysScan; - SLogicNode* pNode; // tmp record LogicSubplan->pNode } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey, int32_t keyBufSize) { @@ -595,23 +594,8 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog } // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { - // case1 root select - SSelectStmt* pSelect = (SSelectStmt*)pRoot; - if (pSelect->hasCountFunc) { - if (NULL != pSelect->pGroupByList) { - return !keysHasCol(pSelect->pGroupByList); - } else if (NULL != pSelect->pPartitionByList) { - return !keysHasCol(pSelect->pPartitionByList); - } - } - // case2 inner agg - if (pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { - SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; - if (pNode->isCountByTag) { - return true; - } - } + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) && pScanLogicNode->isCountByTag) { + return true; } return false; @@ -2312,9 +2296,6 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pLogicSubplan->pNode)) { - pCxt->pNode = pLogicSubplan->pNode; - } code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); @@ -2416,7 +2397,6 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi break; } } - pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code) { *pPhysiPlan = pPlan; @@ -2468,8 +2448,7 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, - .hasSysScan = false, - .pNode = NULL}; + .hasSysScan = false}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e63f86743b..5e7ef407f0 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -348,6 +348,10 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py From baee536cb06976332fe4d22305305edc0ee1d013 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 11:54:23 +0800 Subject: [PATCH 082/236] adjust --- include/libs/nodes/plannodes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 55ac3b86bf..f0383e8168 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -119,6 +119,7 @@ typedef struct SScanLogicNode { bool groupOrderScan; bool onlyMetaCtbIdx; // for tag scan with no tbname bool filesetDelimited; // returned blocks delimited by fileset + bool isCountByTag; // true if selectstmt hasCountFunc & part by tag/tbname } SScanLogicNode; typedef struct SJoinLogicNode { @@ -147,7 +148,6 @@ typedef struct SAggLogicNode { bool isGroupTb; bool isPartTb; // true if partition keys has tbname bool hasGroup; - bool isCountByTag; // true if hasCountFunc & part by tag/tbname } SAggLogicNode; typedef struct SProjectLogicNode { From 31c54e5edabc71924d72befc3194942b8b40fb73 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 12:42:25 +0800 Subject: [PATCH 083/236] adjust --- tests/system-test/2-query/count.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index a80c6f07e9..9bf9a4b9cc 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -182,8 +182,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') From 60977d56bd2f6756542e5eb1d773cb2b04976b1f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 12:47:16 +0800 Subject: [PATCH 084/236] refactor: set attribute only on Linux --- source/libs/stream/src/streamMeta.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 3ee51ffd03..d09d370a36 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -376,12 +376,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->rid = taosAddRef(streamMetaId, pMeta); + // set the attribute when running on Linux OS +#if defined LINUX TdThreadRwlockAttr attr; taosThreadRwlockAttrInit(&attr); pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); taosThreadRwlockInit(&pMeta->lock, &attr); + taosThreadRwlockAttrDestroy(&attr); +#endif int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); From f672df239e176a5256e76f9937d9e19a8da5b2cc Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 13:33:45 +0800 Subject: [PATCH 085/236] adjust --- tests/system-test/2-query/count.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 9bf9a4b9cc..3ba651622a 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -182,8 +182,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') From f567c19d05acd296799aa0d34551f5745f5177bf Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 25 Dec 2023 15:59:29 +0800 Subject: [PATCH 086/236] fix: add new-frame public file --- .../enterprise/multi-level/mlevel_basic.json | 58 +++++++++++++ .../enterprise/multi-level/mlevel_basic.py | 61 +++++++++++--- tests/army/frame/caseBase.py | 83 ++++++++++++++++++- tests/army/frame/eos.py | 35 ++++++++ tests/army/frame/epath.py | 53 ++++++++++++ tests/army/frame/etime.py | 23 +++++ tests/army/frame/etool.py | 40 +++++++++ tests/army/frame/pathFinding.py | 83 ------------------- tests/army/frame/sql.py | 13 ++- 9 files changed, 352 insertions(+), 97 deletions(-) create mode 100644 tests/army/enterprise/multi-level/mlevel_basic.json create mode 100644 tests/army/frame/eos.py create mode 100644 tests/army/frame/epath.py create mode 100644 tests/army/frame/etime.py create mode 100644 tests/army/frame/etool.py delete mode 100644 tests/army/frame/pathFinding.py diff --git a/tests/army/enterprise/multi-level/mlevel_basic.json b/tests/army/enterprise/multi-level/mlevel_basic.json new file mode 100644 index 0000000000..7df5b1caa8 --- /dev/null +++ b/tests/army/enterprise/multi-level/mlevel_basic.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"1d", + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"now-10d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index e3a3f57c74..a12b1b59eb 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -15,26 +15,65 @@ import sys import time import taos +import frame + from frame.log import * from frame.cases import * from frame.sql import * +from frame.caseBase import * + + + +class TDTestCase(TBase): + + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = frame.etool.curFile(__file__, "mlevel_basic.json") + frame.etool.runBenchmark(json=json) + + # set insert data information + self.childtable_count = 2 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + def doAction(self): + tdLog.info(f"trim database.") + self.trimDb() + + def saveData(self): + tdLog.info(f"check correct.") + + def checkSaveCorrent(self): + tdLog.info(f"check correct.") + + def checkCorrect(self): + tdLog.info(f"check correct.") -class TDTestCase: - # init - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) # run def run(self): - # check two db query result same - tdLog.info(f"hello world.") + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() - # stop - def stop(self): - tdSql.close() tdLog.success(f"{__file__} successfully executed") + + tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index abaa92a51c..7b371cd822 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -16,10 +16,89 @@ import os import time import datetime -class caseBase: - def __init__(self): +from log import * +from sql import * + +# test case base +class DbBase: + +# +# frame call +# + + # init + def init(self, conn, logSql, replicaVar=1): + # save param + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), True) + # record server information self.dnodeNum = 0 self.mnodeNum = 0 self.mLevel = 0 self.mLevelDisk = 0 + + # test case information + self.db = "db" + self.stb = "stb" + + # variant in taosBenchmark json + self.childtable_count = 2 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + # sql + self.sqlSum = f"select sum(ic) from {self.stb}" + self.sqlMax = f"select max(ic) from {self.stb}" + self.sqlMin = f"select min(ic) from {self.stb}" + self.sqlAvg = f"select avg(ic) from {self.stb}" + + + # stop + def stop(self): + tdSql.close() + + +# +# db action +# + + def trimDb(self): + tdSql.execute(f"trim database {self.db}") + + def compactDb(self): + tdSql.execute(f"compact database {self.db}") + + +# +# check db correct +# + + # basic + def checkInsertCorrect(self): + # check count + sql = f"select count(*) from {self.stb}" + tdSql.checkAgg(sql, self.childtable_count * self.insert_rows) + + # check child table count + sql = f" select count(*) from (select count(*) as cnt , tbname from {self.stb} group by tbname) where cnt = {self.insert_rows} " + tdSql.checkAgg(sql, self.childtable_count) + + # check step + sql = f"select count(*) from (select diff(ts) as dif from {self.stb} group by tbname) where dif != {self.timestamp_step}" + tdSql.checkAgg(sql, 0) + + # save agg result + def snapshotAgg(self): + + self.sum = tdSql.getFirstValue(self.sqlSum) + self.avg = tdSql.getFirstValue(self.sqlAvg) + self.min = tdSql.getFirstValue(self.sqlMin) + self.max = tdSql.getFirstValue(self.sqlMax) + + # check agg + def checkAggCorrect(self): + tdSql.checkAgg(self.sqlSum, self.sum) + tdSql.checkAgg(self.sqlAvg, self.avg) + tdSql.checkAgg(self.sqlMin, self.min) + tdSql.checkAgg(self.sqlMax, self.max) diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py new file mode 100644 index 0000000000..2d8559f3eb --- /dev/null +++ b/tests/army/frame/eos.py @@ -0,0 +1,35 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about system funciton extension +# + +import sys +import os +import time +import datetime + +# if windows platform return True +def isWin(): + return sys.platform.system().lower() == 'windows' + +# wait util execute file finished +def exe(file): + return os.system(file) + +# execute file and return immediately +def exeNoWait(file): + print("exe no wait") + + diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py new file mode 100644 index 0000000000..644e448fb7 --- /dev/null +++ b/tests/army/frame/epath.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about path function extension +# + +import os +from frame.log import * + +# build/bin path +binPath = "" + +def binPath(self): + global binPath + + if binPath != "": + return binPath + + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community/tests" in selfPath): + projPath = selfPath[:selfPath.find("community/tests")] + else: + projPath = selfPath[:selfPath.find("TDengine/tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + # check + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + # return + binPath = buildPath + "/build/bin/" + return binPath + +def binFile(self, file): + return binPath() + file \ No newline at end of file diff --git a/tests/army/frame/etime.py b/tests/army/frame/etime.py new file mode 100644 index 0000000000..2ee223e122 --- /dev/null +++ b/tests/army/frame/etime.py @@ -0,0 +1,23 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about tools funciton extension +# + +import sys +import os +import time +import datetime + + diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py new file mode 100644 index 0000000000..5ba29ee1c3 --- /dev/null +++ b/tests/army/frame/etool.py @@ -0,0 +1,40 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about system funciton extension +# + +import sys +import os +import time +import datetime +import epath +import eos + +# run taosBenchmark with command or json file mode +def runBenchmark(command = "", json = "") : + # get taosBenchmark path + bmFile = epath.binFile("taosBenchmark") + if eos.isWin(): + bmFile += ".exe" + + # run + if command != "": + eos.exe(bmFile + " " + command) + if json != "": + eos.exe(bmFile + " -f " + json) + +# get current directory file name +def curFile(fullPath, file): + return os.path.dirname(fullPath) + "/" + file diff --git a/tests/army/frame/pathFinding.py b/tests/army/frame/pathFinding.py deleted file mode 100644 index df03f0ed68..0000000000 --- a/tests/army/frame/pathFinding.py +++ /dev/null @@ -1,83 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - - -import os -from frame.log import * - - - -class TDFindPath: - """This class is for finding path within TDengine - """ - def __init__(self): - self.file = "" - - - def init(self, file): - """[summary] - - Args: - file (str): the file location you want to start the query. Generally using __file__ - """ - self.file = file - - def getTaosdemoPath(self): - """for finding the path of directory containing taosdemo - - Returns: - str: the path to directory containing taosdemo - """ - selfPath = os.path.dirname(os.path.realpath(self.file)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info(f"taosd found in {buildPath}") - return buildPath + "/build/bin/" - - def getTDenginePath(self): - """for finding the root path of TDengine - - Returns: - str: the root path of TDengine - """ - selfPath = os.path.dirname(os.path.realpath(self.file)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - print(projPath) - for root, dirs, files in os.walk(projPath): - if ("sim" in dirs): - print(root) - rootRealPath = os.path.realpath(root) - if (rootRealPath == ""): - tdLog.exit("TDengine not found!") - else: - tdLog.info(f"TDengine found in {rootRealPath}") - return rootRealPath - -tdFindPath = TDFindPath() \ No newline at end of file diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 409dc6c96c..b3ee1ed5f8 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -535,6 +535,16 @@ class TDSql: tdLog.info("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) raise Exception + # check like select count(*) ... sql + def checkAgg(self, sql, expectCnt): + self.query(sql) + self.checkRowCol(0, 0, expectCnt) + + # get first value + def getFirstValue(self, sql) : + self.query(sql) + return self.getData(0, 0) + def get_times(self, time_str, precision="ms"): caller = inspect.getframeinfo(inspect.stack()[1][0]) if time_str[-1] not in TAOS_TIME_INIT: @@ -602,6 +612,7 @@ class TDSql: if self.cursor.istype(col, "BIGINT UNSIGNED"): return "BIGINT UNSIGNED" + ''' def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 @@ -630,7 +641,7 @@ class TDSql: tdLog.exit("taosd state is %d != expect:%d" %args) pass - ''' + def haveFile(self, dir, state): if os.path.exists(dir) and os.path.isdir(dir): if not os.listdir(dir): From a7c8563c0616859f4fbc6bf8eca8cd96b7c84899 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 25 Dec 2023 16:13:30 +0800 Subject: [PATCH 087/236] fix: remove useless code --- source/dnode/vnode/src/inc/tsdb.h | 38 +- source/dnode/vnode/src/inc/vnodeInt.h | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 1628 ---------------------- source/dnode/vnode/src/tsdb/tsdbFS.c | 313 +---- source/dnode/vnode/src/tsdb/tsdbRead2.c | 119 +- 5 files changed, 70 insertions(+), 2032 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d347ce30d8..ebaf0851e6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -256,18 +256,7 @@ void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); // tsdbFS.c ============================================================================================== int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback); int32_t tsdbFSClose(STsdb *pTsdb); -int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS); -void tsdbFSDestroy(STsdbFS *pFS); -int32_t tDFileSetCmprFn(const void *p1, const void *p2); -int32_t tsdbFSCommit(STsdb *pTsdb); -int32_t tsdbFSRollback(STsdb *pTsdb); -int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFS); -int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS); -void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS); void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); - -int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet); -int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile); // tsdbReaderWriter.c ============================================================================================== // SDataFWriter int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); @@ -737,7 +726,6 @@ struct STsdbReadSnap { SMemTable *pIMem; SQueryNode *pINode; TFileSetArray *pfSetArray; - STsdbFS fs; }; struct SDataFWriter { @@ -796,16 +784,16 @@ typedef struct { } SSttTableRowsInfo; typedef struct SSttBlockLoadInfo { - SBlockDataInfo blockData[2]; // buffered block data - SArray *aSttBlk; - int32_t currentLoadBlockIndex; - STSchema *pSchema; - int16_t *colIds; - int32_t numOfCols; - bool checkRemainingRow; // todo: no assign value? - bool isLast; - bool sttBlockLoaded; - SSttTableRowsInfo info; + SBlockDataInfo blockData[2]; // buffered block data + SArray *aSttBlk; + int32_t currentLoadBlockIndex; + STSchema *pSchema; + int16_t *colIds; + int32_t numOfCols; + bool checkRemainingRow; // todo: no assign value? + bool isLast; + bool sttBlockLoaded; + SSttTableRowsInfo info; SSttBlockLoadCostInfo cost; } SSttBlockLoadInfo; @@ -894,15 +882,15 @@ typedef struct { _load_tomb_fn loadTombFn; void *pReader; void *idstr; - bool rspRows; // response the rows in stt-file, if possible + bool rspRows; // response the rows in stt-file, if possible } SMergeTreeConf; typedef struct SSttDataInfoForTable { - SArray* pTimeWindowList; + SArray *pTimeWindowList; int64_t numOfRows; } SSttDataInfoForTable; -int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable* pTableInfo); +int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable *pTableInfo); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); void tMergeTreePinSttBlock(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 84f51827ba..c1a4754b62 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -95,7 +95,7 @@ typedef struct SQueryNode SQueryNode; #define VNODE_RSMA2_DIR "rsma2" #define VNODE_TQ_STREAM "stream" -#if SUSPEND_RESUME_TEST // only for test purpose +#if SUSPEND_RESUME_TEST // only for test purpose #define VNODE_BUFPOOL_SEGMENTS 1 #else #define VNODE_BUFPOOL_SEGMENTS 3 @@ -216,8 +216,6 @@ int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbCacheCommit(STsdb* pTsdb); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); -// int32_t tsdbFinishCommit(STsdb* pTsdb); -// int32_t tsdbRollbackCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 55ae25aee4..37413ef920 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -15,92 +15,6 @@ #include "tsdb.h" -typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; - -#define USE_STREAM_COMPRESSION 0 - -typedef struct { - SRBTreeNode n; - SRowInfo r; - EDataIterT type; - union { - struct { - int32_t iTbDataP; - STbDataIter iter; - }; // memory data iter - struct { - int32_t iStt; - SArray *aSttBlk; - int32_t iSttBlk; - SBlockData bData; - int32_t iRow; - }; // stt file data iter - }; -} SDataIter; - -typedef struct { - STsdb *pTsdb; - /* commit data */ - int64_t commitID; - int32_t minutes; - int8_t precision; - int32_t minRow; - int32_t maxRow; - int8_t cmprAlg; - int8_t sttTrigger; - SArray *aTbDataP; // memory - STsdbFS fs; // disk - // -------------- - TSKEY nextKey; // reset by each table commit - int32_t commitFid; - int32_t expLevel; - TSKEY minKey; - TSKEY maxKey; - // commit file data - struct { - SDataFReader *pReader; - SArray *aBlockIdx; // SArray - int32_t iBlockIdx; - SBlockIdx *pBlockIdx; - SMapData mBlock; // SMapData - SBlockData bData; - } dReader; - struct { - SDataIter *pIter; - SRBTree rbt; - SDataIter dataIter; - SDataIter aDataIter[TSDB_STT_TRIGGER_ARRAY_SIZE]; - int8_t toLastOnly; - }; - struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; // SArray - SArray *aSttBlk; // SArray - SMapData mBlock; // SMapData - SBlockData bData; -#if USE_STREAM_COMPRESSION - SDiskDataBuilder *pBuilder; -#else - SBlockData bDatal; -#endif - } dWriter; - SSkmInfo skmTable; - SSkmInfo skmRow; - /* commit del */ - SDelFReader *pDelFReader; - SDelFWriter *pDelFWriter; - SArray *aDelIdx; // SArray - SArray *aDelIdxN; // SArray - SArray *aDelData; // SArray -} SCommitter; - -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo); -static int32_t tsdbCommitData(SCommitter *pCommitter); -static int32_t tsdbCommitDel(SCommitter *pCommitter); -static int32_t tsdbCommitCache(SCommitter *pCommitter); -static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno); -static int32_t tsdbNextCommitRow(SCommitter *pCommitter); - int32_t tRowInfoCmprFn(const void *p1, const void *p2) { SRowInfo *pInfo1 = (SRowInfo *)p1; SRowInfo *pInfo2 = (SRowInfo *)p2; @@ -150,1545 +64,3 @@ _exit: } return code; } - -int32_t tsdbPrepareCommit(STsdb *pTsdb) { - taosThreadMutexLock(&pTsdb->mutex); - ASSERT(pTsdb->imem == NULL); - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - taosThreadMutexUnlock(&pTsdb->mutex); - - return 0; -} - -int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { - if (!pTsdb) return 0; - - int32_t code = 0; - int32_t lino = 0; - SCommitter commith; - SMemTable *pMemTable = pTsdb->imem; - - // check - if (pMemTable->nRow == 0 && pMemTable->nDel == 0) { - taosThreadMutexLock(&pTsdb->mutex); - pTsdb->imem = NULL; - taosThreadMutexUnlock(&pTsdb->mutex); - - tsdbUnrefMemTable(pMemTable, NULL, true); - goto _exit; - } - - // start commit - code = tsdbStartCommit(pTsdb, &commith, pInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - // commit impl - code = tsdbCommitData(&commith); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommitDel(&commith); - TSDB_CHECK_CODE(code, lino, _exit); - - // end commit - code = tsdbEndCommit(&commith, 0); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbEndCommit(&commith, code); - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - if ((pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if ((pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if ((pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - SDelFile *pDelFileR = pCommitter->fs.pDelFile; - if (pDelFileR) { - code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // prepare new - SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); - } - return code; -} - -static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) { - int32_t code = 0; - int32_t lino = 0; - SDelData *pDelData; - tb_uid_t suid; - tb_uid_t uid; - - if (pTbData) { - suid = pTbData->suid; - uid = pTbData->uid; - - if (pTbData->pHead == NULL) { - pTbData = NULL; - } - } - - if (pDelIdx) { - suid = pDelIdx->suid; - uid = pDelIdx->uid; - - code = tsdbReadDelDatav1(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - taosArrayClear(pCommitter->aDelData); - } - - if (pTbData == NULL && pDelIdx == NULL) goto _exit; - - SDelIdx delIdx = {.suid = suid, .uid = uid}; - - // memory - pDelData = pTbData ? pTbData->pHead : NULL; - for (; pDelData; pDelData = pDelData->pNext) { - if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - // write - code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - // put delIdx - if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - - code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->pDelFReader) { - code = tsdbDelFReaderClose(&pCommitter->pDelFReader); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosArrayDestroy(pCommitter->aDelIdx); - taosArrayDestroy(pCommitter->aDelData); - taosArrayDestroy(pCommitter->aDelIdxN); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { - int32_t code = 0; - int32_t lino = 0; - - if (suid) { - if (pSkmInfo->suid == suid) { - pSkmInfo->uid = uid; - goto _exit; - } - } else { - if (pSkmInfo->uid == uid) goto _exit; - } - - pSkmInfo->suid = suid; - pSkmInfo->uid = uid; - tDestroyTSchema(pSkmInfo->pTSchema); - code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - return code; -} - -static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { - int32_t code = 0; - int32_t lino = 0; - - if (pCommitter->skmRow.pTSchema) { - if (pCommitter->skmRow.suid == suid) { - if (suid == 0) { - if (pCommitter->skmRow.uid == uid && sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } - } - } - - pCommitter->skmRow.suid = suid; - pCommitter->skmRow.uid = uid; - tDestroyTSchema(pCommitter->skmRow.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - return code; -} - -static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - ASSERT(pCommitter->dReader.pBlockIdx); - - pCommitter->dReader.iBlockIdx++; - if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { - pCommitter->dReader.pBlockIdx = - (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - - code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - TSDB_CHECK_CODE(code, lino, _exit); - - ASSERT(pCommitter->dReader.mBlock.nItem > 0); - } else { - pCommitter->dReader.pBlockIdx = NULL; - } - -_exit: - return code; -} - -static int32_t tDataIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { - SDataIter *pIter1 = (SDataIter *)((uint8_t *)n1 - offsetof(SDataIter, n)); - SDataIter *pIter2 = (SDataIter *)((uint8_t *)n2 - offsetof(SDataIter, n)); - - return tRowInfoCmprFn(&pIter1->r, &pIter2->r); -} - -static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - pCommitter->pIter = NULL; - tRBTreeCreate(&pCommitter->rbt, tDataIterCmprFn); - - // memory - TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; - SDataIter *pIter = &pCommitter->dataIter; - pIter->type = MEMORY_DATA_ITER; - pIter->iTbDataP = 0; - for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) { - STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); - tsdbTbDataIterOpen(pTbData, &tKey, 0, &pIter->iter); - TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - pRow = NULL; - } - - if (pRow == NULL) continue; - - pIter->r.suid = pTbData->suid; - pIter->r.uid = pTbData->uid; - pIter->r.row = *pRow; - break; - } - ASSERT(pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)); - tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); - - // disk - pCommitter->toLastOnly = 0; - SDataFReader *pReader = pCommitter->dReader.pReader; - if (pReader) { - if (pReader->pSet->nSttF >= pCommitter->sttTrigger) { - int8_t iIter = 0; - for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { - pIter = &pCommitter->aDataIter[iIter]; - pIter->type = STT_DATA_ITER; - pIter->iStt = iStt; - - code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; - - pIter->iSttBlk = 0; - SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); - code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->iRow = 0; - pIter->r.suid = pIter->bData.suid; - pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); - - tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); - iIter++; - } - } else { - for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { - SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; - if (pSttFile->size > pSttFile->offset) { - pCommitter->toLastOnly = 1; - break; - } - } - } - } - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - SDFileSet *pRSet = NULL; - - // memory - pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); - pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec()); - tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, - &pCommitter->maxKey); -#if 0 - ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey); -#endif - - pCommitter->nextKey = TSKEY_MAX; - - // Reader - SDFileSet tDFileSet = {.fid = pCommitter->commitFid}; - pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); - if (pRSet) { - code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); - TSDB_CHECK_CODE(code, lino, _exit); - - // data - code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - pCommitter->dReader.iBlockIdx = 0; - if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { - pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); - code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - pCommitter->dReader.pBlockIdx = NULL; - } - tBlockDataReset(&pCommitter->dReader.bData); - } else { - pCommitter->dReader.pBlockIdx = NULL; - } - - // Writer - SHeadFile fHead = {.commitID = pCommitter->commitID}; - SDataFile fData = {.commitID = pCommitter->commitID}; - SSmaFile fSma = {.commitID = pCommitter->commitID}; - SSttFile fStt = {.commitID = pCommitter->commitID}; - SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; - if (pRSet) { - ASSERT(pRSet->nSttF <= pCommitter->sttTrigger); - fData = *pRSet->pDataF; - fSma = *pRSet->pSmaF; - wSet.diskId = pRSet->diskId; - if (pRSet->nSttF < pCommitter->sttTrigger) { - for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) { - wSet.aSttF[iStt] = pRSet->aSttF[iStt]; - } - wSet.nSttF = pRSet->nSttF + 1; - } else { - wSet.nSttF = 1; - } - } else { - SDiskID did = {0}; - if (tfsAllocDisk(pTsdb->pVnode->pTfs, pCommitter->expLevel, &did) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - code = tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); - TSDB_CHECK_CODE(code, lino, _exit); - wSet.diskId = did; - wSet.nSttF = 1; - } - wSet.aSttF[wSet.nSttF - 1] = &fStt; - code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); - TSDB_CHECK_CODE(code, lino, _exit); - - taosArrayClear(pCommitter->dWriter.aBlockIdx); - taosArrayClear(pCommitter->dWriter.aSttBlk); - tMapDataReset(&pCommitter->dWriter.mBlock); - tBlockDataReset(&pCommitter->dWriter.bData); -#if USE_STREAM_COMPRESSION - tDiskDataBuilderClear(pCommitter->dWriter.pBuilder); -#else - tBlockDataReset(&pCommitter->dWriter.bDatal); -#endif - - // open iter - code = tsdbOpenCommitIter(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { - int32_t code = 0; - int32_t lino = 0; - - if (pBlockData->nRow == 0) return code; - - SDataBlk dataBlk; - tDataBlkReset(&dataBlk); - - // info - dataBlk.nRow += pBlockData->nRow; - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; - - if (iRow == 0) { - if (tsdbKeyCmprFn(&dataBlk.minKey, &key) > 0) { - dataBlk.minKey = key; - } - } else { - if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { - dataBlk.hasDup = 1; - } - } - - if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&dataBlk.maxKey, &key) < 0) { - dataBlk.maxKey = key; - } - - dataBlk.minVer = TMIN(dataBlk.minVer, key.version); - dataBlk.maxVer = TMAX(dataBlk.maxVer, key.version); - } - - // write - dataBlk.nSubBlock++; - code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], - ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); - TSDB_CHECK_CODE(code, lino, _exit); - - // put SDataBlk - code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - // clear - tBlockDataClear(pBlockData); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { - int32_t code = 0; - int32_t lino = 0; - SSttBlk sstBlk; - - if (pBlockData->nRow == 0) return code; - - // info - sstBlk.suid = pBlockData->suid; - sstBlk.nRow = pBlockData->nRow; - sstBlk.minKey = TSKEY_MAX; - sstBlk.maxKey = TSKEY_MIN; - sstBlk.minVer = VERSION_MAX; - sstBlk.maxVer = VERSION_MIN; - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - sstBlk.minKey = TMIN(sstBlk.minKey, pBlockData->aTSKEY[iRow]); - sstBlk.maxKey = TMAX(sstBlk.maxKey, pBlockData->aTSKEY[iRow]); - sstBlk.minVer = TMIN(sstBlk.minVer, pBlockData->aVersion[iRow]); - sstBlk.maxVer = TMAX(sstBlk.maxVer, pBlockData->aVersion[iRow]); - } - sstBlk.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; - sstBlk.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; - - // write - code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - // push SSttBlk - if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // clear - tBlockDataClear(pBlockData); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilder, SArray *aSttBlk) { - int32_t code = 0; - int32_t lino = 0; - - if (pBuilder->nRow == 0) return code; - - // gnrt - const SDiskData *pDiskData; - const SBlkInfo *pBlkInfo; - code = tGnrtDiskData(pBuilder, &pDiskData, &pBlkInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - SSttBlk sttBlk = {.suid = pBuilder->suid, - .minUid = pBlkInfo->minUid, - .maxUid = pBlkInfo->maxUid, - .minKey = pBlkInfo->minKey, - .maxKey = pBlkInfo->maxKey, - .minVer = pBlkInfo->minVer, - .maxVer = pBlkInfo->maxVer, - .nRow = pBuilder->nRow}; - // write - code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - // push - if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // clear - tDiskDataBuilderClear(pBuilder); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - // write aBlockIdx - code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - // write aSttBlk - code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - // update file header - code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - - // upsert SDFileSet - code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); - TSDB_CHECK_CODE(code, lino, _exit); - - // close and sync - code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->dReader.pReader) { - code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { - int32_t code = 0; - int32_t lino = 0; - - while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { - SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; - code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbCommitterNextTableData(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); -static int32_t tsdbCommitFileData(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - // commit file data start - code = tsdbCommitFileDataStart(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - // impl - code = tsdbCommitFileDataImpl(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - // commit file data end - code = tsdbCommitFileDataEnd(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - tsdbDataFReaderClose(&pCommitter->dReader.pReader); - tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); - } - return code; -} - -// ---------------------------------------------------------------------------- -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo) { - int32_t code = 0; - int32_t lino = 0; - - memset(pCommitter, 0, sizeof(*pCommitter)); - ASSERT(pTsdb->imem && "last tsdb commit incomplete"); - - pCommitter->pTsdb = pTsdb; - pCommitter->commitID = pInfo->info.state.commitID; - pCommitter->minutes = pTsdb->keepCfg.days; - pCommitter->precision = pTsdb->keepCfg.precision; - pCommitter->minRow = pInfo->info.config.tsdbCfg.minRows; - pCommitter->maxRow = pInfo->info.config.tsdbCfg.maxRows; - pCommitter->cmprAlg = pInfo->info.config.tsdbCfg.compression; - pCommitter->sttTrigger = pInfo->info.config.sttTrigger; - pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); - if (pCommitter->aTbDataP == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - code = tsdbFSCopy(pTsdb, &pCommitter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - // reader - pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dReader.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pCommitter->dReader.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - // merger - for (int32_t iStt = 0; iStt < TSDB_STT_TRIGGER_ARRAY_SIZE; iStt++) { - SDataIter *pIter = &pCommitter->aDataIter[iStt]; - pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); - if (pIter->aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pIter->bData); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // writer - pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dWriter.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); - if (pCommitter->dWriter.aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pCommitter->dWriter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - -#if USE_STREAM_COMPRESSION - code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder); -#else - code = tBlockDataCreate(&pCommitter->dWriter.bDatal); -#endif - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static void tsdbCommitDataEnd(SCommitter *pCommitter) { - // reader - taosArrayDestroy(pCommitter->dReader.aBlockIdx); - tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataDestroy(&pCommitter->dReader.bData); - - // merger - for (int32_t iStt = 0; iStt < TSDB_STT_TRIGGER_ARRAY_SIZE; iStt++) { - SDataIter *pIter = &pCommitter->aDataIter[iStt]; - taosArrayDestroy(pIter->aSttBlk); - tBlockDataDestroy(&pIter->bData); - } - - // writer - taosArrayDestroy(pCommitter->dWriter.aBlockIdx); - taosArrayDestroy(pCommitter->dWriter.aSttBlk); - tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataDestroy(&pCommitter->dWriter.bData); -#if USE_STREAM_COMPRESSION - tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); -#else - tBlockDataDestroy(&pCommitter->dWriter.bDatal); -#endif - tDestroyTSchema(pCommitter->skmTable.pTSchema); - tDestroyTSchema(pCommitter->skmRow.pTSchema); -} - -static int32_t tsdbCommitData(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - // check - if (pMemTable->nRow == 0) goto _exit; - - // start ==================== - code = tsdbCommitDataStart(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - // impl ==================== - pCommitter->nextKey = pMemTable->minKey; - while (pCommitter->nextKey < TSKEY_MAX) { - code = tsdbCommitFileData(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // end ==================== - tsdbCommitDataEnd(pCommitter); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDel(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - if (pMemTable->nDel == 0) { - goto _exit; - } - - // start - code = tsdbCommitDelStart(pCommitter); - if (code) { - TSDB_CHECK_CODE(code, lino, _exit); - } - - // impl - int32_t iDelIdx = 0; - int32_t nDelIdx = taosArrayGetSize(pCommitter->aDelIdx); - int32_t iTbData = 0; - int32_t nTbData = taosArrayGetSize(pCommitter->aTbDataP); - STbData *pTbData; - SDelIdx *pDelIdx; - - ASSERT(nTbData > 0); - - pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); - pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; - while (true) { - if (pTbData == NULL && pDelIdx == NULL) break; - - if (pTbData && pDelIdx) { - int32_t c = tTABLEIDCmprFn(pTbData, pDelIdx); - - if (c == 0) { - goto _commit_mem_and_disk_del; - } else if (c < 0) { - goto _commit_mem_del; - } else { - goto _commit_disk_del; - } - } else if (pTbData) { - goto _commit_mem_del; - } else { - goto _commit_disk_del; - } - - _commit_mem_del: - code = tsdbCommitTableDel(pCommitter, pTbData, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; - continue; - - _commit_disk_del: - code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - iDelIdx++; - pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; - continue; - - _commit_mem_and_disk_del: - code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; - iDelIdx++; - pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; - continue; - } - - // end - code = tsdbCommitDelEnd(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); - } - return code; -} - -static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - - if (eno) { - code = eno; - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbFSPrepareCommit(pCommitter->pTsdb, &pCommitter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - tsdbFSDestroy(&pCommitter->fs); - taosArrayDestroy(pCommitter->aTbDataP); - pCommitter->aTbDataP = NULL; - if (code || eno) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); - } - return code; -} - -// ================================================================================ - -static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { - return (pCommitter->pIter) ? &pCommitter->pIter->r : NULL; -} - -static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - if (pCommitter->pIter) { - SDataIter *pIter = pCommitter->pIter; - if (pCommitter->pIter->type == MEMORY_DATA_ITER) { // memory - tsdbTbDataIterNext(&pIter->iter); - TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); - while (true) { - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - pRow = NULL; - } - - if (pRow) { - pIter->r.suid = pIter->iter.pTbData->suid; - pIter->r.uid = pIter->iter.pTbData->uid; - pIter->r.row = *pRow; - break; - } - - pIter->iTbDataP++; - if (pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)) { - STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); - TSDBKEY keyFrom = {.ts = pCommitter->minKey, .version = VERSION_MIN}; - tsdbTbDataIterOpen(pTbData, &keyFrom, 0, &pIter->iter); - pRow = tsdbTbDataIterGet(&pIter->iter); - continue; - } else { - pCommitter->pIter = NULL; - break; - } - } - } else if (pCommitter->pIter->type == STT_DATA_ITER) { // last file - pIter->iRow++; - if (pIter->iRow < pIter->bData.nRow) { - pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; - pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); - } else { - pIter->iSttBlk++; - if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) { - SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); - - code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData); - if (code) goto _exit; - - pIter->iRow = 0; - pIter->r.suid = pIter->bData.suid; - pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); - } else { - pCommitter->pIter = NULL; - } - } - } else { - ASSERT(0); - } - - // compare with min in RB Tree - pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); - if (pCommitter->pIter && pIter) { - int32_t c = tRowInfoCmprFn(&pCommitter->pIter->r, &pIter->r); - if (c > 0) { - tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); - pCommitter->pIter = NULL; - } else { - ASSERT(c); - } - } - } - - if (pCommitter->pIter == NULL) { - pCommitter->pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); - if (pCommitter->pIter) { - tRBTreeDrop(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); - } - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; - int32_t lino = 0; - - SBlockData *pBlockData = &pCommitter->dWriter.bData; - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; - - tBlockDataClear(pBlockData); - while (pRowInfo) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo) { - if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { - pRowInfo = NULL; - } else { - TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); - if (tsdbKeyCmprFn(&tKey, &pDataBlk->minKey) >= 0) pRowInfo = NULL; - } - } - - if (pBlockData->nRow >= pCommitter->maxRow) { - code = - tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; - int32_t lino = 0; - - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; - SBlockData *pBDataR = &pCommitter->dReader.bData; - SBlockData *pBDataW = &pCommitter->dWriter.bData; - - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); - TSDB_CHECK_CODE(code, lino, _exit); - - tBlockDataClear(pBDataW); - int32_t iRow = 0; - TSDBROW row = tsdbRowFromBlockData(pBDataR, 0); - TSDBROW *pRow = &row; - - while (pRow && pRowInfo) { - int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); - if (c < 0) { - code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - iRow++; - if (iRow < pBDataR->nRow) { - row = tsdbRowFromBlockData(pBDataR, iRow); - } else { - pRow = NULL; - } - } else if (c > 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo) { - if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { - pRowInfo = NULL; - } else { - TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); - if (tsdbKeyCmprFn(&tKey, &pDataBlk->maxKey) > 0) pRowInfo = NULL; - } - } - } else { - ASSERT(0 && "dup rows not allowed"); - } - - if (pBDataW->nRow >= pCommitter->maxRow) { - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - while (pRow) { - code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - iRow++; - if (iRow < pBDataR->nRow) { - row = tsdbRowFromBlockData(pBDataR, iRow); - } else { - pRow = NULL; - } - - if (pBDataW->nRow >= pCommitter->maxRow) { - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - int32_t lino = 0; - - SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; - - ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); - if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { - int32_t iBlock = 0; - SDataBlk block; - SDataBlk *pDataBlk = █ - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - - ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid); - - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - while (pDataBlk && pRowInfo) { - SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; - int32_t c = tDataBlkCmprFn(pDataBlk, &tBlock); - - if (c < 0) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - } else { - pDataBlk = NULL; - } - } else if (c > 0) { - code = tsdbCommitAheadBlock(pCommitter, pDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; - } else { - code = tsdbCommitMergeBlock(pCommitter, pDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - } else { - pDataBlk = NULL; - } - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; - } - } - - while (pDataBlk) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - } else { - pDataBlk = NULL; - } - } - - code = tsdbCommitterNextTableData(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - int32_t lino = 0; - -#if USE_STREAM_COMPRESSION - SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; - if (pBuilder->suid || pBuilder->uid) { - if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - tDiskDataBuilderClear(pBuilder); - } - } - - if (!pBuilder->suid && !pBuilder->uid) { - ASSERT(pCommitter->skmTable.suid == id.suid); - ASSERT(pCommitter->skmTable.uid == id.uid); - code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); - TSDB_CHECK_CODE(code, lino, _exit); - } -#else - SBlockData *pBData = &pCommitter->dWriter.bDatal; - if (pBData->suid || pBData->uid) { - if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - - tBlockDataReset(pBData); - } - } - - if (!pBData->suid && !pBData->uid) { - ASSERT(pCommitter->skmTable.suid == id.suid); - ASSERT(pCommitter->skmTable.uid == id.uid); - TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; - code = tBlockDataInit(pBData, &tid, pCommitter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - } -#endif - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - SBlockData *pBData = &pCommitter->dWriter.bData; - TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { - TSDBROW row = tsdbRowFromBlockData(pBData, iRow); - -#if USE_STREAM_COMPRESSION - code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - } -#else - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &row, NULL, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } -#endif - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - int32_t lino = 0; - - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - - if (pRowInfo == NULL) goto _exit; - - if (pCommitter->toLastOnly) { - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == TSDBROW_ROW_FMT) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - pTSchema = pCommitter->skmRow.pTSchema; - } - -#if USE_STREAM_COMPRESSION - code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); -#else - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pRowInfo->row, pTSchema, id.uid); -#endif - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - -#if USE_STREAM_COMPRESSION - if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - } -#else - if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } -#endif - } - } else { - SBlockData *pBData = &pCommitter->dWriter.bData; - ASSERT(pBData->nRow == 0); - - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == TSDBROW_ROW_FMT) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - pTSchema = pCommitter->skmRow.pTSchema; - } - - code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - - if (pBData->nRow >= pCommitter->maxRow) { - code = - tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (pBData->nRow) { - if (pBData->nRow > pCommitter->minRow) { - code = - tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbAppendLastBlock(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - SRowInfo *pRowInfo; - TABLEID id = {0}; - while ((pRowInfo = tsdbGetCommitRow(pCommitter)) != NULL) { - ASSERT(pRowInfo->suid != id.suid || pRowInfo->uid != id.uid); - id.suid = pRowInfo->suid; - id.uid = pRowInfo->uid; - - code = tsdbMoveCommitData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - // start - tMapDataReset(&pCommitter->dWriter.mBlock); - - // impl - code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); - TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - - /* merge with data in .data file */ - code = tsdbMergeTableData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - /* handle remain table data */ - code = tsdbCommitTableData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - // end - if (pCommitter->dWriter.mBlock.nItem > 0) { - SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; - code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - } - - id.suid = INT64_MAX; - id.uid = INT64_MAX; - code = tsdbMoveCommitData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - -#if USE_STREAM_COMPRESSION - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); -#else - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); -#endif - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -int32_t tsdbFinishCommit(STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - SMemTable *pMemTable = pTsdb->imem; - - // lock - taosThreadMutexLock(&pTsdb->mutex); - - code = tsdbFSCommit(pTsdb); - if (code) { - taosThreadMutexUnlock(&pTsdb->mutex); - TSDB_CHECK_CODE(code, lino, _exit); - } - - pTsdb->imem = NULL; - - // unlock - taosThreadMutexUnlock(&pTsdb->mutex); - if (pMemTable) { - tsdbUnrefMemTable(pMemTable, NULL, true); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d, tsdb finish commit", TD_VID(pTsdb->pVnode)); - } - return code; -} - -int32_t tsdbRollbackCommit(STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - - code = tsdbFSRollback(pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d, tsdb rollback commit", TD_VID(pTsdb->pVnode)); - } - return code; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 732f46467e..a796f5121c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -702,7 +702,7 @@ _exit: } // EXPOSED APIS ==================================================================================== -int32_t tsdbFSCommit(STsdb *pTsdb) { +static int32_t tsdbFSCommit(STsdb *pTsdb) { int32_t code = 0; int32_t lino = 0; STsdbFS fs = {0}; @@ -738,7 +738,7 @@ _exit: return code; } -int32_t tsdbFSRollback(STsdb *pTsdb) { +static int32_t tsdbFSRollback(STsdb *pTsdb) { int32_t code = 0; int32_t lino = 0; @@ -833,312 +833,3 @@ int32_t tsdbFSClose(STsdb *pTsdb) { return code; } - -int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - - pFS->pDelFile = NULL; - if (pFS->aDFileSet) { - taosArrayClear(pFS->aDFileSet); - } else { - pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet)); - if (pFS->aDFileSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (pTsdb->fs.pDelFile) { - pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); - if (pFS->pDelFile == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - *pFS->pDelFile = *pTsdb->fs.pDelFile; - } - - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid}; - - // head - fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.pHeadF = *pSet->pHeadF; - - // data - fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.pDataF = *pSet->pDataF; - - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.pSmaF = *pSet->pSmaF; - - // stt - for (fSet.nSttF = 0; fSet.nSttF < pSet->nSttF; fSet.nSttF++) { - fSet.aSttF[fSet.nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (fSet.aSttF[fSet.nSttF] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.aSttF[fSet.nSttF] = *pSet->aSttF[fSet.nSttF]; - } - - if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) { - int32_t code = 0; - - if (pDelFile) { - if (pFS->pDelFile == NULL) { - pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); - if (pFS->pDelFile == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - *pFS->pDelFile = *pDelFile; - } else { - if (pFS->pDelFile) { - taosMemoryFree(pFS->pDelFile); - pFS->pDelFile = NULL; - } - } - -_exit: - return code; -} - -int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { - int32_t code = 0; - int32_t idx = taosArraySearchIdx(pFS->aDFileSet, pSet, tDFileSetCmprFn, TD_GE); - - if (idx < 0) { - idx = taosArrayGetSize(pFS->aDFileSet); - } else { - SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, idx); - int32_t c = tDFileSetCmprFn(pSet, pDFileSet); - if (c == 0) { - *pDFileSet->pHeadF = *pSet->pHeadF; - *pDFileSet->pDataF = *pSet->pDataF; - *pDFileSet->pSmaF = *pSet->pSmaF; - // stt - if (pSet->nSttF > pDFileSet->nSttF) { - ASSERT(pSet->nSttF == pDFileSet->nSttF + 1); - - pDFileSet->aSttF[pDFileSet->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pDFileSet->aSttF[pDFileSet->nSttF] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *pDFileSet->aSttF[pDFileSet->nSttF] = *pSet->aSttF[pSet->nSttF - 1]; - pDFileSet->nSttF++; - } else if (pSet->nSttF < pDFileSet->nSttF) { - ASSERT(pSet->nSttF == 1); - for (int32_t iStt = 1; iStt < pDFileSet->nSttF; iStt++) { - taosMemoryFree(pDFileSet->aSttF[iStt]); - } - - *pDFileSet->aSttF[0] = *pSet->aSttF[0]; - pDFileSet->nSttF = 1; - } else { - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - *pDFileSet->aSttF[iStt] = *pSet->aSttF[iStt]; - } - } - - pDFileSet->diskId = pSet->diskId; - goto _exit; - } - } - - ASSERT(pSet->nSttF == 1); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSttF = 1}; - - // head - fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.pHeadF = *pSet->pHeadF; - - // data - fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.pDataF = *pSet->pDataF; - - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.pSmaF = *pSet->pSmaF; - - // stt - fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (fSet.aSttF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.aSttF[0] = *pSet->aSttF[0]; - - if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFSNew) { - int32_t code = 0; - int32_t lino = 0; - char tfname[TSDB_FILENAME_LEN]; - - tsdbGetCurrentFName(pTsdb, NULL, tfname); - - // gnrt CURRENT.t - code = tsdbSaveFSToFile(pFSNew, tfname); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { - int32_t code = 0; - int32_t nRef; - - pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet)); - if (pFS->aDFileSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - pFS->pDelFile = pTsdb->fs.pDelFile; - if (pFS->pDelFile) { - nRef = atomic_fetch_add_32(&pFS->pDelFile->nRef, 1); - ASSERT(nRef > 0); - } - - SDFileSet fSet; - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - fSet = *pSet; - - nRef = atomic_fetch_add_32(&pSet->pHeadF->nRef, 1); - ASSERT(nRef > 0); - - nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1); - ASSERT(nRef > 0); - - nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); - ASSERT(nRef > 0); - - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - nRef = atomic_fetch_add_32(&pSet->aSttF[iStt]->nRef, 1); - ASSERT(nRef > 0); - } - - if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} - -void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { - int32_t nRef; - char fname[TSDB_FILENAME_LEN]; - - if (pFS->pDelFile) { - nRef = atomic_sub_fetch_32(&pFS->pDelFile->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbDelFileName(pTsdb, pFS->pDelFile, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pFS->pDelFile); - } - } - - for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet); - - // head - nRef = atomic_sub_fetch_32(&pSet->pHeadF->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->pHeadF); - } - - // data - nRef = atomic_sub_fetch_32(&pSet->pDataF->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->pDataF); - } - - // sma - nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->pSmaF); - } - - // stt - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - nRef = atomic_sub_fetch_32(&pSet->aSttF[iStt]->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->aSttF[iStt]); - /* code */ - } - } - } - - taosArrayDestroy(pFS->aDFileSet); -} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 5ad067c113..1f54f13592 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -22,8 +22,8 @@ #include "tsdbUtil2.h" #include "tsimplehash.h" -#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define getCurrentKeyInSttBlock(_r) ((_r)->currentKey) +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define getCurrentKeyInSttBlock(_r) ((_r)->currentKey) typedef struct { bool overlapWithNeighborBlock; @@ -41,7 +41,7 @@ static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, - SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); + SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo); @@ -67,7 +67,7 @@ static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); -static bool hasDataInSttBlock(STableBlockScanInfo *pInfo); +static bool hasDataInSttBlock(STableBlockScanInfo* pInfo); static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); static void resetTableListIndex(SReaderStatus* pStatus); @@ -1138,7 +1138,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo return false; } - int32_t step = asc ? 1 : -1; + int32_t step = asc ? 1 : -1; STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); @@ -1316,17 +1316,17 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; - int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); + int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); double el = (taosGetTimestampUs() - st) / 1000.0; updateComposedBlockInfo(pReader, el, pBlockScanInfo); tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 " - %" PRId64 ", uid:%" PRIu64 ", %s", - pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, - pBlockScanInfo->uid, pReader->idStr); + pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, + pReader->idStr); pReader->cost.buildmemBlock += el; return code; @@ -1390,13 +1390,9 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc } } -static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { - tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); -} +static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); } -static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { - tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); -} +static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); } static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, @@ -1535,8 +1531,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == k.ts) { @@ -1585,8 +1580,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == key) { @@ -1648,7 +1642,7 @@ static int32_t doMergeFileBlockAndLastBlock(SSttBlockReader* pSttBlockReader, ST TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); tsdbRowMergerAdd(pMerger, pRow1, NULL); doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); + pReader->idStr); code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { @@ -1671,7 +1665,7 @@ static int32_t doMergeFileBlockAndLastBlock(SSttBlockReader* pSttBlockReader, ST } doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); + pReader->idStr); // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { @@ -1740,7 +1734,7 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* // the following for key == tsLast // ASC: file block ------> stt block // DESC: stt block ------> file block - SRow* pTSRow = NULL; + SRow* pTSRow = NULL; if (ASCENDING_TRAVERSE(pReader->info.order)) { code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { @@ -1889,8 +1883,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == ik.ts) { @@ -1948,8 +1941,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == key) { @@ -2120,7 +2112,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan }; SSttDataInfoForTable info = {.pTimeWindowList = taosArrayInit(4, sizeof(STimeWindow))}; - int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); + int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); if (code != TSDB_CODE_SUCCESS) { return false; } @@ -2138,7 +2130,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pScanInfo->sttWindow.ekey = INT64_MIN; // calculate the time window for data in stt files - for(int32_t i = 0; i < taosArrayGetSize(info.pTimeWindowList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(info.pTimeWindowList); ++i) { STimeWindow* pWindow = taosArrayGet(info.pTimeWindowList, i); if (pScanInfo->sttWindow.skey > pWindow->skey) { pScanInfo->sttWindow.skey = pWindow->skey; @@ -2149,8 +2141,9 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan } } - pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList)? STT_FILE_HAS_DATA:STT_FILE_NO_DATA; - pScanInfo->sttKeyInfo.nextProcKey = ASCENDING_TRAVERSE(pReader->info.order)? pScanInfo->sttWindow.skey:pScanInfo->sttWindow.ekey; + pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA; + pScanInfo->sttKeyInfo.nextProcKey = + ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; hasData = true; } else { hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); @@ -2168,9 +2161,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan return hasData; } -static bool hasDataInSttBlock(STableBlockScanInfo *pInfo) { - return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; -} +static bool hasDataInSttBlock(STableBlockScanInfo* pInfo) { return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; } bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { @@ -2225,10 +2216,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } } -int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { - bool copied = false; - SRow* pTSRow = NULL; - int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); +int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader) { + bool copied = false; + SRow* pTSRow = NULL; + int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); SRowMerger* pMerger = &pReader->status.merger; TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); @@ -2559,16 +2551,16 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { pReader->status.bProcMemFirstFileset = false; } - int32_t fid = pReader->status.pCurrentFileset->fid; + int32_t fid = pReader->status.pCurrentFileset->fid; STimeWindow winFid = {0}; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); if (ASCENDING_TRAVERSE(pReader->info.order)) { pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < pReader->status.prevFilesetStartKey || - (winFid.skey-1) < pReader->status.memTableMinKey); + (winFid.skey - 1) < pReader->status.memTableMinKey); } else { - pReader->status.bProcMemPreFileset = !( pReader->status.memTableMaxKey < (winFid.ekey+1) || - pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < (winFid.ekey + 1) || + pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); } if (pReader->status.bProcMemPreFileset) { @@ -2802,7 +2794,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) { ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); - if(pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) { + if (pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) { return true; } else { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; @@ -2923,7 +2915,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_t endKey) { SReaderStatus* pStatus = &pReader->status; - tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid, pReader->idStr); + tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid, + pReader->idStr); while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { @@ -3847,7 +3840,6 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e pBlockScanInfo->lastProcKey = row.pBlockData->aTSKEY[row.iRow]; } - // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { break; @@ -3945,7 +3937,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; -// } else if (READER_EXEC_DATA == pReader->info.readMode) { + // } else if (READER_EXEC_DATA == pReader->info.readMode) { // DO NOTHING } else { code = initForFirstBlockInFile(pReader, pBlockIter); @@ -4092,7 +4084,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } pReader->flag = READER_STATUS_SUSPEND; - pReader->info.execMode = pCond->notLoadData? READER_EXEC_ROWS : READER_EXEC_DATA; + pReader->info.execMode = pCond->notLoadData ? READER_EXEC_ROWS : READER_EXEC_DATA; pReader->pIgnoreTables = pIgnoreTables; tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64 @@ -4157,7 +4149,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { } SReadCostSummary* pCost = &pReader->cost; - SFilesetIter* pFilesetIter = &pReader->status.fileIter; + SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pSttBlockReader != NULL) { SSttBlockReader* pLReader = pFilesetIter->pSttBlockReader; tMergeTreeClose(&pLReader->mergeTree); @@ -4204,7 +4196,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableBlockScanInfo* pBlockScanInfo = NULL; - pReader->status.suspendInvoked = true; // record the suspend status + pReader->status.suspendInvoked = true; // record the suspend status if (pStatus->loadFromFile) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); @@ -4227,7 +4219,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t step = ASCENDING_TRAVERSE(pReader->info.order)? 1:-1; + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { @@ -4248,7 +4240,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { pReader->flag = READER_STATUS_SUSPEND; #if SUSPEND_RESUME_TEST - tsem_post(&pReader->resumeAfterSuspend); + tsem_post(&pReader->resumeAfterSuspend); #endif tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, @@ -4331,13 +4323,13 @@ _err: } static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SReaderStatus* pStatus = &pReader->status; SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; - int32_t fid = pReader->status.pCurrentFileset->fid; - STimeWindow win = {0}; + int32_t fid = pReader->status.pCurrentFileset->fid; + STimeWindow win = {0}; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey; @@ -4359,8 +4351,8 @@ static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) { static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; - int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; if (pStatus->loadFromFile) { if (pStatus->bProcMemPreFileset) { @@ -4375,12 +4367,12 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { return code; } - tsdbTrace("block from file rows: %"PRId64", will process pre-file set buffer: %d. %s", - pBlock->info.rows, pStatus->bProcMemFirstFileset, pReader->idStr); + tsdbTrace("block from file rows: %" PRId64 ", will process pre-file set buffer: %d. %s", pBlock->info.rows, + pStatus->bProcMemFirstFileset, pReader->idStr); if (pStatus->bProcMemPreFileset) { if (pBlock->info.rows > 0) { if (pReader->notifyFn) { - int32_t fid = pReader->status.pCurrentFileset->fid; + int32_t fid = pReader->status.pCurrentFileset->fid; STsdReaderNotifyInfo info = {0}; info.duration.filesetId = fid; pReader->notifyFn(TSD_READER_NOTIFY_NEXT_DURATION_BLOCK, &info, pReader->notifyParam); @@ -4404,8 +4396,8 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; - int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; if (pStatus->loadFromFile) { code = buildBlockFromFiles(pReader); @@ -4940,7 +4932,7 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t int64_t minKey = INT64_MAX; void* pHashIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); - while (pHashIter!= NULL) { + while (pHashIter != NULL) { STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pHashIter; STbData* d = NULL; @@ -5144,7 +5136,6 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } - tsdbFSUnref(pTsdb, &pSnap->fs); if (pSnap->pNode) taosMemoryFree(pSnap->pNode); if (pSnap->pINode) taosMemoryFree(pSnap->pINode); @@ -5165,9 +5156,7 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } -void tsdbSetFilesetDelimited(STsdbReader* pReader) { - pReader->bFilesetDelimited = true; -} +void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; } void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { pReader->notifyFn = notifyFn; From f14c3b93c138420dea53a21297446417879d30a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 18:22:36 +0800 Subject: [PATCH 088/236] refactor: do some internal refactor. --- source/libs/stream/src/streamTaskSm.c | 81 ++++++++++++++++----------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 68ae1cce36..51e6ad23fe 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -160,6 +160,45 @@ static STaskStateTrans* streamTaskFindTransform(ETaskStatus state, const EStream return NULL; } +static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, SStreamTask* pTask) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t el = (taosGetTimestampMs() - pSM->startTs); + stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, + pEventName, el, pSM->prev.state.name, pSM->current.name); + + SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + + // OK, let's handle the attached event, since the task has reached the required status now + if (pSM->current.state == pEvtInfo->status) { + stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr, + GET_EVT_NAME(pEvtInfo->event), pSM->current.name); + + // remove it + taosArrayPop(pSM->pWaitingEventList); + + STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event); + ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL); + + pSM->pActiveTrans = pNextTrans; + pSM->startTs = taosGetTimestampMs(); + taosThreadMutexUnlock(&pTask->lock); + + code = pNextTrans->pAction(pSM->pTask); + if (pNextTrans->autoInvokeEndFn) { + return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event); + } else { + return code; + } + } else { + taosThreadMutexUnlock(&pTask->lock); + stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, + pSM->current.name, GET_EVT_NAME(pEvtInfo->event), + StreamTaskStatusList[pEvtInfo->status].name); + } + + return code; +} + void streamTaskRestoreStatus(SStreamTask* pTask) { SStreamTaskSM* pSM = pTask->status.pSM; @@ -175,7 +214,13 @@ void streamTaskRestoreStatus(SStreamTask* pTask) { pSM->prev.evt = 0; pSM->startTs = taosGetTimestampMs(); - stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + + if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { + stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); + } else { + stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + } taosThreadMutexUnlock(&pTask->lock); } @@ -343,39 +388,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even pTrans->pSuccAction(pTask); if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { - int64_t el = (taosGetTimestampMs() - pSM->startTs); - stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, - GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name); - - SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); - - // OK, let's handle the attached event, since the task has reached the required status now - if (pSM->current.state == pEvtInfo->status) { - stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr, - GET_EVT_NAME(pEvtInfo->event), pSM->current.name); - - // remove it - taosArrayPop(pSM->pWaitingEventList); - - STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event); - ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL); - - pSM->pActiveTrans = pNextTrans; - pSM->startTs = taosGetTimestampMs(); - taosThreadMutexUnlock(&pTask->lock); - - int32_t code = pNextTrans->pAction(pSM->pTask); - if (pNextTrans->autoInvokeEndFn) { - return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event); - } else { - return code; - } - } else { - taosThreadMutexUnlock(&pTask->lock); - stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, - pSM->current.name, GET_EVT_NAME(pEvtInfo->event), - StreamTaskStatusList[pEvtInfo->status].name); - } + doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); } else { taosThreadMutexUnlock(&pTask->lock); From ac3c76f60895023168689eb9adfca9eb02f402c2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 25 Dec 2023 19:38:20 +0800 Subject: [PATCH 089/236] init dummy function --- .../executor/src/streamtimewindowoperator.c | 4 +++ tests/script/tsim/stream/session1.sim | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 7b68b65c17..c9490e2c55 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1623,6 +1623,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num pDummy[i].functionId = pCtx[i].functionId; pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc; pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc; + pDummy[i].fpSet.init = pCtx[i].fpSet.init; } } @@ -2774,6 +2775,9 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); + if (!IS_VALID_SESSION_WIN(winInfo)) { + continue; + } compactSessionSemiWindow(pOperator, &winInfo); saveSessionOutputBuf(pAggSup, &winInfo); } diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index cf42159d84..aae17053b2 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -327,4 +327,40 @@ if $rows != 1 then goto loop17 endi +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams4 trigger at_once ignore update 0 ignore expired 0 into streamt4 as select _wstart, count(*) c1, count(a) c2 from st session(ts, 2s) ; + +sql insert into t1 values(1648791255100,1,2,3); +sql insert into t1 values(1648791255300,1,2,3); + +sleep 1000 + +sql insert into t1 values(1648791253000,1,2,3) (1648791254000,1,2,3); + +$loop_count = 0 +loop18: +sleep 1000 +sql select * from streamt4; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop18 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop18 +endi + +print =====over + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5f0b0b4338e69437e34bbe82ab6d524e8e3392fd Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Dec 2023 19:56:56 +0800 Subject: [PATCH 090/236] case: mlevel_basic.py passed --- .../enterprise/multi-level/mlevel_basic.py | 10 ++++++---- tests/army/frame/caseBase.py | 8 ++++---- tests/army/frame/eos.py | 2 +- tests/army/frame/epath.py | 18 +++++++++--------- tests/army/frame/etool.py | 18 ++++++++++-------- tests/army/frame/sql.py | 2 +- 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index a12b1b59eb..70bfb9b66e 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -16,12 +16,13 @@ import time import taos import frame +import frame.etool from frame.log import * from frame.cases import * from frame.sql import * from frame.caseBase import * - +from frame import * class TDTestCase(TBase): @@ -30,9 +31,10 @@ class TDTestCase(TBase): def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run - json = frame.etool.curFile(__file__, "mlevel_basic.json") - frame.etool.runBenchmark(json=json) + json = etool.curFile(__file__, "mlevel_basic.json") + etool.runBenchmark(json=json) + tdSql.execute(f"use {self.db}") # set insert data information self.childtable_count = 2 self.insert_rows = 1000000 @@ -76,4 +78,4 @@ class TDTestCase(TBase): tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 7b371cd822..11c44b28e9 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -16,11 +16,11 @@ import os import time import datetime -from log import * -from sql import * +from frame.log import * +from frame.sql import * # test case base -class DbBase: +class TBase: # # frame call @@ -85,7 +85,7 @@ class DbBase: tdSql.checkAgg(sql, self.childtable_count) # check step - sql = f"select count(*) from (select diff(ts) as dif from {self.stb} group by tbname) where dif != {self.timestamp_step}" + sql = f"select count(*) from (select diff(ts) as dif from {self.stb} partition by tbname) where dif != {self.timestamp_step}" tdSql.checkAgg(sql, 0) # save agg result diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 2d8559f3eb..ab4d2c1bd0 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -22,7 +22,7 @@ import datetime # if windows platform return True def isWin(): - return sys.platform.system().lower() == 'windows' + return sys.platform.lower() == 'windows' # wait util execute file finished def exe(file): diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py index 644e448fb7..edff9c78a4 100644 --- a/tests/army/frame/epath.py +++ b/tests/army/frame/epath.py @@ -19,13 +19,13 @@ import os from frame.log import * # build/bin path -binPath = "" +binDir = "" -def binPath(self): - global binPath +def binPath(): + global binDir - if binPath != "": - return binPath + if binDir != "": + return binDir selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -46,8 +46,8 @@ def binPath(self): else: tdLog.info(f"taosd found in {buildPath}") # return - binPath = buildPath + "/build/bin/" - return binPath + binDir = buildPath + "/build/bin/" + return binDir -def binFile(self, file): - return binPath() + file \ No newline at end of file +def binFile(filename): + return binPath() + filename diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index 5ba29ee1c3..f24871b3dd 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -19,22 +19,24 @@ import sys import os import time import datetime -import epath -import eos +import frame.epath +import frame.eos # run taosBenchmark with command or json file mode def runBenchmark(command = "", json = "") : # get taosBenchmark path - bmFile = epath.binFile("taosBenchmark") - if eos.isWin(): + bmFile = frame.epath.binFile("taosBenchmark") + if frame.eos.isWin(): bmFile += ".exe" # run if command != "": - eos.exe(bmFile + " " + command) + frame.eos.exe(bmFile + " " + command) if json != "": - eos.exe(bmFile + " -f " + json) + cmd = f"{bmFile} -f {json}" + print(cmd) + frame.eos.exe(cmd) # get current directory file name -def curFile(fullPath, file): - return os.path.dirname(fullPath) + "/" + file +def curFile(fullPath, filename): + return os.path.dirname(fullPath) + "/" + filename diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index b3ee1ed5f8..2e14f0c2f0 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -538,7 +538,7 @@ class TDSql: # check like select count(*) ... sql def checkAgg(self, sql, expectCnt): self.query(sql) - self.checkRowCol(0, 0, expectCnt) + self.checkData(0, 0, expectCnt) # get first value def getFirstValue(self, sql) : From 7b967ffdc63c397e684b820f7e30dc1671a5ec86 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 25 Dec 2023 20:22:18 +0800 Subject: [PATCH 091/236] enh: retry on fsm commit when rpc out of memory in syncFsmExecute --- source/libs/sync/src/syncPipeline.c | 37 ++++++++++++++++++----------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f2386797c1..28ee5ba841 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -558,22 +558,31 @@ int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTe pEntry->term); } - SRpcMsg rpcMsg = {.code = applyCode}; - syncEntry2OriginalRpc(pEntry, &rpcMsg); + int32_t code = 0; + bool retry = false; + do { + SRpcMsg rpcMsg = {.code = applyCode}; + syncEntry2OriginalRpc(pEntry, &rpcMsg); - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pEntry->index; - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); - cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = applyCode; - cbMeta.state = role; - cbMeta.seqNum = pEntry->seqNum; - cbMeta.term = pEntry->term; - cbMeta.currentTerm = term; - cbMeta.flag = -1; + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = applyCode; + cbMeta.state = role; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.term = pEntry->term; + cbMeta.currentTerm = term; + cbMeta.flag = -1; - (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); - int32_t code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); + (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); + code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); + retry = (code != 0) && (terrno == TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE); + if (retry) { + taosMsleep(10); + sError("vgId:%d, retry on fsm commit since %s. index:%" PRId64, pNode->vgId, terrstr(), pEntry->index); + } + } while (retry); return code; } From 9071ff39736daa9ec3d61c76cddbffa189955246 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Dec 2023 21:20:59 +0800 Subject: [PATCH 092/236] fix: add compactDb flushDb --- .../enterprise/multi-level/mlevel_basic.json | 4 ++-- .../enterprise/multi-level/mlevel_basic.py | 19 +++++++------------ tests/army/frame/caseBase.py | 2 ++ tests/army/frame/eos.py | 3 ++- tests/army/frame/etool.py | 6 +++++- tests/army/frame/server/dnodes.py | 8 +++++++- tests/army/test.py | 2 ++ 7 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/army/enterprise/multi-level/mlevel_basic.json b/tests/army/enterprise/multi-level/mlevel_basic.json index 7df5b1caa8..1c2b9274d2 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.json +++ b/tests/army/enterprise/multi-level/mlevel_basic.json @@ -24,12 +24,12 @@ { "name": "stb", "child_table_exists": "no", - "childtable_count": 2, + "childtable_count": 4, "insert_rows": 1000000, "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-10d", + "start_timestamp":"now-12d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index 70bfb9b66e..3bec2bfb72 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -36,23 +36,15 @@ class TDTestCase(TBase): tdSql.execute(f"use {self.db}") # set insert data information - self.childtable_count = 2 + self.childtable_count = 4 self.insert_rows = 1000000 self.timestamp_step = 1000 def doAction(self): - tdLog.info(f"trim database.") + tdLog.info(f"do action.") + self.flushDb() self.trimDb() - - def saveData(self): - tdLog.info(f"check correct.") - - def checkSaveCorrent(self): - tdLog.info(f"check correct.") - - def checkCorrect(self): - tdLog.info(f"check correct.") - + self.compactDb() # run def run(self): @@ -73,6 +65,9 @@ class TDTestCase(TBase): # check save agg result correct self.checkAggCorrect() + # check insert correct again + self.checkInsertCorrect() + tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 11c44b28e9..441196f050 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -69,6 +69,8 @@ class TBase: def compactDb(self): tdSql.execute(f"compact database {self.db}") + def flushDb(self): + tdSql.execute(f"flush database {self.db}") # # check db correct diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index ab4d2c1bd0..dcb63ff3aa 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -19,10 +19,11 @@ import sys import os import time import datetime +import platform # if windows platform return True def isWin(): - return sys.platform.lower() == 'windows' + return platform.system().lower() == 'windows' # wait util execute file finished def exe(file): diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index f24871b3dd..35c390dc1a 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -21,6 +21,7 @@ import time import datetime import frame.epath import frame.eos +from frame.log import * # run taosBenchmark with command or json file mode def runBenchmark(command = "", json = "") : @@ -35,7 +36,10 @@ def runBenchmark(command = "", json = "") : if json != "": cmd = f"{bmFile} -f {json}" print(cmd) - frame.eos.exe(cmd) + status = frame.eos.exe(cmd) + if status !=0: + tdLog.exit(f"run failed {cmd} status={status}") + # get current directory file name def curFile(fullPath, filename): diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 1e51fbc3be..0d40b665dd 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -911,4 +911,10 @@ class TDDnodes: def getAsan(self): return self.asan -tdDnodes = TDDnodes() \ No newline at end of file + def setLevelDisk(self, level, disk): + for i in range(len(self.dnodes)): + self.dnodes[i].level = int(level) + self.dnodes[i].disk = int(disk) + + +tdDnodes = TDDnodes() diff --git a/tests/army/test.py b/tests/army/test.py index 83d065b24b..a3e28b772d 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -373,6 +373,7 @@ if __name__ == "__main__": tAdapter.stop(force_kill=True) if dnodeNums == 1 : + tdDnodes.setLevelDisk(level, disk) tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) @@ -542,6 +543,7 @@ if __name__ == "__main__": if dnodeNums == 1 : # dnode is one + tdDnodes.setLevelDisk(level, disk) tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) From 62cfcefbb0e26c29c06ff1d22f414a71018274c8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 00:34:04 +0800 Subject: [PATCH 093/236] refactor: remove stream-scan-history event for stream task. --- include/libs/stream/tstream.h | 19 +++-- source/dnode/mnode/impl/src/mndStream.c | 88 +++++++++++----------- source/dnode/vnode/src/tq/tq.c | 27 +++---- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/libs/parser/src/parTranslater.c | 2 +- source/libs/stream/src/streamExec.c | 2 +- source/libs/stream/src/streamMeta.c | 2 +- source/libs/stream/src/streamStart.c | 20 +++-- source/libs/stream/src/streamTaskSm.c | 32 ++++---- 9 files changed, 99 insertions(+), 95 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2d2db5c1dc..0c09e0e1ea 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -81,7 +81,7 @@ typedef enum ETaskStatus { TASK_STATUS__HALT, // pause, but not be manipulated by user command TASK_STATUS__PAUSE, // pause TASK_STATUS__CK, // stream task is in checkpoint status, no data are allowed to put into inputQ anymore - TASK_STATUS__STREAM_SCAN_HISTORY, +// TASK_STATUS__STREAM_SCAN_HISTORY, } ETaskStatus; enum { @@ -138,15 +138,14 @@ enum { typedef enum EStreamTaskEvent { TASK_EVENT_INIT = 0x1, TASK_EVENT_INIT_SCANHIST = 0x2, - TASK_EVENT_INIT_STREAM_SCANHIST = 0x3, - TASK_EVENT_SCANHIST_DONE = 0x4, - TASK_EVENT_STOP = 0x5, - TASK_EVENT_GEN_CHECKPOINT = 0x6, - TASK_EVENT_CHECKPOINT_DONE = 0x7, - TASK_EVENT_PAUSE = 0x8, - TASK_EVENT_RESUME = 0x9, - TASK_EVENT_HALT = 0xA, - TASK_EVENT_DROPPING = 0xB, + TASK_EVENT_SCANHIST_DONE = 0x3, + TASK_EVENT_STOP = 0x4, + TASK_EVENT_GEN_CHECKPOINT = 0x5, + TASK_EVENT_CHECKPOINT_DONE = 0x6, + TASK_EVENT_PAUSE = 0x7, + TASK_EVENT_RESUME = 0x8, + TASK_EVENT_HALT = 0x9, + TASK_EVENT_DROPPING = 0xA, } EStreamTaskEvent; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 25f51b85df..2464a9207e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2886,38 +2886,38 @@ static SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { return NULL; } -static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStreamExecInfo *pExecNode) { - if (pTaskEntry->status == TASK_STATUS__STREAM_SCAN_HISTORY && pTaskEntry->statusLastDuration >= 10) { - if (!pTaskEntry->inputQChanging && pTaskEntry->inputQUnchangeCounter > 10) { - int32_t numOfReady = 0; - int32_t numOfTotal = 0; - for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { - STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); - if (pTaskEntry->id.streamId == pId->streamId) { - numOfTotal++; - - if (pTaskEntry->id.taskId != pId->taskId) { - STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); - if (pEntry->status == TASK_STATUS__READY) { - numOfReady++; - } - } - } - } - - if (numOfReady > 0) { - mDebug("stream:0x%" PRIx64 - " %d tasks are ready, %d tasks in stream-scan-history for more than 50s, drop related fill-history task", - pTaskEntry->id.streamId, numOfReady, numOfTotal - numOfReady); - return true; - } else { - return false; - } - } - } - - return false; -} +//static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStreamExecInfo *pExecNode) { +// if (pTaskEntry->status == TASK_STATUS__STREAM_SCAN_HISTORY && pTaskEntry->statusLastDuration >= 10) { +// if (!pTaskEntry->inputQChanging && pTaskEntry->inputQUnchangeCounter > 10) { +// int32_t numOfReady = 0; +// int32_t numOfTotal = 0; +// for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { +// STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); +// if (pTaskEntry->id.streamId == pId->streamId) { +// numOfTotal++; +// +// if (pTaskEntry->id.taskId != pId->taskId) { +// STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); +// if (pEntry->status == TASK_STATUS__READY) { +// numOfReady++; +// } +// } +// } +// } +// +// if (numOfReady > 0) { +// mDebug("stream:0x%" PRIx64 +// " %d tasks are ready, %d tasks in stream-scan-history for more than 50s, drop related fill-history task", +// pTaskEntry->id.streamId, numOfReady, numOfTotal - numOfReady); +// return true; +// } else { +// return false; +// } +// } +// } +// +// return false; +//} // currently only handle the sink task // 1. sink task, drop related fill-history task msg is missing @@ -3091,18 +3091,18 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->status != TASK_STATUS__READY) { mDebug("received s-task:0x%" PRIx64 " not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status)); - if (p->status == TASK_STATUS__STREAM_SCAN_HISTORY) { - bool drop = needDropRelatedFillhistoryTask(pTaskEntry, &execInfo); - if (drop) { - SStreamObj *pStreamObj = mndGetStreamObj(pMnode, pTaskEntry->id.streamId); - if (pStreamObj == NULL) { - mError("failed to acquire the streamObj:0x%" PRIx64 " it may have been dropped", pStreamObj->uid); - } else { - mndDropRelatedFillhistoryTask(pMnode, pTaskEntry, pStreamObj); - mndReleaseStream(pMnode, pStreamObj); - } - } - } +// if (p->status == TASK_STATUS__STREAM_SCAN_HISTORY) { +// bool drop = needDropRelatedFillhistoryTask(pTaskEntry, &execInfo); +// if (drop) { +// SStreamObj *pStreamObj = mndGetStreamObj(pMnode, pTaskEntry->id.streamId); +// if (pStreamObj == NULL) { +// mError("failed to acquire the streamObj:0x%" PRIx64 " it may have been dropped", pStreamObj->uid); +// } else { +// mndDropRelatedFillhistoryTask(pMnode, pTaskEntry, pStreamObj); +// mndReleaseStream(pMnode, pStreamObj); +// } +// } +// } } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c8cc4f9d54..a8b5842b93 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1037,15 +1037,16 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pStreamTask); } else { - STimeWindow* pWindow = &pTask->dataRange.window; - ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) || streamTaskShouldStop(pTask)); - - // Not update the fill-history time window until the state transfer is completed. - tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64 - ", window:%" PRId64 " - %" PRId64, - id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); - - code = streamTaskScanHistoryDataComplete(pTask); + ASSERT(0); +// STimeWindow* pWindow = &pTask->dataRange.window; +// ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) || streamTaskShouldStop(pTask)); +// +// // Not update the fill-history time window until the state transfer is completed. +// tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64 +// ", window:%" PRId64 " - %" PRId64, +// id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); +// +// code = streamTaskScanHistoryDataComplete(pTask); } atomic_store_32(&pTask->status.inScanHistorySentinel, 0); @@ -1170,7 +1171,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, } else if (status == TASK_STATUS__UNINIT) { // todo: fill-history task init ? if (pTask->info.fillHistory == 0) { - EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; streamTaskHandleEvent(pTask->status.pSM, event); } } @@ -1363,9 +1364,9 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { taosThreadMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask, NULL); - if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); - } +// if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { +// streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); +// } SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index f576345f64..2991e3cef5 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -545,7 +545,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = /*(HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; streamTaskHandleEvent(p->status.pSM, event); } else if (!restored) { tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..6f3915c4d7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7989,7 +7989,7 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void } if (TSDB_CODE_SUCCESS == code) { if (interval.interval > 0) { - pStmt->pReq->lastTs = taosTimeTruncate(lastTs, &interval); + pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, interval.precision); } else { pStmt->pReq->lastTs = lastTs; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 25f32195be..7d2a572cda 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -646,7 +646,7 @@ bool streamTaskIsIdle(const SStreamTask* pTask) { bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { ETaskStatus st = streamTaskGetStatus(pTask, NULL); - return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__STREAM_SCAN_HISTORY || + return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY/* || st == TASK_STATUS__STREAM_SCAN_HISTORY*/ || st == TASK_STATUS__CK); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d09d370a36..f2dcc3bcc6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1484,7 +1484,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { continue; } - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = /*(HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); if (ret != TSDB_CODE_SUCCESS) { stError("vgId:%d failed to handle event:%d", pMeta->vgId, event); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index ea5e2edc09..70f83e27a5 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -52,7 +52,7 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); ETaskStatus status = streamTaskGetStatus(pTask, &p); - if ((status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY) && + if ((status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->upstreamInfo.pList); stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", @@ -158,7 +158,7 @@ int32_t streamTaskStartScanHistory(SStreamTask* pTask) { ETaskStatus status = streamTaskGetStatus(pTask, NULL); ASSERT(pTask->status.downstreamReady == 1 && - ((status == TASK_STATUS__SCAN_HISTORY) || (status == TASK_STATUS__STREAM_SCAN_HISTORY))); + ((status == TASK_STATUS__SCAN_HISTORY)/* || (status == TASK_STATUS__STREAM_SCAN_HISTORY)*/)); if (level == TASK_LEVEL__SOURCE) { return doStartScanHistoryTask(pTask); @@ -374,10 +374,14 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { char* p = NULL; ETaskStatus status = streamTaskGetStatus(pTask, &p); - ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY); + ASSERT(status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/); - stDebug("s-task:%s enter into scan-history data stage, status:%s", id, p); - streamTaskStartScanHistory(pTask); + if (pTask->info.fillHistory == 1) { + stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", id, p); + streamTaskStartScanHistory(pTask); + } else { + stDebug("s-task:%s scan wal data, status:%s", id, p); + } // NOTE: there will be an deadlock if launch fill history here. // // start the related fill-history task, when current task is ready @@ -391,7 +395,7 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { void doProcessDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event; if (pTask->info.fillHistory == 0) { - event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; } else { event = TASK_EVENT_INIT_SCANHIST; } @@ -631,7 +635,7 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory char* p = NULL; ETaskStatus status = streamTaskGetStatus(pTask, &p); - if (status != TASK_STATUS__SCAN_HISTORY && status != TASK_STATUS__STREAM_SCAN_HISTORY) { + if (status != TASK_STATUS__SCAN_HISTORY /*&& status != TASK_STATUS__STREAM_SCAN_HISTORY*/) { stError("s-task:%s not in scan-history status, status:%s return upstream:0x%x scan-history finish directly", id, p, pReq->upstreamTaskId); @@ -693,7 +697,7 @@ int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) { return TSDB_CODE_INVALID_MSG; } - ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY); + ASSERT(status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/); SStreamMeta* pMeta = pTask->pMeta; // execute in the scan history complete call back msg, ready to process data from inputQ diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 51e6ad23fe..4bd6483f7f 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -31,14 +31,14 @@ SStreamTaskState StreamTaskStatusList[9] = { {.state = TASK_STATUS__HALT, .name = "halt"}, {.state = TASK_STATUS__PAUSE, .name = "paused"}, {.state = TASK_STATUS__CK, .name = "checkpoint"}, - {.state = TASK_STATUS__STREAM_SCAN_HISTORY, .name = "stream-scan-history"}, +// {.state = TASK_STATUS__STREAM_SCAN_HISTORY, .name = "stream-scan-history"}, }; SStreamEventInfo StreamTaskEventList[12] = { {.event = 0, .name = ""}, // dummy event, place holder {.event = TASK_EVENT_INIT, .name = "initialize"}, {.event = TASK_EVENT_INIT_SCANHIST, .name = "scan-history-init"}, - {.event = TASK_EVENT_INIT_STREAM_SCANHIST, .name = "stream-scan-history-init"}, +// {.event = TASK_EVENT_INIT_STREAM_SCANHIST, .name = "stream-scan-history-init"}, {.event = TASK_EVENT_SCANHIST_DONE, .name = "scan-history-completed"}, {.event = TASK_EVENT_STOP, .name = "stopping"}, {.event = TASK_EVENT_GEN_CHECKPOINT, .name = "checkpoint"}, @@ -110,12 +110,12 @@ int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) { // todo check rsp code for handle Event:TASK_EVENT_SCANHIST_DONE static bool isInvalidStateTransfer(ETaskStatus state, const EStreamTaskEvent event) { - if (event == TASK_EVENT_INIT_STREAM_SCANHIST || event == TASK_EVENT_INIT || event == TASK_EVENT_INIT_SCANHIST) { + if (/*event == TASK_EVENT_INIT_STREAM_SCANHIST || */event == TASK_EVENT_INIT || event == TASK_EVENT_INIT_SCANHIST) { return (state != TASK_STATUS__UNINIT); } if (event == TASK_EVENT_SCANHIST_DONE) { - return (state != TASK_STATUS__SCAN_HISTORY && state != TASK_STATUS__STREAM_SCAN_HISTORY); + return (state != TASK_STATUS__SCAN_HISTORY/* && state != TASK_STATUS__STREAM_SCAN_HISTORY*/); } if (event == TASK_EVENT_GEN_CHECKPOINT) { @@ -482,14 +482,14 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); +// taosArrayPush(streamTaskSMTrans, &trans); // scan-history related event trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); +// taosArrayPush(streamTaskSMTrans, &trans); // halt stream task, from other task status trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); @@ -499,8 +499,8 @@ void doInitStateTransferTable(void) { SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); +// taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); @@ -519,8 +519,8 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); +// taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); @@ -554,8 +554,8 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); +// taosArrayPush(streamTaskSMTrans, &trans); // dropping related event trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); @@ -574,7 +574,7 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); +// taosArrayPush(streamTaskSMTrans, &trans); } //clang-format on \ No newline at end of file From 2c9fa56e9c3ea6a0bbca237800da86995aa98cee Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 26 Dec 2023 10:24:50 +0800 Subject: [PATCH 094/236] calcNeedCountEmpty --- source/libs/planner/src/planPhysiCreater.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0db0fd77fb..444bda84cd 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,14 +587,12 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - // refuse interval +static bool calcNeedCountEmpty(SScanLogicNode* pScanLogicNode) { if (pScanLogicNode->interval > 0) { return false; } - // limit: root node is select - SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) && pScanLogicNode->isCountByTag) { + + if (pScanLogicNode->isCountByTag) { return true; } @@ -637,7 +635,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; - pTableScan->needCountEmptyTable = calcNeedCountEmpty(pCxt, pScanLogicNode); + pTableScan->needCountEmptyTable = calcNeedCountEmpty(pScanLogicNode); int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { From 0e6ea7af712f4c918df6b1a323b825ba1169f494 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 26 Dec 2023 10:54:40 +0800 Subject: [PATCH 095/236] enh: check snapshot begin index for unexpected change in syncNodeOnSnapshotBegin --- source/libs/sync/src/syncSnapshot.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 2277b70c8f..10a8734617 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -806,6 +806,13 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p goto _SEND_REPLY; } + SyncIndex beginIndex = syncNodeGetSnapBeginIndex(pSyncNode); + if (pReceiver->snapshotParam.start != beginIndex) { + sRError(pReceiver, "snapshot begin index is changed unexpectedly. sver:%" PRId64 ", beginIndex:%" PRId64, + pReceiver->snapshotParam.start, beginIndex); + goto _SEND_REPLY; + } + code = 0; _SEND_REPLY: if (code != 0 && terrno != 0) { From b74e98b25c11d40a290d03988dd848d6a75319c2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 14:17:46 +0800 Subject: [PATCH 096/236] fix(stream): set the correct split timestamp for session/state window. --- source/dnode/mnode/impl/src/mndScheduler.c | 6 +++--- source/libs/parser/src/parTranslater.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 3ef4c9a4d2..39121cecd9 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -251,7 +251,7 @@ int32_t doAddSinkTask(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, in static int32_t addSourceTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SArray* pSinkTaskList, SStreamObj* pStream, SSubplan* plan, uint64_t uid, SEpSet* pEpset, bool fillHistory, - bool hasExtraSink, int64_t firstWindowSkey, bool hasFillHistory) { + bool hasExtraSink, int64_t nextWindowSkey, bool hasFillHistory) { SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, pStream->conf.triggerParam, pTaskList, hasFillHistory); if (pTask == NULL) { @@ -262,7 +262,7 @@ static int32_t addSourceTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, STimeWindow* pWindow = &pTask->dataRange.window; pWindow->skey = INT64_MIN; - pWindow->ekey = firstWindowSkey - 1; + pWindow->ekey = nextWindowSkey - 1; mDebug("add source task 0x%x window:%" PRId64 " - %" PRId64, pTask->id.taskId, pWindow->skey, pWindow->ekey); // sink or dispatch @@ -382,7 +382,7 @@ static int32_t doAddSourceTask(SArray* pTaskList, bool isFillhistory, int64_t ui epsetAssign(&(pTask)->info.mnodeEpset, pEpset); - // todo set the correct ts, which should be last key of queried table. + // set the correct ts, which is the last key of queried table. STimeWindow* pWindow = &pTask->dataRange.window; pWindow->skey = INT64_MIN; pWindow->ekey = nextWindowSkey - 1; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6f3915c4d7..cf6db9f549 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7991,7 +7991,7 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void if (interval.interval > 0) { pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, interval.precision); } else { - pStmt->pReq->lastTs = lastTs; + pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window } code = buildCmdMsg(&cxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, pStmt->pReq); } From 2cd61804bcfd3b70b10d85464f6334a557c97e26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 14:40:15 +0800 Subject: [PATCH 097/236] test:update the test case. --- tests/system-test/8-stream/stream_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index e838950bb8..3ebc255114 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -89,7 +89,7 @@ class TDTestCase: sql = "select count(*) from sta" # loop wait max 60s to check count is ok tdLog.info("loop wait result ...") - tdSql.checkDataLoop(0, 0, 99999, sql, loopCount=120, waitTime=0.5) + tdSql.checkDataLoop(0, 0, 100000, sql, loopCount=120, waitTime=0.5) time.sleep(5) From 1aae0e52e38bb5b3c3e6950ee4926a71cda82ff4 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 26 Dec 2023 14:48:05 +0800 Subject: [PATCH 098/236] fix: like wildcard match --- source/util/src/tcompare.c | 13 ++- source/util/test/utilTests.cpp | 56 ++++++++++- tests/parallel_test/cases.task | 4 + tests/system-test/2-query/like.py | 148 ++++++++++++++++++++++++++++++ tests/system-test/win-test-file | 4 + 5 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 tests/system-test/2-query/like.py diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index b6b71855f3..2baa21f5d9 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1070,6 +1070,11 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ } + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + c = pattern[i]; + i++; + } + char rejectList[2] = {toupper(c), tolower(c)}; str += nMatchChar; @@ -1139,6 +1144,11 @@ int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, return TSDB_PATTERN_MATCH; } + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + c = pattern[i]; + i++; + } + TdUcs4 rejectList[2] = {towupper(c), towlower(c)}; str += nMatchChar; @@ -1166,7 +1176,8 @@ int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, c1 = str[j++]; nMatchChar++; - if (c == L'\\' && pattern[i] == L'_' && c1 == L'_') { + if (c == '\\' && pattern[i] == c1 && + (c1 == '_' || c1 == '%')) { i++; continue; } diff --git a/source/util/test/utilTests.cpp b/source/util/test/utilTests.cpp index 01a55ae710..6e32167f88 100644 --- a/source/util/test/utilTests.cpp +++ b/source/util/test/utilTests.cpp @@ -76,6 +76,16 @@ TEST(utilTest, wchar_pattern_match_test) { const TdWchar* str12 = L""; ret = wcsPatternMatch(reinterpret_cast(pattern12), 4, reinterpret_cast(str12), 0, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); + + const TdWchar* pattern13 = L"%\\_6 "; + const TdWchar* str13 = L"6a6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern13), 6, reinterpret_cast(str13), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern14 = L"%\\%6 "; + const TdWchar* str14 = L"6a6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern14), 6, reinterpret_cast(str14), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); } TEST(utilTest, wchar_pattern_match_no_terminated) { @@ -126,14 +136,24 @@ TEST(utilTest, wchar_pattern_match_no_terminated) { ret = wcsPatternMatch(reinterpret_cast(pattern8), 8, reinterpret_cast(str8), 6, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - const TdWchar* pattern9 = L"6\\_6 "; + const TdWchar* pattern9 = L"6\\_6 "; const TdWchar* str9 = L"6_6 "; - ret = wcsPatternMatch(reinterpret_cast(pattern9), 4, reinterpret_cast(str9), 3, &pInfo); + ret = wcsPatternMatch(reinterpret_cast(pattern9), 6, reinterpret_cast(str9), 4, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_MATCH); const TdWchar* pattern10 = L"% "; const TdWchar* str10 = L"6_6 "; - ret = wcsPatternMatch(reinterpret_cast(pattern10), 1, reinterpret_cast(str10), 3, &pInfo); + ret = wcsPatternMatch(reinterpret_cast(pattern10), 2, reinterpret_cast(str10), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern11 = L"%\\_6 "; + const TdWchar* str11 = L"6_6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern11), 6, reinterpret_cast(str11), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern12 = L"%\\%6 "; + const TdWchar* str12 = L"6%6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern12), 6, reinterpret_cast(str12), 4, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_MATCH); } @@ -209,6 +229,36 @@ TEST(utilTest, char_pattern_match_test) { const char* str13 = "a%c"; ret = patternMatch(pattern13, 5, str13, strlen(str13), &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern14 = "%a\\%c"; + const char* str14 = "a%c"; + ret = patternMatch(pattern14, strlen(pattern14), str14, strlen(str14), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern15 = "_a\\%c"; + const char* str15 = "ba%c"; + ret = patternMatch(pattern15, strlen(pattern15), str15, strlen(str15), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern16 = "_\\%c"; + const char* str16 = "a%c"; + ret = patternMatch(pattern16, strlen(pattern16), str16, strlen(str16), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern17 = "_\\%c"; + const char* str17 = "ba%c"; + ret = patternMatch(pattern17, strlen(pattern17), str17, strlen(str17), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); + + const char* pattern18 = "%\\%c"; + const char* str18 = "abc"; + ret = patternMatch(pattern18, strlen(pattern18), str18, strlen(str18), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern19 = "%\\_c"; + const char* str19 = "abc"; + ret = patternMatch(pattern19, strlen(pattern19), str19, strlen(str19), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); } TEST(utilTest, char_pattern_match_no_terminated) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..0aab8de088 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -131,6 +131,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py new file mode 100644 index 0000000000..95e2daf6d3 --- /dev/null +++ b/tests/system-test/2-query/like.py @@ -0,0 +1,148 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +from util.common import tdCom + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def initDB(self): + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db") + + def stopTest(self): + tdSql.execute("drop database if exists db") + + def like_wildcard_test(self): + tdSql.execute("create table db.t1x (ts timestamp, c1 varchar(100))") + tdSql.execute("create table db.t_1x (ts timestamp, c1 varchar(100))") + + tdSql.query("select * from information_schema.ins_columns where table_name like '%1x'") + tdSql.checkRows(4) + + tdSql.query("select * from information_schema.ins_columns where table_name like '%\_1x'") + tdSql.checkRows(2) + + + tdSql.query("insert into db.t1x values(now, 'abc'), (now+1s, 'a%c'),(now+2s, 'a_c'),(now+3s, '_c'),(now+4s, '%c')") + + tdSql.query("select * from db.t1x") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '%_c'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '%__c'") + tdSql.checkRows(3) + + tdSql.query("select * from db.t1x where c1 like '%\_c'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t1x where c1 like '%\%c'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t1x where c1 like '_\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a%c") + + tdSql.query("select * from db.t1x where c1 like '_\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a_c") + + tdSql.query("select * from db.t1x where c1 like '%%_c'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '%_%c'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '__%c'") + tdSql.checkRows(3) + + tdSql.query("select * from db.t1x where c1 not like '__%c'") + tdSql.checkRows(2) + + def like_cnc_wildcard_test(self): + tdSql.execute("create table db.t3x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t3x values(now, '我是中文'), (now+1s, '我是_中文'), (now+2s, '我是%中文'), (now+3s, '%中文'),(now+4s, '_中文')") + tdSql.query("select * from db.t3x") + tdSql.checkRows(5) + + tdSql.query("select * from db.t3x where c1 like '%中文'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t3x where c1 like '%中_文'") + tdSql.checkRows(0) + + tdSql.query("select * from db.t3x where c1 like '%\%中文'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t3x where c1 like '%\_中文'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t3x where c1 like '_中文'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t3x where c1 like '\_中文'") + tdSql.checkRows(1) + + def like_multi_wildcard_test(self): + tdSql.execute("create table db.t4x (ts timestamp, c1 varchar(100))") + + # 插入测试数据 + tdSql.execute("insert into db.t4x values(now, 'abc'), (now+1s, 'a%c'),(now+2s, 'a_c'),(now+3s, '_c'),(now+4s, '%c')") + tdSql.execute("insert into db.t4x values(now+5s, '%%%c'),(now+6s, '___c'),(now+7s, '%_%c'),(now+8s, '%\\c')") + + tdSql.query("select * from db.t4x where c1 like '%%%_'") + tdSql.checkRows(9) + + tdSql.query("select * from db.t4x where c1 like '\%\%\%_'") + tdSql.checkRows(1) + + tdSql.query("select * from db.t4x where c1 like '%\_%%'") + tdSql.checkRows(4) + + tdSql.query("select * from db.t4x where c1 like '_\%\%'") + tdSql.checkRows(0) + + tdSql.query("select * from db.t4x where c1 like '%abc%'") + tdSql.checkRows(1) + + tdSql.query("select * from db.t4x where c1 like '_%abc%'") + tdSql.checkRows(0) + + tdSql.query("select * from db.t4x where c1 like '\%%\%%'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t4x where c1 like '\%\_%\%%'") + tdSql.checkRows(1) + + def run(self): + tdLog.printNoPrefix("==========start like_wildcard_test run ...............") + tdSql.prepare(replica = self.replicaVar) + + + self.initDB() + self.like_wildcard_test() + self.like_cnc_wildcard_test() + self.like_multi_wildcard_test() + tdLog.printNoPrefix("==========end like_wildcard_test run ...............") + + self.stopTest() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index aefdb1e824..0b4ceace93 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -99,6 +99,10 @@ python3 ./test.py -f 2-query/ts-4233.py python3 ./test.py -f 2-query/ts-4233.py -Q 2 python3 ./test.py -f 2-query/ts-4233.py -Q 3 python3 ./test.py -f 2-query/ts-4233.py -Q 4 +python3 ./test.py -f 2-query/like.py +python3 ./test.py -f 2-query/like.py -Q 2 +python3 ./test.py -f 2-query/like.py -Q 3 +python3 ./test.py -f 2-query/like.py -Q 4 python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 -i False From dc01625ea14aba9e2f303c58193a8a91153c4299 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 26 Dec 2023 06:52:33 +0000 Subject: [PATCH 099/236] fix tag index double free --- source/dnode/vnode/src/meta/metaTable.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 976ee616f9..35a0adc4da 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -36,7 +36,6 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); - static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->uid = pEntry->uid; pInfo->version = pEntry->version; @@ -562,6 +561,7 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaULock(pMeta); metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } nStbEntry.version = version; @@ -692,6 +692,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); metaULock(pMeta); metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } // clear idx flag @@ -1076,7 +1077,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t* pSysTbl) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) { void *pData = NULL; int nData = 0; int rc = 0; @@ -1146,6 +1147,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); } metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } } tDecoderClear(&tdc); @@ -1865,6 +1867,7 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb } tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } tdbTbcClose(pCtbIdxc); return 0; @@ -2122,7 +2125,7 @@ int metaUpdateChangeTimeWithLock(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeM if (!tsTtlChangeOnWrite) return 0; metaWLock(pMeta); - int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs); + int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs); metaULock(pMeta); return ret; } @@ -2237,6 +2240,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); } metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } } end: From 40aa5f697f6c31e9ac21c180e2e684c77483322a Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 26 Dec 2023 07:21:02 +0000 Subject: [PATCH 100/236] fix tag index double free --- source/dnode/vnode/src/meta/metaTable.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 35a0adc4da..ac2486eda1 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -2231,14 +2231,12 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { nTagData = tDataTypes[pTagColumn->type].bytes; } - if (pTagData != NULL) { - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, - pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - ret = -1; - goto end; - } - tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, + pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { + ret = -1; + goto end; } + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); pTagIdxKey = NULL; } From 2ba586513f9c62031cca9bb3152858f83a64bd45 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 26 Dec 2023 16:41:33 +0800 Subject: [PATCH 101/236] fix(tsdb/cache): erase entry from lru if deleted --- source/dnode/vnode/src/tsdb/tsdbCache.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d7fea4956d..a39c74a679 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1183,8 +1183,10 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { + if (pLastCol->dirty) { pLastCol->dirty = 0; + } + if (pLastCol->ts <= eKey && pLastCol->ts >= sKey) { erase = true; } taosLRUCacheRelease(pTsdb->lruCache, h, erase); @@ -1197,8 +1199,10 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[num_keys + i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { + if (pLastCol->dirty) { pLastCol->dirty = 0; + } + if (pLastCol->ts <= eKey && pLastCol->ts >= sKey) { erase = true; } taosLRUCacheRelease(pTsdb->lruCache, h, erase); From e2564a61899ca75df168f62d8ad1c0dc4a3a241c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 26 Dec 2023 18:35:49 +0800 Subject: [PATCH 102/236] fix issue --- source/libs/executor/src/scanoperator.c | 4 ++- .../executor/src/streameventwindowoperator.c | 1 + source/libs/stream/src/streamSessionState.c | 8 ++--- tests/script/tsim/stream/event2.sim | 33 +++++++++++++++++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..cf5ea95088 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1961,8 +1961,10 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { } static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { - if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { + if (pInfo->pUpdateInfo) { pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + } + if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { checkUpdateData(pInfo, true, pBlock, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); if (pInfo->pUpdateDataRes->info.rows > 0) { diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 6adab74344..bd247eba07 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -622,6 +622,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { } setEventWindowFlag(pAggSup, &curInfo); if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { + saveSessionOutputBuf(pAggSup, &curInfo.winInfo); continue; } diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 45d656c456..c71edccb99 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -449,13 +449,13 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void SSHashObj* pSessionBuff = getRowStateBuff(pCur->pStreamFileState); void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); - if (!ppBuff) { - return TSDB_CODE_FAILED; + SArray* pWinStates = NULL; + if (ppBuff) { + pWinStates = (SArray*)(*ppBuff); } - SArray* pWinStates = (SArray*)(*ppBuff); - int32_t size = taosArrayGetSize(pWinStates); if (pCur->buffIndex >= 0) { + int32_t size = taosArrayGetSize(pWinStates); if (pCur->buffIndex >= size) { return TSDB_CODE_FAILED; } diff --git a/tests/script/tsim/stream/event2.sim b/tests/script/tsim/stream/event2.sim index eb9fca46e6..9fc7615fb8 100644 --- a/tests/script/tsim/stream/event2.sim +++ b/tests/script/tsim/stream/event2.sim @@ -31,6 +31,8 @@ sql insert into t4 values(1648791223000,1,1,10,3.0); sql insert into t4 values(1648791233000,0,2,11,1.0); sql insert into t4 values(1648791243000,1,9,12,2.0); +sleep 1000 + sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 1 into streamt0 as select _wstart as s, count(*) c1, sum(b), max(c), _wend as e from st partition by tbname event_window start with a = 0 end with b = 9; sleep 1000 @@ -79,6 +81,37 @@ if $data21 != 2 then goto loop0 endi +sql insert into t3 values(1648791222000,0,1,7,3.0); + +$loop_count = 0 +loop1: + +sleep 300 +print 2 sql select * from streamt0 order by 1, 2, 3, 4; +sql select * from streamt0 order by 1, 2, 3, 4; + +print +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then + print ======rows=$rows + goto loop1 +endi + +if $data01 != 5 then + print ======data01=$data01 + goto loop0 +endi + print event1 end system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5592e11235cb04824e46782eb5a172a6324af0bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:02:52 +0800 Subject: [PATCH 103/236] fix(stream): handle error when checkpoint is interrupted by nodeUpdate. --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 25 ++++++++++++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a8b5842b93..1ee5256775 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1295,8 +1295,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (status == TASK_STATUS__CK) { ASSERT(pTask->chkInfo.checkpointingId == req.checkpointId); tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64 - " already received, ignore this msg and continue process checkpoint", - pTask->id.idStr, pTask->chkInfo.checkpointingId); + " transId:%d already received, ignore this msg and continue process checkpoint", + pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index bfa269b0d5..00a8940b6a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -746,7 +746,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { taosArrayClear(pTask->pReadyMsgList); stDebug("s-task:%s level:%d source checkpoint completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel); } else { - stDebug("s-task:%s level:%d already send rsp to mnode", pTask->id.idStr, pTask->info.taskLevel); + stDebug("s-task:%s level:%d already send rsp checkpoint success to mnode", pTask->id.idStr, pTask->info.taskLevel); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7d2a572cda..eafa5adea8 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -625,10 +625,29 @@ int32_t doStreamExecTask(SStreamTask* pTask) { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. if (type == STREAM_INPUT__CHECKPOINT) { + + // todo add lock char* p = NULL; - streamTaskGetStatus(pTask, &p); - stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); - streamTaskBuildCheckpoint(pTask); + ETaskStatus s = streamTaskGetStatus(pTask, &p); + if (s == TASK_STATUS__CK) { + stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); + streamTaskBuildCheckpoint(pTask); + } else { + // todo refactor + int32_t code = 0; + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + 0, tstrerror(code)); + } + } + return 0; } } From e553b84904309b7ea51edecd039820ebce965a50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:16:49 +0800 Subject: [PATCH 104/236] fix(stream): fix syntax error. --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index eafa5adea8..1f7bb56ec1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -643,7 +643,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", pTask->id.idStr, 0, tstrerror(code)); } } From 3faf1514f43d1e03cb6557349a644ffcc598f93b Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 26 Dec 2023 20:47:15 +0800 Subject: [PATCH 105/236] doGroupedTableScan --- source/libs/executor/src/scanoperator.c | 45 +++++++++++----------- source/libs/planner/src/planLogicCreater.c | 1 - source/libs/planner/src/planPhysiCreater.c | 1 - 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6df46ef4d0..0f9b2f1eac 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -657,7 +657,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, // record processed (non empty) table -static int32_t insertTableToProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { +static int32_t markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (!pTableScanInfo->needCountEmptyTable) { return TSDB_CODE_SUCCESS; } @@ -770,6 +770,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; + bool outputAll = pTableScanInfo->base.pTableListInfo->oneTableForEachGroup; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { @@ -780,7 +781,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - insertTableToProcessed(pTableScanInfo, p->info.id.uid); + markTableProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -809,7 +810,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - insertTableToProcessed(pTableScanInfo, p->info.id.uid); + markTableProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -827,30 +828,30 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->needCountEmptyTable) { - if (num == 0 && 0 == taosHashGetSize(pTableScanInfo->pValuedTables)) { - // table by table, num is 0 - if (!pTableScanInfo->processingEmptyTable) { - pTableScanInfo->processingEmptyTable = true; - // current table is empty, fill result block info & return - const STableKeyInfo* info = tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->currentTable); - return getBlockForEmptyTable(pOperator, info); - } - - } else if (num > taosHashGetSize(pTableScanInfo->pValuedTables)) { - // group by group, num >= 1 + // pList is NULL in mode TABLE_SCAN__TABLE_ORDER for streamscan, no need to process + // pList not NULL, group by group, num >= 1 + int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pValuedTables); + if (pList && num > tb_cnt) { if (!pTableScanInfo->processingEmptyTable) { pTableScanInfo->processingEmptyTable = true; pTableScanInfo->currentTable = 0; } if (pTableScanInfo->currentTable < num) { - // loop: get empty table uid & process - while (pTableScanInfo->currentTable < num) { - const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; - if (pTableScanInfo->pValuedTables && - NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { - } else { - return getBlockForEmptyTable(pOperator, info); + if (outputAll) { + // loop: get empty table uid & process + while (pTableScanInfo->currentTable < num) { + const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; + if (pTableScanInfo->pValuedTables && + NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { + } else { + return getBlockForEmptyTable(pOperator, info); + } } + } else if (tb_cnt == 0) { + // only need one & all empty table in this group + // output first one + pTableScanInfo->currentTable = num; + return getBlockForEmptyTable(pOperator, pList); } } } @@ -1005,7 +1006,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - + if (pOperator->pOperatorGetParam) { pOperator->dynamicTask = true; int32_t code = createTableListInfoFromParam(pOperator); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f55689c472..e642b5bd5f 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -758,7 +758,6 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; - if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 444bda84cd..96205904e3 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2293,7 +2293,6 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); From 42f9d54a70b67170278adf3dac701ec10742d4a9 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 00:28:42 +0800 Subject: [PATCH 106/236] adjust --- source/libs/executor/src/scanoperator.c | 3 +- tests/system-test/2-query/group_partition.py | 42 +++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f9b2f1eac..c69d963c79 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -770,7 +770,8 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - bool outputAll = pTableScanInfo->base.pTableListInfo->oneTableForEachGroup; + // Only when all tables are scanned can you determine how many groups the tag has + bool outputAll = true; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index c08ea98ef6..3d9874bfe2 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -40,9 +40,6 @@ class TDTestCase: tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) @@ -57,12 +54,13 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # tag - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by t2 ") - tdSql.checkRows(check_num) - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") tdSql.checkRows(check_num) + # multi tag + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") + tdSql.checkRows(check_num) + # having tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -87,10 +85,16 @@ class TDTestCase: tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") tdSql.checkRows(check_num) - # count + sum(col) + # multi agg tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(real_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") num = 0 @@ -105,14 +109,20 @@ class TDTestCase: tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} group by t2, t3, c1 ") + tdSql.checkRows(real_num * self.row_nums) + def test_partitionby(self, check_num, real_num): tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + tdSql.query(f" select count(c5), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(real_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") tdSql.checkRows(check_num) @@ -124,12 +134,12 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) #tag - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} partition by t2 ") - tdSql.checkRows(check_num) - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") tdSql.checkRows(check_num) + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} partition by t2, t3, tbname") + tdSql.checkRows(check_num) + # having tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -150,6 +160,13 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + #multi agg + tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(real_num) + #inner select tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) @@ -170,6 +187,9 @@ class TDTestCase: tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} partition by t2, t3, c1 ") + tdSql.checkRows(real_num * self.row_nums) + def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") From 86351d5487ae55010b6922d58e058c8a58420173 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 01:05:04 +0800 Subject: [PATCH 107/236] check count & window --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e642b5bd5f..ea8bb666b1 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -487,7 +487,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } bool isCountByTag = false; - if (pSelect->hasCountFunc) { + if (pSelect->hasCountFunc && !pSelect->pWindow) { if (pSelect->pGroupByList) { isCountByTag = !keysHasCol(pSelect->pGroupByList); } else if (pSelect->pPartitionByList) { From a46c921f688b1643fc4de8131fa11e21bd01822e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 11:29:03 +0800 Subject: [PATCH 108/236] check block rows --- source/libs/executor/src/streamtimewindowoperator.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..b9f3dc19d8 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2343,10 +2343,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); - if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - ASSERT(pBlock->info.rows > 0); - break; - } if (code == -1) { // for history @@ -2363,6 +2359,11 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa continue; } + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + break; + } + pGroupResInfo->index += 1; for (int32_t j = 0; j < numOfExprs; ++j) { @@ -2383,6 +2384,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { + qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From a97c2e0e165b32ddc02392041ee843aff76172c3 Mon Sep 17 00:00:00 2001 From: charles Date: Wed, 27 Dec 2023 13:27:38 +0800 Subject: [PATCH 109/236] add test case for jira TS-4382 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/test_ts4382.py | 75 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/system-test/2-query/test_ts4382.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e35bc60e20..e7c26b9833 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -518,6 +518,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py diff --git a/tests/system-test/2-query/test_ts4382.py b/tests/system-test/2-query/test_ts4382.py new file mode 100644 index 0000000000..50ec67ed23 --- /dev/null +++ b/tests/system-test/2-query/test_ts4382.py @@ -0,0 +1,75 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TS-4382 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.stbname = 'st' + self.ctbname_list = ["ct1", "ct2"] + self.tag_value_list = ['{"instance":"100"}', '{"instance":"200"}'] + + def prepareData(self): + # db + tdSql.execute("create database {};".format(self.dbname)) + tdSql.execute("use {};".format(self.dbname)) + tdLog.debug("Create database %s" % self.dbname) + + # super table + tdSql.execute("create table {} (ts timestamp, col1 int) tags (t1 json);".format(self.stbname)) + tdLog.debug("Create super table %s" % self.stbname) + + # child table + for i in range(len(self.ctbname_list)): + tdSql.execute("create table {} using {} tags('{}');".format(self.ctbname_list[i], self.stbname, self.tag_value_list[i])) + tdLog.debug("Create child table %s" % self.ctbname_list) + + # insert data + tdSql.execute("insert into {} values(now, 1)(now+1s, 2)".format(self.ctbname_list[0])) + tdSql.execute("insert into {} values(now, null)(now+1s, null)".format(self.ctbname_list[1])) + + def run(self): + self.prepareData() + sql_list = [ + # super table query with correct tag name of json type + { + "sql": "select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(col1) from st group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), t1->'instance' order by time;", + "result_check": "0.0" + }, + # child table query with incorrect tag name of json type + { + "sql": "select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(col1) from ct1 group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), t1->'name' order by time;", + "result_check": "None" + }, + # child table query with null value + { + "sql": "select ts, avg(col1) from ct2 group by ts, t1->'name' order by ts;", + "result_check": "None" + } + ] + for sql_dic in sql_list: + tdSql.query(sql_dic["sql"]) + tdLog.debug("execute sql: %s" % sql_dic["sql"]) + for item in [row[1] for row in tdSql.queryResult]: + if sql_dic["result_check"] in str(item): + tdLog.debug("Check query result of '{}' successfully".format(sql_dic["sql"])) + break + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + From 382569591fd797e9a89208857124114be63b11d3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 14:01:38 +0800 Subject: [PATCH 110/236] log --- source/libs/executor/src/streamtimewindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index b9f3dc19d8..024a44d107 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2384,7 +2384,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From 158c5996dfb98aadb767847fce4d63555d63f957 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 15:51:36 +0800 Subject: [PATCH 111/236] adjust case --- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 14 +- tests/system-test/2-query/group_partition.py | 219 +++++++++---------- 3 files changed, 101 insertions(+), 134 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ea8bb666b1..12b7360165 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -487,7 +487,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } bool isCountByTag = false; - if (pSelect->hasCountFunc && !pSelect->pWindow) { + if (pSelect->hasCountFunc && NULL == pSelect->pWindow) { if (pSelect->pGroupByList) { isCountByTag = !keysHasCol(pSelect->pGroupByList); } else if (pSelect->pPartitionByList) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 96205904e3..e266c55425 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,18 +587,6 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static bool calcNeedCountEmpty(SScanLogicNode* pScanLogicNode) { - if (pScanLogicNode->interval > 0) { - return false; - } - - if (pScanLogicNode->isCountByTag) { - return true; - } - - return false; -} - static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, @@ -635,7 +623,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; - pTableScan->needCountEmptyTable = calcNeedCountEmpty(pScanLogicNode); + pTableScan->needCountEmptyTable = pScanLogicNode->isCountByTag; int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 3d9874bfe2..0ac1be8482 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -35,161 +35,132 @@ class TDTestCase: tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") - def test_groupby(self, check_num, real_num): - # tbname - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") + def test_groupby(self, keyword, check_num, nonempty_tb_num): + ####### by tbname + tdSql.query(f"select count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} {keyword} by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname order by count(*)") + tdSql.query(f"select tbname from {self.dbname}.{self.stable} {keyword} by tbname order by count(*)") tdSql.checkRows(check_num) - tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname having count(*)>=0") + # last + tdSql.query(f"select last(ts), count(*) from {self.dbname}.{self.stable} {keyword} by tbname order by last(ts)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} {keyword} by tbname having count(*)>=0") tdSql.checkRows(check_num) # having filter out empty - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") - tdSql.checkRows(check_num - real_num) + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} {keyword} by tbname having count(*) <= 0") + tdSql.checkRows(check_num - nonempty_tb_num) - # tag - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") + ####### by tag + tdSql.query(f"select t2, count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by t2 ") tdSql.checkRows(check_num) - # multi tag - tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} {keyword} by t2 having count(*) <= 0") + tdSql.checkRows(check_num - nonempty_tb_num) + + # where + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now {keyword} by t2 ") tdSql.checkRows(check_num) - # having - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") - tdSql.checkRows(check_num - real_num) - - # col where filter nothing - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 {keyword} by t2 ") tdSql.checkRows(check_num) - # col where filter all - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 {keyword} by t2 ") tdSql.checkRows(check_num) - # col where filter part - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") - tdSql.checkRows(check_num) - - # col - tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - #inner select - tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") - tdSql.checkRows(check_num) - - # multi agg - tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") - tdSql.checkRows(real_num) - - ############### same with old ############### - tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") + ####### by col + tdSql.query(f"select c1, count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by c1 ") num = 0 - if real_num > 0: + if nonempty_tb_num > 0: num = self.row_nums tdSql.checkRows(num) - tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} group by ts ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} {keyword} by ts ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) # col + tag - tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} {keyword} by t2, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} group by t2, t3, c1 ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} {keyword} by t2, t3, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - - def test_partitionby(self, check_num, real_num): - tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) - tdSql.query(f" select count(c5), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") - tdSql.checkRows(real_num) - - tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") - tdSql.checkRows(check_num) - - tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname having count(*)>=0") - tdSql.checkRows(check_num) - - # having filter out empty - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") - tdSql.checkRows(check_num - real_num) - - #tag - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") + def test_multi_group_key(self, check_num, nonempty_tb_num): + # multi tag/tbname + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") tdSql.checkRows(check_num) tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} partition by t2, t3, tbname") tdSql.checkRows(check_num) - # having - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") - tdSql.checkRows(check_num - real_num) + # multi tag + col + tdSql.query(f"select t1, t2, c1, count(*) from {self.dbname}.{self.stable} partition by t1, t2, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - # col where filter nothing - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") - tdSql.checkRows(check_num) + # tag + multi col + tdSql.query(f"select t2, c1, c2, count(*) from {self.dbname}.{self.stable} partition by t2, c1, c2 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - # col where filter all - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") - tdSql.checkRows(check_num) - # col where filter part - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") - tdSql.checkRows(check_num) + def test_multi_agg(self, all_tb_num, nonempty_tb_num): + tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(all_tb_num) - #col - tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) + tdSql.query(f"select count(1), sum(1), avg(c1), apercentile(c1, 50), spread(ts) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(all_tb_num) - #multi agg - tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) + tdSql.query(f"select count(c1), sum(c1), min(c1), mode(c1), stddev(c1), spread(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(all_tb_num) + + # elapsed: continuous duration in a statistical period, table merge scan + tdSql.query(f" select count(c1), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(nonempty_tb_num) tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") - tdSql.checkRows(real_num) + tdSql.checkRows(nonempty_tb_num) + + + def test_innerSelect(self, check_num): + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") + tdSql.checkRows(check_num) - #inner select tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) + tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.stable} partition by t1)") + tdSql.checkRows(check_num) + + + def test_window(self, nonempty_tb_num): + # empty group optimization condition is not met + # time window tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") - tdSql.checkRows(real_num) + tdSql.checkRows(nonempty_tb_num) - ############### same with old ############### - tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") - num = 0 - if real_num > 0: - num = self.row_nums - tdSql.checkRows(num) - - tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} partition by ts ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select _wstart, _wend, count(c1), max(c1), apercentile(c1, 50) from {self.dbname}.{self.stable} partition by tbname interval(1d)") + tdSql.checkRows(nonempty_tb_num) - tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") - tdSql.checkRows(real_num * self.row_nums) + # state window + tdSql.query(f"select tbname, count(*), c1 from {self.dbname}.{self.stable} partition by tbname state_window(c1)") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} partition by t2, t3, c1 ") - tdSql.checkRows(real_num * self.row_nums) + # session window + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname session(ts, 5s)") + tdSql.checkRows(nonempty_tb_num) + # event window + tdSql.query(f"select tbname, count(*) from {self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") + tdSql.checkRows(nonempty_tb_num) + + + def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") @@ -199,21 +170,29 @@ class TDTestCase: def run(self): tdSql.prepare() self.prepare_db() - check_num = self.tb_nums - self.test_groupby(check_num, 0) - self.test_partitionby(check_num, 0) - # insert into half of tables - real_num = 5 - self.insert_db(real_num, self.row_nums) - self.test_groupby(check_num, real_num) - self.test_partitionby(check_num, real_num) + # empty table only + self.test_groupby('group', self.tb_nums, 0) + self.test_groupby('partition', self.tb_nums, 0) + self.test_innerSelect(self.tb_nums) + self.test_multi_group_key(self.tb_nums, 0) + self.test_multi_agg(self.tb_nums, 0) + self.test_window(0) - # test old version before changed - # self.test_groupby(0, 0) - # self.test_partitionby(0, 0) + # insert data to 5 tables + nonempty_tb_num = 5 + self.insert_db(nonempty_tb_num, self.row_nums) + + self.test_groupby('group', self.tb_nums, nonempty_tb_num) + self.test_groupby('partition', self.tb_nums, nonempty_tb_num) + self.test_innerSelect(self.tb_nums) + self.test_multi_group_key(self.tb_nums, nonempty_tb_num) + self.test_multi_agg(self.tb_nums, nonempty_tb_num) + self.test_window(nonempty_tb_num) + + ## test old version before changed + # self.test_groupby('group', 0, 0) # self.insert_db(5, self.row_nums) - # self.test_groupby(5, 5) - # self.test_partitionby(5, 5) + # self.test_groupby('group', 5, 5) self.test_error() From f6e55d358b375e86c5d82aca6ed089303d4b9347 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 16:12:50 +0800 Subject: [PATCH 112/236] =?UTF-8?q?=C2=A0adjust=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/system-test/2-query/group_partition.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 0ac1be8482..8b06f0d6fd 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -134,7 +134,7 @@ class TDTestCase: tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) - tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.stable} partition by t1)") + tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.dbname}.{self.stable} partition by t1)") tdSql.checkRows(check_num) @@ -156,7 +156,7 @@ class TDTestCase: tdSql.checkRows(nonempty_tb_num) # event window - tdSql.query(f"select tbname, count(*) from {self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") tdSql.checkRows(nonempty_tb_num) From 10e86fc733a53b57c056c1a4e7bc61a3624cb2df Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Dec 2023 19:25:08 +0800 Subject: [PATCH 113/236] fix:[TS-4391] rebalance cnt always 1 if msg lost --- source/dnode/mnode/impl/inc/mndConsumer.h | 1 + source/dnode/mnode/impl/src/mndConsumer.c | 28 +++++++++++++++------- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 94c937e8f4..f075510428 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -47,6 +47,7 @@ int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj int32_t mndSetConsumerDropLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); bool mndRebTryStart(); +bool mndRebCanStart(); void mndRebEnd(); void mndRebCntInc(); void mndRebCntDec(); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index c9ee66d3a0..11c929c898 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -101,6 +101,16 @@ bool mndRebTryStart() { return old == 0; } +bool mndRebCanStart() { + int32_t val = atomic_load_32(&mqRebInExecCnt); + if (val < 0) { + mError("rebalance trans end, rebalance counter:%d should not be less equalled than 0, ignore counter desc", val); + return false; + } + mInfo("tq timer, rebalance counter val:%d", val); + return val == 0; +} + void mndRebEnd() { mndRebCntDec(); } void mndRebCntInc() { @@ -119,7 +129,7 @@ void mndRebCntDec() { int32_t newVal = val - 1; int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal); if (oldVal == val) { - mDebug("rebalance trans end, rebalance counter:%d", newVal); + mInfo("rebalance trans end, rebalance counter:%d", newVal); break; } } @@ -284,10 +294,10 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqConsumerObj *pConsumer; void *pIter = NULL; - mDebug("start to process mq timer"); + mInfo("start to process mq timer"); // rebalance cannot be parallel - if (!mndRebTryStart()) { + if (!mndRebCanStart()) { mInfo("mq rebalance already in progress, do nothing"); return 0; } @@ -295,7 +305,6 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); if (pRebMsg == NULL) { mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t)sizeof(SMqDoRebalanceMsg)); - mndRebEnd(); return TSDB_CODE_OUT_OF_MEMORY; } @@ -303,7 +312,6 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { if (pRebMsg->rebSubHash == NULL) { mError("failed to create rebalance hashmap"); rpcFreeCont(pRebMsg); - mndRebEnd(); return TSDB_CODE_OUT_OF_MEMORY; } @@ -390,6 +398,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } + + if (newTopicNum == 0 && removedTopicNum == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } + taosRUnLockLatch(&pConsumer->lock); } @@ -397,7 +410,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq rebalance will be triggered"); + mInfo("mq send msg to rebalance"); SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_DO_REBALANCE, .pCont = pRebMsg, @@ -407,8 +420,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); - mDebug("mq timer finished, no need to re-balance"); - mndRebEnd(); + mInfo("mq timer finished, no need to re-balance"); } return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 01d4d1029c..8cd7f6bc9c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -732,6 +732,10 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMqDoRebalanceMsg *pReq = pMsg->pCont; void *pIter = NULL; // bool rebalanceOnce = false; // to ensure only once. + if (!mndRebTryStart()) { + mInfo("mq rebalance already in progress, do nothing"); + return 0; + } mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); From d795798648ac0d1945e90bbbf39d445ed0f88679 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 21:37:00 +0800 Subject: [PATCH 114/236] count state --- source/libs/executor/inc/executorInt.h | 11 ++- source/libs/executor/src/scanoperator.c | 94 ++++++++++++++----------- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 2523b87cfb..8b0bef5fa6 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -217,6 +217,12 @@ enum { TABLE_SCAN__BLOCK_ORDER = 2, }; +typedef enum ETableCountState { + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // scanning + TABLE_COUNT_STATE_END = 2, // finish or noneed to process +} ETableCountState; + typedef struct SAggSupporter { SSHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer @@ -262,17 +268,18 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SHashObj* pValuedTables; // non empty table uids + SHashObj* pRemainTables; // remain table to process SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; + uint8_t countState; // empty table count state + bool isOneGroup; // whether or not only one group in this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; bool needCountEmptyTable; - bool processingEmptyTable; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c69d963c79..31fe31007c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,26 +655,44 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } - -// record processed (non empty) table -static int32_t markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { +static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { return TSDB_CODE_SUCCESS; } - if (NULL == pTableScanInfo->pValuedTables) { + pTableScanInfo->isOneGroup = true; + if (NULL == pTableScanInfo->pRemainTables) { int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); - pTableScanInfo->pValuedTables = + pTableScanInfo->pRemainTables = taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (NULL == pTableScanInfo->pValuedTables) { + if (NULL == pTableScanInfo->pRemainTables) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_OUT_OF_MEMORY; } } - - taosHashPut(pTableScanInfo->pValuedTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, - sizeof(pTableScanInfo->scanTimes)); + uint64_t groupId = 0; + for (int32_t i = 0; i < num; i++) { + const STableKeyInfo* pInfo = pList + i; + if (pTableScanInfo->isOneGroup) { + if (i == 0) { + groupId = pInfo->groupId; + } else if (groupId != pInfo->groupId) { + pTableScanInfo->isOneGroup = false; + } + } + taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); + } return TSDB_CODE_SUCCESS; } +static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { + // case0 group scanning, mark + // case1 stream scan: no need to mark + if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { + return; + } + taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); +} + static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -770,14 +788,17 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - // Only when all tables are scanned can you determine how many groups the tag has - bool outputAll = true; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; } + if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { + initRemainTable(pTableScanInfo, pList, num); + pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; + } + // do the ascending order traverse in the first place. while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); @@ -829,37 +850,28 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->needCountEmptyTable) { - // pList is NULL in mode TABLE_SCAN__TABLE_ORDER for streamscan, no need to process - // pList not NULL, group by group, num >= 1 - int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pValuedTables); - if (pList && num > tb_cnt) { - if (!pTableScanInfo->processingEmptyTable) { - pTableScanInfo->processingEmptyTable = true; - pTableScanInfo->currentTable = 0; - } - if (pTableScanInfo->currentTable < num) { - if (outputAll) { - // loop: get empty table uid & process - while (pTableScanInfo->currentTable < num) { - const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; - if (pTableScanInfo->pValuedTables && - NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { - } else { - return getBlockForEmptyTable(pOperator, info); - } - } - } else if (tb_cnt == 0) { - // only need one & all empty table in this group - // output first one - pTableScanInfo->currentTable = num; - return getBlockForEmptyTable(pOperator, pList); + int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); + if (tb_cnt) { + if (!pTableScanInfo->isOneGroup) { + // get first empty table uid, mark processed & rm from hash + void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); + if (pIte != NULL) { + size_t keySize = 0; + uint64_t* pUid = taosHashGetKey(pIte, &keySize); + STableKeyInfo info = {.uid = *pUid, .groupId = *(uint64_t*)pIte}; + taosHashCancelIterate(pTableScanInfo->pRemainTables, pIte); + markTableProcessed(pTableScanInfo, *pUid); + return getBlockForEmptyTable(pOperator, &info); } + } else { + // output one table for this group + taosHashClear(pTableScanInfo->pRemainTables); + return getBlockForEmptyTable(pOperator, pList); } } - pTableScanInfo->processingEmptyTable = false; + pTableScanInfo->countState = TABLE_COUNT_STATE_END; } - taosHashClear(pTableScanInfo->pValuedTables); return NULL; } @@ -937,7 +949,8 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - + pInfo->countState = TABLE_COUNT_STATE_NONE; + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -968,6 +981,7 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { } tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); + pInfo->countState = TABLE_COUNT_STATE_NONE; ASSERT(pInfo->base.dataReader == NULL); int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -1034,6 +1048,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { int32_t numOfTables = 0; // tableListGetSize(pTaskInfo->pTableListInfo); STableKeyInfo tInfo = {0}; + pInfo->countState = TABLE_COUNT_STATE_END; while (1) { SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); @@ -1096,7 +1111,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - taosHashCleanup(pTableScanInfo->pValuedTables); + taosHashCleanup(pTableScanInfo->pRemainTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1161,7 +1176,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->exprSupp.numOfExprs = numOfCols; pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; - pInfo->processingEmptyTable = false; pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); From 7eb0e42bb9906fe0c11df5d28ddae2af193b0fb2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:02:52 +0800 Subject: [PATCH 115/236] fix(stream): handle error when checkpoint is interrupted by nodeUpdate. --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 25 ++++++++++++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c8cc4f9d54..33f937a909 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1294,8 +1294,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (status == TASK_STATUS__CK) { ASSERT(pTask->chkInfo.checkpointingId == req.checkpointId); tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64 - " already received, ignore this msg and continue process checkpoint", - pTask->id.idStr, pTask->chkInfo.checkpointingId); + " transId:%d already received, ignore this msg and continue process checkpoint", + pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index bfa269b0d5..00a8940b6a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -746,7 +746,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { taosArrayClear(pTask->pReadyMsgList); stDebug("s-task:%s level:%d source checkpoint completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel); } else { - stDebug("s-task:%s level:%d already send rsp to mnode", pTask->id.idStr, pTask->info.taskLevel); + stDebug("s-task:%s level:%d already send rsp checkpoint success to mnode", pTask->id.idStr, pTask->info.taskLevel); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 25f32195be..5822b18bc3 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -625,10 +625,29 @@ int32_t doStreamExecTask(SStreamTask* pTask) { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. if (type == STREAM_INPUT__CHECKPOINT) { + + // todo add lock char* p = NULL; - streamTaskGetStatus(pTask, &p); - stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); - streamTaskBuildCheckpoint(pTask); + ETaskStatus s = streamTaskGetStatus(pTask, &p); + if (s == TASK_STATUS__CK) { + stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); + streamTaskBuildCheckpoint(pTask); + } else { + // todo refactor + int32_t code = 0; + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + 0, tstrerror(code)); + } + } + return 0; } } From 646f24659e94336b67906c4f7181e0d76b2e7fd6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:16:49 +0800 Subject: [PATCH 116/236] fix(stream): fix syntax error. --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 5822b18bc3..5b4f603f87 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -643,7 +643,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", pTask->id.idStr, 0, tstrerror(code)); } } From 1659164926c8c862819be06dfced8af8142c27d6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 27 Dec 2023 23:00:18 +0800 Subject: [PATCH 117/236] fix(stream): remove invalid checkpoint id check. --- source/dnode/vnode/src/tq/tq.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 33f937a909..58f1fe954a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1292,7 +1292,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // check if the checkpoint msg already sent or not. if (status == TASK_STATUS__CK) { - ASSERT(pTask->chkInfo.checkpointingId == req.checkpointId); tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64 " transId:%d already received, ignore this msg and continue process checkpoint", pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId); From abed781bd324dda7c636e162a25646292214004a Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 23:15:11 +0800 Subject: [PATCH 118/236] opt --- source/libs/executor/src/scanoperator.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 31fe31007c..0714019352 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -657,6 +657,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_SUCCESS; } pTableScanInfo->isOneGroup = true; @@ -681,6 +682,7 @@ static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyIn } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); } + pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; return TSDB_CODE_SUCCESS; } @@ -690,6 +692,11 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { return; } + // case2 only one group, uid ready + if (pTableScanInfo->isOneGroup) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; + return; + } taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); } @@ -796,7 +803,6 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { initRemainTable(pTableScanInfo, pList, num); - pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; } // do the ascending order traverse in the first place. @@ -849,7 +855,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } } - if (pTableScanInfo->needCountEmptyTable) { + if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); if (tb_cnt) { if (!pTableScanInfo->isOneGroup) { @@ -865,14 +871,14 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } } else { // output one table for this group - taosHashClear(pTableScanInfo->pRemainTables); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return getBlockForEmptyTable(pOperator, pList); } } - pTableScanInfo->countState = TABLE_COUNT_STATE_END; } + taosHashClear(pTableScanInfo->pRemainTables); return NULL; } From a5f88a86e9b18ac7b3a3a61592659fa72a40c9a1 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 09:45:01 +0800 Subject: [PATCH 119/236] adjust --- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/scanoperator.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 8b0bef5fa6..b87dee475d 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -275,7 +275,7 @@ typedef struct STableScanInfo { int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state - bool isOneGroup; // whether or not only one group in this scan + bool isSameGroup; // whether all tables are in the same group this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0714019352..dd6dff0301 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,12 +655,12 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } -static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { +static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_SUCCESS; } - pTableScanInfo->isOneGroup = true; + pTableScanInfo->isSameGroup = true; if (NULL == pTableScanInfo->pRemainTables) { int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); pTableScanInfo->pRemainTables = @@ -673,11 +673,11 @@ static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyIn uint64_t groupId = 0; for (int32_t i = 0; i < num; i++) { const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isOneGroup) { + if (pTableScanInfo->isSameGroup) { if (i == 0) { groupId = pInfo->groupId; } else if (groupId != pInfo->groupId) { - pTableScanInfo->isOneGroup = false; + pTableScanInfo->isSameGroup = false; } } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); @@ -692,8 +692,8 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { return; } - // case2 only one group, uid ready - if (pTableScanInfo->isOneGroup) { + // case2 if all table in same group, process only once + if (pTableScanInfo->isSameGroup) { pTableScanInfo->countState = TABLE_COUNT_STATE_END; return; } @@ -802,7 +802,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { - initRemainTable(pTableScanInfo, pList, num); + initTableCountEnv(pTableScanInfo, pList, num); } // do the ascending order traverse in the first place. @@ -858,7 +858,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); if (tb_cnt) { - if (!pTableScanInfo->isOneGroup) { + if (!pTableScanInfo->isSameGroup) { // get first empty table uid, mark processed & rm from hash void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); if (pIte != NULL) { From 659c0a831e118455826f3720d100cf61140747ae Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 28 Dec 2023 10:25:37 +0800 Subject: [PATCH 120/236] fix: change the result schema --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 16e0332ee9..3ab038be9d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8769,7 +8769,7 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; - strcpy((*pSchema)[0].name, "result"); + strcpy((*pSchema)[0].name, "name"); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; @@ -8791,7 +8791,7 @@ static int32_t extractCompactDbResultSchema(int32_t* numOfCols, SSchema** pSchem (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = COMPACT_DB_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "name"); + strcpy((*pSchema)[0].name, "result"); (*pSchema)[1].type = TSDB_DATA_TYPE_INT; (*pSchema)[1].bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; From 631eb9a72c6a727aaaee2489e7edcd11512076af Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 28 Dec 2023 10:55:50 +0800 Subject: [PATCH 121/236] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/compact-col.py | 71 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 tests/system-test/2-query/compact-col.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..4bc226d93f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -29,6 +29,7 @@ ,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3 diff --git a/tests/system-test/2-query/compact-col.py b/tests/system-test/2-query/compact-col.py new file mode 100644 index 0000000000..0a4614d710 --- /dev/null +++ b/tests/system-test/2-query/compact-col.py @@ -0,0 +1,71 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-] + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tbname_vgroup") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tbname_vgroup") + tdSql.execute("create database if not exists tbname_vgroup") + tdSql.execute('use tbname_vgroup') + tdSql.execute('drop database if exists dbvg') + tdSql.execute('create database dbvg vgroups 8;') + + tdSql.execute('use dbvg;') + + tdSql.execute('create table st(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 2)") + + tdSql.execute("insert into ct3 using st tags(3) values('2021-04-19 00:00:03', 3)") + + tdSql.execute("insert into ct4 using st tags(4) values('2021-04-19 00:00:04', 4)") + + + tdSql.execute('create table st2(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct21 using st2 tags(1) values('2021-04-19 00:00:01', 1)") + + tdSql.execute("insert into ct22 using st2 tags(2) values('2021-04-19 00:00:02', 2)") + + tdSql.execute("insert into ct23 using st2 tags(3) values('2021-04-19 00:00:03', 3)") + + tdSql.execute("insert into ct24 using st2 tags(4) values('2021-04-19 00:00:04', 4)") + + + col_names = tdSql.getColNameList("compact database tbname_vgroup") + if col_names[0] != "result": + raise Exception("first column name of compact result shall be result") + + col_names = tdSql.getColNameList("show variables") + if col_names[0] != "name": + raise Exception("first column name of compact result shall be name") + + tdSql.execute('drop database tbname_vgroup') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From c124bcf52fca3a9528ba93d51d7563564a083525 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 11:29:03 +0800 Subject: [PATCH 122/236] check block rows --- source/libs/executor/src/streamtimewindowoperator.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..b9f3dc19d8 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2343,10 +2343,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); - if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - ASSERT(pBlock->info.rows > 0); - break; - } if (code == -1) { // for history @@ -2363,6 +2359,11 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa continue; } + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + break; + } + pGroupResInfo->index += 1; for (int32_t j = 0; j < numOfExprs; ++j) { @@ -2383,6 +2384,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { + qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From f1304680c590473a3c9c24eaaa582be9119b60f4 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 14:01:38 +0800 Subject: [PATCH 123/236] log --- source/libs/executor/src/streamtimewindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index b9f3dc19d8..024a44d107 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2384,7 +2384,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From 448b7c939f122dd1f25d11a4ae59a4cef5b79ed0 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 28 Dec 2023 11:09:38 +0800 Subject: [PATCH 124/236] reset datablock window info --- source/libs/executor/src/scanoperator.c | 1 + .../executor/src/streamtimewindowoperator.c | 27 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..b38b9bca49 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2316,6 +2316,7 @@ FETCH_NEXT_BLOCK: doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); int64_t numOfUpdateRes = pInfo->pUpdateDataRes->info.rows; qDebug("%s %" PRId64 " rows in datablock, update res:%" PRId64, id, pBlockInfo->rows, numOfUpdateRes); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 024a44d107..c749c3814d 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -801,10 +801,24 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat SRowBuffPos* pResPos = NULL; SResultRow* pResult = NULL; int32_t forwardRows = 0; + int32_t endRowId = pSDataBlock->info.rows - 1; SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; + if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { + qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 + ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); + + // timestamp of the data is incorrect + if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { + qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + } + } + int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = {0}; @@ -902,19 +916,6 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat pInfo->delKey = key; } int32_t prevEndPos = (forwardRows - 1) * step + startPos; - if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { - qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 - ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - blockDataUpdateTsWindow(pSDataBlock, 0); - - // timestamp of the data is incorrect - if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { - qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - } - } - if (IS_FINAL_INTERVAL_OP(pOperator)) { startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos); } else { From 9c997e34ea84f3c80eee0b1e353a903f74dc7154 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 28 Dec 2023 13:32:13 +0800 Subject: [PATCH 125/236] update test case to replace 'gdb' with 'local variables' --- .../test_hot_refresh_configurations.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 59222281ae..09e634974b 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -159,20 +159,21 @@ class TDTestCase: ] } - def get_param_value_with_gdb(self, config_name, process_name): - res = subprocess.Popen("gdb -q -nx -p `pidof {}` --batch -ex 'set height 0' -ex 'p {}'".format(process_name, config_name), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - r_lines = res.stdout.read() - if r_lines: - for line in r_lines.decode().split("\n"): - if "$1 = " in line: - tdLog.debug("gdb result: {}".format(line)) - return line.split(" = ")[1] + def get_param_value(self, config_name): + tdSql.query("show local variables;") + for row in tdSql.queryResult: + if config_name == row[0]: + tdLog.debug("Found variable '{}'".format(row[0])) + return row[1] def cli_check(self, name, values, except_values=False): if not except_values: for v in values: tdLog.debug("Set {} to {}".format(name, v)) tdSql.execute(f'alter local "{name} {v}";') + value = self.get_param_value(name) + tdLog.debug("Get {} value: {}".format(name, value)) + assert(v == int(value)) else: for v in values: tdLog.debug("Set {} to {}".format(name, v)) @@ -190,9 +191,7 @@ class TDTestCase: for v in values: dnode = random.choice(p_list) tdSql.execute(f'alter {dnode} "{name} {v}";') - if platform.system() == "Linux" and platform.machine() == "aarch64": - continue - value = self.get_param_value_with_gdb(alias, "taosd") + value = self.get_param_value(alias) if value: tdLog.debug(f"value: {value}") assert(value == str(bool(v)).lower() if is_bool else str(v)) From 51cd8e9fb77822374cff9b0583874638151d8828 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 28 Dec 2023 15:34:13 +0800 Subject: [PATCH 126/236] reset unclose window info --- source/libs/executor/inc/executorInt.h | 1 + source/libs/executor/src/executil.c | 7 ++- source/libs/executor/src/scanoperator.c | 1 - .../executor/src/streameventwindowoperator.c | 14 +++--- .../executor/src/streamtimewindowoperator.c | 44 ++++++++++--------- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e3e504cdbc..e98777544b 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -845,6 +845,7 @@ void resetWinRange(STimeWindow* winRange); bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts); int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval); void resetUnCloseSessionWinInfo(SSHashObj* winMap); +void setStreamOperatorCompleted(struct SOperatorInfo* pOperator); int32_t encodeSSessionKey(void** buf, SSessionKey* key); void* decodeSSessionKey(void* buf, SSessionKey* key); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 377de99fc0..825d716926 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2244,8 +2244,11 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr } void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr) { - if (!pBlock || pBlock->info.rows == 0) { - qDebug("%s===stream===%s: Block is Null or Empty", taskIdStr, flag); + if (!pBlock) { + qDebug("%s===stream===%s: Block is Null", taskIdStr, flag); + return; + } else if (pBlock->info.rows == 0) { + qDebug("%s===stream===%s: Block is Empty. type:%d", taskIdStr, flag, pBlock->info.type); return; } if (qDebugFlag & DEBUG_DEBUG) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..32407dcf71 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1997,7 +1997,6 @@ void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { int32_t len = streamScanOperatorEncode(pInfo, &pBuf); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); taosMemoryFree(pBuf); - pInfo->stateStore.streamStateCommit(pInfo->pState); } // other properties are recovered from the execution plan diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 6adab74344..3868edf645 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -480,18 +480,18 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { return resBlock; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + if (pInfo->reCkBlock) { pInfo->reCkBlock = false; printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - if (pInfo->recvGetAll) { - pInfo->recvGetAll = false; - resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); - } - - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -561,7 +561,7 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { if (resBlock != NULL) { return resBlock; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..a96d0ef633 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1223,7 +1223,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pCheckpointRes; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); if (!IS_FINAL_INTERVAL_OP(pOperator)) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer @@ -2609,18 +2609,18 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return opRes; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + if (pInfo->reCkBlock) { pInfo->reCkBlock = false; printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - if (pInfo->recvGetAll) { - pInfo->recvGetAll = false; - resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); - } - - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -2718,7 +2718,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return opRes; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3001,7 +3001,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi session operator clear disk buffer clearStreamSessionOperator(pInfo); - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); pInfo->clearState = false; return NULL; } @@ -3074,7 +3074,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi session operator clear disk buffer clearStreamSessionOperator(pInfo); - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3551,18 +3551,18 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return resBlock; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + if (pInfo->reCkBlock) { pInfo->reCkBlock = false; printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - if (pInfo->recvGetAll) { - pInfo->recvGetAll = false; - resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); - } - - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3628,7 +3628,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (resBlock != NULL) { return resBlock; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3864,11 +3864,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->reCkBlock) { pInfo->reCkBlock = false; - // printDataBlock(pInfo->pCheckpointRes, "single interval ck"); + printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3900,7 +3900,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - qDebug("===stream===%s recv|block type STREAM_GET_ALL", getStreamOpName(pOperator->operatorType)); pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; @@ -4083,3 +4082,8 @@ _error: pTaskInfo->code = code; return NULL; } + +void setStreamOperatorCompleted(SOperatorInfo* pOperator) { + setOperatorCompleted(pOperator); + qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); +} From 424ab1bbe32672570d4b8549b9b8f375e55bcbb5 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 16:18:38 +0800 Subject: [PATCH 127/236] support table merge scan --- source/libs/executor/inc/executorInt.h | 3 ++- source/libs/executor/src/scanoperator.c | 27 ++++++++++++++------ tests/system-test/2-query/group_partition.py | 17 +++++++++--- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index b87dee475d..ae38d4940c 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -307,7 +307,8 @@ typedef struct STableMergeScanInfo { SHashObj* mSkipTables; int64_t mergeLimit; SSortExecInfo sortExecInfo; - + bool needCountEmptyTable; + bool bGroupProcessed; // the group return data means processed bool filesetDelimited; bool bNewFilesetEvent; bool bNextDurationBlockEvent; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index dd6dff0301..a1f9baa082 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -700,17 +700,14 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); } -static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { - STableScanInfo* pTableScanInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - +static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, + const STableKeyInfo* tbInfo) { blockDataEmpty(pBlock); pBlock->info.rows = 1; pBlock->info.id.uid = tbInfo->uid; pBlock->info.id.groupId = tbInfo->groupId; - // only one row: set all col data to null & hasNull + // only one row: set all col data to null & hasNull int32_t col_num = blockDataGetNumOfCols(pBlock); for (int32_t i = 0; i < col_num; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); @@ -718,7 +715,14 @@ static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STable } // set tag/tbname - doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); + doSetTagColumnData(pBase, pBlock, pTaskInfo, pBlock->info.rows); + return pBlock; +} + +static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { + STableScanInfo* pTableScanInfo = pOperator->info; + SSDataBlock* pBlock = + getOneRowResultBlock(pOperator->pTaskInfo, &pTableScanInfo->base, pTableScanInfo->pResBlock, tbInfo); pOperator->resultInfo.totalRows++; return pBlock; @@ -3585,7 +3589,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { } pInfo->tableEndIndex = i - 1; } - + pInfo->bGroupProcessed = false; int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; @@ -3707,9 +3711,14 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity, pOperator); + if (pBlock == NULL && !pInfo->bGroupProcessed && pInfo->needCountEmptyTable) { + STableKeyInfo* tbInfo = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + pBlock = getOneRowResultBlock(pTaskInfo, &pInfo->base, pInfo->pResBlock, tbInfo); + } if (pBlock != NULL) { pBlock->info.id.groupId = pInfo->groupId; pOperator->resultInfo.totalRows += pBlock->info.rows; + pInfo->bGroupProcessed = true; return pBlock; } else { if (pInfo->bNewFilesetEvent) { @@ -3864,6 +3873,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN } else { pInfo->filesetDelimited = pTableScanNode->filesetDelimited; } + pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 8b06f0d6fd..e228351f0e 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -120,12 +120,21 @@ class TDTestCase: tdSql.checkRows(all_tb_num) # elapsed: continuous duration in a statistical period, table merge scan - tdSql.query(f" select count(c1), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") - tdSql.checkRows(nonempty_tb_num) - - tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.query(f" select count(c1), max(c5), last_row(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select count(c1), min(c1), avg(c1), elapsed(ts), mode(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select count(c1), elapsed(ts), twa(c1), irate(c1), leastsquares(c1,1,1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select avg(c1), elapsed(ts), twa(c1), irate(c1) from {self.dbname}.{self.stable} partition by tbname") tdSql.checkRows(nonempty_tb_num) + # if nonempty_tb_num > 0: + # tdSql.query(f" select avg(c1), percentile(c1, 50) from {self.dbname}.sub_{self.stable}_1") + # tdSql.checkRows(1) def test_innerSelect(self, check_num): tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") From 53ce344b362b10d0e7afd5f49d06893ee9b10f15 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 28 Dec 2023 17:09:52 +0800 Subject: [PATCH 128/236] fix:set error msg if build consumer error --- source/client/src/clientTmq.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index d05cdc0156..15c8903978 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1028,11 +1028,16 @@ static void tmqMgmtInit(void) { } } +#define SET_ERROR_MSG(MSG) if(errstr!=NULL)snprintf(errstr,errstrLen,MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - if(conf == NULL) return NULL; + if(conf == NULL) { + SET_ERROR_MSG("configure is null") + return NULL; + } taosThreadOnce(&tmqInit, tmqMgmtInit); if (tmqInitRes != 0) { terrno = tmqInitRes; + SET_ERROR_MSG("tmq timer init error") return NULL; } @@ -1040,6 +1045,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr()); + SET_ERROR_MSG("malloc tmq failed") return NULL; } @@ -1055,6 +1061,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG("malloc tmq element failed or group is empty") goto _failed; } @@ -1086,6 +1093,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (tsem_init(&pTmq->rspSem, 0, 0) != 0) { tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG("init t_sem failed") goto _failed; } @@ -1094,11 +1102,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq->pTscObj == NULL) { tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); tsem_destroy(&pTmq->rspSem); + SET_ERROR_MSG("init tscObj failed") goto _failed; } pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq); if (pTmq->refId < 0) { + SET_ERROR_MSG("add tscObj ref failed") goto _failed; } From 2526df745ee62db8786adcd732e8b0cd4646a723 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:15:09 +0800 Subject: [PATCH 129/236] enh: adjust threshold of snap replication timeout --- include/libs/sync/sync.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e54237fe8b..cb053d2548 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -46,8 +46,8 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 -#define SYNC_SNAP_RESEND_MS 1000 * 300 -#define SYNC_SNAP_TIMEOUT_MS 1000 * 1800 +#define SYNC_SNAP_RESEND_MS 1000 * 60 +#define SYNC_SNAP_TIMEOUT_MS 1000 * 300 #define SYNC_VND_COMMIT_MIN_MS 3000 From d83f85bf2642982d8fdcb5bd53b58011cc8f81e1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:26:46 +0800 Subject: [PATCH 130/236] enh: send rsp msg on rejecting snap replication due to smaller term --- source/libs/sync/src/syncSnapshot.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 10a8734617..f0e457ef8d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1001,6 +1001,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { sRError(pReceiver, "reject snap replication with smaller term. msg term:%" PRId64 ", seq:%d", pMsg->term, pMsg->seq); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno); return -1; } From baa7f9c895faf0ed12d59c5ff02460ae8eb3f1da Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:29:49 +0800 Subject: [PATCH 131/236] enh: adjust error msg as warn for not ready to propose etc --- source/dnode/vnode/src/vnd/vnodeSync.c | 8 ++++---- source/libs/sync/src/syncMain.c | 6 +++--- source/libs/sync/src/syncPipeline.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 048092131d..0f491e9a58 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -220,8 +220,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) isWeak, isBlock, msg, numOfMsgs, arrayPos, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + vGWarn("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, + TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_SYN_RESTORING; vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); @@ -284,8 +284,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + vGWarn("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, + TMSG_INFO(pMsg->msgType)); vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 6f3b3fdf98..52557f7b9c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -662,14 +662,14 @@ ESyncRole syncGetRole(int64_t rid) { int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { terrno = TSDB_CODE_SYN_NOT_LEADER; - sNError(pSyncNode, "sync propose not leader, type:%s", TMSG_INFO(pMsg->msgType)); + sNWarn(pSyncNode, "sync propose not leader, type:%s", TMSG_INFO(pMsg->msgType)); return -1; } if (!pSyncNode->restoreFinish) { terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; - sNError(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, - TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); + sNWarn(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, + TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); return -1; } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 28ee5ba841..70bdd4a837 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -206,7 +206,7 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { } if (pLogStore->syncLogGetEntry(pLogStore, index, &pEntry) < 0) { - sError("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); + sWarn("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); break; } @@ -1237,7 +1237,7 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, } else { *pInBuf = false; if (pNode->pLogStore->syncLogGetEntry(pNode->pLogStore, index, &pEntry) < 0) { - sError("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); + sWarn("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); } } return pEntry; @@ -1253,7 +1253,7 @@ int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex ind pEntry = syncLogBufferGetOneEntry(pBuf, pNode, index, &inBuf); if (pEntry == NULL) { - sError("vgId:%d, failed to get raft entry for index:%" PRId64 "", pNode->vgId, index); + sWarn("vgId:%d, failed to get raft entry for index:%" PRId64 "", pNode->vgId, index); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); if (pMgr) { From ef22830ca39a70be588db086c6d769d49159e444 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 18:01:30 +0800 Subject: [PATCH 132/236] adjust --- tests/system-test/2-query/irate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index d976edb49c..d882c9b8fb 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -209,7 +209,7 @@ class TDTestCase: tdSql.error(f"select irate(c1), abs(c1) from {dbname}.ct4 ") # agg functions mix with agg functions - tdSql.query(f"select irate(c1), count(c5) from {dbname}.stb1 partition by tbname order by tbname") + tdSql.query(f"select irate(c1), count(c5) from {dbname}.stb1 partition by tbname having count(c5)>0 order by tbname") tdSql.checkData(0, 0, 0.000000000) tdSql.checkData(1, 0, 0.000000000) tdSql.checkData(0, 1, 13) From 9052ca6eded8c879f3c6a9728ce308e3bd2eac15 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 28 Dec 2023 18:05:28 +0800 Subject: [PATCH 133/236] add test case for ts-4403 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/test_ts4403.py | 77 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tests/system-test/2-query/test_ts4403.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e7c26b9833..a04a9f6d0a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -519,6 +519,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py diff --git a/tests/system-test/2-query/test_ts4403.py b/tests/system-test/2-query/test_ts4403.py new file mode 100644 index 0000000000..7f0561d084 --- /dev/null +++ b/tests/system-test/2-query/test_ts4403.py @@ -0,0 +1,77 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TS-4403 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.stbname = 'st' + + def prepareData(self): + # db + tdSql.execute("create database {};".format(self.dbname)) + tdSql.execute("use {};".format(self.dbname)) + tdLog.debug("Create database %s" % self.dbname) + + # super table + tdSql.execute("create table {} (ts timestamp, col1 int) tags (t1 int, t2 binary(16), t3 nchar(16));".format(self.stbname)) + tdLog.debug("Create super table %s" % self.stbname) + + # create index for all tags + tdSql.execute("create index t2_st on {} (t2);".format(self.stbname)) + tdSql.execute("create index t3_st on {} (t3);".format(self.stbname)) + + def run(self): + self.prepareData() + # check index number + tdSql.query("show indexes from {};".format(self.stbname)) + assert(3 == len(tdSql.queryResult)) + tdLog.debug("All tags of super table have index successfully") + + # drop default first tag index + tdSql.execute("drop index t1_st;") + tdSql.query("show indexes from {};".format(self.stbname)) + assert(2 == len(tdSql.queryResult)) + tdLog.debug("Delete the default index of first tag successfully") + + # create index for first tag + tdSql.execute("create index t1_st on {} (t1);".format(self.stbname)) + tdSql.query("show indexes from {};".format(self.stbname)) + assert(3 == len(tdSql.queryResult)) + tdLog.debug("Create index for first tag successfully") + + # null as index value to create child table + tdSql.execute("create table ct1 using {} tags(null, null, null);".format(self.stbname)) + tdSql.query("show tables;") + assert(1 == len(tdSql.queryResult)) + tdLog.debug("Create child table with tags value as 'null' successfully") + + # redundant index with different index name for some tag + tdSql.error("create index t2_ct1 on st (t2);") + tdLog.debug("Verify redundant index with different index name for some tag successfully") + + # same index name for different tag + tdSql.error("create index t2_st on st (t3);") + tdLog.debug("Verify same index name for some tag successfully") + + # add index for multiple tags(TD-28078) + tdSql.error("create index tt on {} (t2, t3);".format(self.stbname)) + tdLog.debug("Verify add index for multiple tags successfully") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 9f8f69a791f9544e11475af4f0f7c0fa04663b56 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 21:54:16 +0800 Subject: [PATCH 134/236] adjust --- source/libs/executor/src/scanoperator.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a1f9baa082..7b2df4e206 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -670,15 +670,11 @@ static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKey return TSDB_CODE_OUT_OF_MEMORY; } } - uint64_t groupId = 0; + uint64_t groupId = pList->groupId; for (int32_t i = 0; i < num; i++) { const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isSameGroup) { - if (i == 0) { - groupId = pInfo->groupId; - } else if (groupId != pInfo->groupId) { - pTableScanInfo->isSameGroup = false; - } + if (pTableScanInfo->isSameGroup && groupId != pInfo->groupId) { + pTableScanInfo->isSameGroup = false; } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); } From e2b61a55fc13aee349cb63abb3c7d003c1de783f Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 29 Dec 2023 01:40:17 +0800 Subject: [PATCH 135/236] refactor --- source/libs/executor/inc/executorInt.h | 6 +- source/libs/executor/src/executor.c | 12 +-- source/libs/executor/src/scanoperator.c | 129 ++++++++---------------- 3 files changed, 52 insertions(+), 95 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ae38d4940c..9acef69f9c 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -268,14 +268,12 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SHashObj* pRemainTables; // remain table to process SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state - bool isSameGroup; // whether all tables are in the same group this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6cee79bff2..fb39de484f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1209,7 +1209,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0); uid = pTableInfo->uid; ts = INT64_MIN; - pScanInfo->currentTable = 0; + pScanInfo->tableEndIndex = 0; } else { taosRUnLockLatch(&pTaskInfo->lock); qError("no table in table list, %s", id); @@ -1223,16 +1223,16 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pInfo->pTableScanOp->resultInfo.totalRows = 0; // start from current accessed position - // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start + // we cannot start from the pScanInfo->tableEndIndex, since the commit offset may cause the rollback of the start // position, let's find it from the beginning. index = tableListFind(pTableListInfo, uid, 0); taosRUnLockLatch(&pTaskInfo->lock); if (index >= 0) { - pScanInfo->currentTable = index; + pScanInfo->tableEndIndex = index; } else { qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, - numOfTables, pScanInfo->currentTable, id); + numOfTables, pScanInfo->tableEndIndex, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1255,12 +1255,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); } else { pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1); pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); } // restore the key value diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7b2df4e206..e990d3d975 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,45 +655,29 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } -static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { - if (!pTableScanInfo->needCountEmptyTable) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return TSDB_CODE_SUCCESS; - } - pTableScanInfo->isSameGroup = true; - if (NULL == pTableScanInfo->pRemainTables) { - int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); - pTableScanInfo->pRemainTables = - taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (NULL == pTableScanInfo->pRemainTables) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return TSDB_CODE_OUT_OF_MEMORY; - } - } - uint64_t groupId = pList->groupId; - for (int32_t i = 0; i < num; i++) { - const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isSameGroup && groupId != pInfo->groupId) { - pTableScanInfo->isSameGroup = false; - } - taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); - } - pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; - return TSDB_CODE_SUCCESS; -} -static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { - // case0 group scanning, mark - // case1 stream scan: no need to mark - if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { - return; +static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) { + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + int32_t i = pInfo->tableStartIndex + 1; + for (; i < numOfTables; ++i) { + STableKeyInfo* pCur = tableListGetInfo(pInfo->base.pTableListInfo, i); + if (pCur->groupId != pStart->groupId) { + break; + } } - // case2 if all table in same group, process only once - if (pTableScanInfo->isSameGroup) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return; + + pInfo->tableEndIndex = i - 1; + if (!pInfo->needCountEmptyTable) { + pInfo->countState = TABLE_COUNT_STATE_END; + } else { + pInfo->countState = TABLE_COUNT_STATE_SCAN; } - taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); + + *pKeyInfo = pStart; + *size = i - pInfo->tableStartIndex; } static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, @@ -791,7 +775,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKeyInfo* pList, int32_t num) { +static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -801,15 +785,11 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey return NULL; } - if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { - initTableCountEnv(pTableScanInfo, pList, num); - } - // do the ascending order traverse in the first place. while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - markTableProcessed(pTableScanInfo, p->info.id.uid); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return p; } @@ -838,7 +818,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - markTableProcessed(pTableScanInfo, p->info.id.uid); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return p; } @@ -856,29 +836,13 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { - int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); - if (tb_cnt) { - if (!pTableScanInfo->isSameGroup) { - // get first empty table uid, mark processed & rm from hash - void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); - if (pIte != NULL) { - size_t keySize = 0; - uint64_t* pUid = taosHashGetKey(pIte, &keySize); - STableKeyInfo info = {.uid = *pUid, .groupId = *(uint64_t*)pIte}; - taosHashCancelIterate(pTableScanInfo->pRemainTables, pIte); - markTableProcessed(pTableScanInfo, *pUid); - return getBlockForEmptyTable(pOperator, &info); - } - } else { - // output one table for this group - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return getBlockForEmptyTable(pOperator, pList); - } - } + // output once for this group pTableScanInfo->countState = TABLE_COUNT_STATE_END; + STableKeyInfo* pStart = + (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); + return getBlockForEmptyTable(pOperator, pStart); } - taosHashClear(pTableScanInfo->pRemainTables); return NULL; } @@ -938,8 +902,8 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + if (pInfo->tableEndIndex + 1 >= numOfTables) { setOperatorCompleted(pOperator); if (pOperator->dynamicTask) { taosArrayClear(pInfo->base.pTableListInfo->pTableList); @@ -954,14 +918,13 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pInfo->countState = TABLE_COUNT_STATE_NONE; + initNextGroupScan(pInfo, &pList, &num); pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; - SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -979,15 +942,14 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - if (pInfo->currentGroupId == -1) { + if (pInfo->tableEndIndex == -1) { int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo) || numOfTables == 0) { + if (pInfo->tableEndIndex + 1 == numOfTables) { setOperatorCompleted(pOperator); return NULL; } - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pInfo->countState = TABLE_COUNT_STATE_NONE; + initNextGroupScan(pInfo, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -1001,11 +963,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } - } else { - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); } - SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -1038,7 +998,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } if (pOperator->status == OP_EXEC_DONE) { - pInfo->currentGroupId = -1; + pInfo->tableEndIndex = -1; pOperator->status = OP_OPENED; SSDataBlock* result = NULL; while (true) { @@ -1057,29 +1017,29 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->countState = TABLE_COUNT_STATE_END; while (1) { - SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } // if no data, switch to next table and continue scan - pInfo->currentTable++; + pInfo->tableEndIndex++; taosRLockLatch(&pTaskInfo->lock); numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if (pInfo->currentTable >= numOfTables) { + if (pInfo->tableEndIndex >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); taosRUnLockLatch(&pTaskInfo->lock); return NULL; } - tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); + tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableEndIndex); taosRUnLockLatch(&pTaskInfo->lock); pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, - pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); + pInfo->tableEndIndex, numOfTables, GET_TASKID(pTaskInfo)); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -1117,7 +1077,6 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - taosHashCleanup(pTableScanInfo->pRemainTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1173,7 +1132,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } - pInfo->currentGroupId = -1; + pInfo->tableEndIndex = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; @@ -1268,7 +1227,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; - pTableScanInfo->currentGroupId = -1; + pTableScanInfo->tableEndIndex = -1; pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -2170,7 +2129,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pTableScanOp->status = OP_OPENED; pTSInfo->scanTimes = 0; - pTSInfo->currentGroupId = -1; + pTSInfo->tableEndIndex = -1; } if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { From 0a0f1a9d22986c5140a5adda57475455b69a43df Mon Sep 17 00:00:00 2001 From: fullhonest Date: Tue, 26 Dec 2023 19:51:14 +0800 Subject: [PATCH 136/236] Fix TD-27989 --- source/libs/function/inc/tpercentile.h | 4 +-- source/libs/function/src/tpercentile.c | 46 +++++++++++++------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 65b7b38a05..5351594b2f 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -26,12 +26,12 @@ extern "C" { typedef struct MinMaxEntry { union { double dMinVal; - int64_t i64MinVal; + //double i64MinVal; uint64_t u64MinVal; }; union { double dMaxVal; - int64_t i64MaxVal; + //double i64MaxVal; int64_t u64MaxVal; }; } MinMaxEntry; diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 93008b565a..c671e7717c 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -63,8 +63,8 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) static void resetBoundingBox(MinMaxEntry *range, int32_t type) { if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MaxVal = INT64_MIN; - range->i64MinVal = INT64_MAX; + range->dMaxVal = INT64_MIN; + range->dMinVal = INT64_MAX; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { range->u64MaxVal = 0; range->u64MinVal = UINT64_MAX; @@ -80,8 +80,8 @@ static int32_t setBoundingBox(MinMaxEntry *range, int16_t type, double minval, d } if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MinVal = (int64_t)minval; - range->i64MaxVal = (int64_t)maxval; + range->dMinVal = (int64_t)minval; + range->dMaxVal = (int64_t)maxval; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { range->u64MinVal = (uint64_t)minval; range->u64MaxVal = (uint64_t)maxval; @@ -137,21 +137,21 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { int32_t index = -1; - if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) { + if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { return index; } // divide the value range into 1024 buckets - uint64_t span = (uint64_t)(pBucket->range.i64MaxVal - pBucket->range.i64MinVal); + uint64_t span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { - int64_t delta = v - pBucket->range.i64MinVal; + int64_t delta = v - pBucket->range.dMinVal; index = (delta % pBucket->numOfSlots); } else { double slotSpan = ((double)span) / pBucket->numOfSlots; - uint64_t delta = (uint64_t)(v - pBucket->range.i64MinVal); + uint64_t delta = (uint64_t)(v - pBucket->range.dMinVal); - index = (int32_t)(delta / slotSpan); - if (v == pBucket->range.i64MaxVal || index == pBucket->numOfSlots) { + index = delta / slotSpan; + if (v == pBucket->range.dMaxVal || index == pBucket->numOfSlots) { index -= 1; } } @@ -318,23 +318,23 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT int64_t v = 0; GET_TYPED_DATA(v, int64_t, dataType, data); - if (r->i64MinVal > v) { - r->i64MinVal = v; + if (r->dMinVal > v) { + r->dMinVal = v; } - if (r->i64MaxVal < v) { - r->i64MaxVal = v; + if (r->dMaxVal < v) { + r->dMaxVal = v; } } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { uint64_t v = 0; GET_TYPED_DATA(v, uint64_t, dataType, data); - if (r->i64MinVal > v) { - r->i64MinVal = v; + if (r->u64MinVal > v) { + r->u64MinVal = v; } - if (r->i64MaxVal < v) { - r->i64MaxVal = v; + if (r->u64MaxVal < v) { + r->u64MaxVal = v; } } else if (IS_FLOAT_TYPE(dataType)) { double v = 0; @@ -438,7 +438,7 @@ static double getIdenticalDataVal(tMemBucket *pMemBucket, int32_t slotIndex) { double finalResult = 0.0; if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - finalResult = (double)pSlot->range.i64MinVal; + finalResult = (double)pSlot->range.dMinVal; } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { finalResult = (double)pSlot->range.u64MinVal; } else { @@ -469,8 +469,8 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction double maxOfThisSlot = 0; double minOfNextSlot = 0; if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - maxOfThisSlot = (double)pSlot->range.i64MaxVal; - minOfNextSlot = (double)next.i64MinVal; + maxOfThisSlot = (double)pSlot->range.dMaxVal; + minOfNextSlot = (double)next.dMinVal; } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { maxOfThisSlot = (double)pSlot->range.u64MaxVal; minOfNextSlot = (double)next.u64MinVal; @@ -577,7 +577,7 @@ int32_t getPercentile(tMemBucket *pMemBucket, double percent, double *result) { MinMaxEntry *pRange = &pMemBucket->range; if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); + *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->dMaxVal : pRange->dMinVal); } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal); } else { @@ -603,6 +603,6 @@ bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) { if (IS_FLOAT_TYPE(pMemBucket->type)) { return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON; } else { - return pSeg->range.i64MinVal == pSeg->range.i64MaxVal; + return pSeg->range.dMinVal == pSeg->range.dMaxVal; } } From 68cb86602eff35725e39592c054ab22954ee9cb6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 29 Dec 2023 11:37:34 +0800 Subject: [PATCH 137/236] fix: reuse cid with fs eid updated in tsdbFSetRAWWriteFileDataBegin --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbFS2.h | 1 + source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 6 ++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index b2ccc3e859..1d2fd60df2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -805,6 +805,12 @@ int64_t tsdbFSAllocEid(STFileSystem *fs) { return cid; } +void tsdbFSUpdateEid(STFileSystem *fs, int64_t cid) { + taosThreadMutexLock(&fs->tsdb->mutex); + fs->neid = TMAX(fs->neid, cid); + taosThreadMutexUnlock(&fs->tsdb->mutex); +} + int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { int32_t code = 0; int32_t lino; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 714bf5bf16..7099ce8b26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -52,6 +52,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); // txn int64_t tsdbFSAllocEid(STFileSystem *fs); +void tsdbFSUpdateEid(STFileSystem *fs, int64_t cid); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); int32_t tsdbFSEditCommit(STFileSystem *fs); int32_t tsdbFSEditAbort(STFileSystem *fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index 03c12502d5..ddb8e7e27e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -14,6 +14,7 @@ */ #include "tsdbFSetRAW.h" +#include "tsdbFS2.h" // SFSetRAWWriter ================================================== typedef struct SFSetRAWWriter { @@ -76,7 +77,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .szPage = writer->config->szPage, .fid = bHdr->file.fid, .did = writer->config->did, - .cid = writer->config->cid, + .cid = bHdr->file.cid, .level = writer->config->level, .file = @@ -84,7 +85,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .type = bHdr->file.type, .fid = bHdr->file.fid, .did = writer->config->did, - .cid = writer->config->cid, + .cid = bHdr->file.cid, .size = bHdr->file.size, .minVer = bHdr->file.minVer, .maxVer = bHdr->file.maxVer, @@ -94,6 +95,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA }, }; + tsdbFSUpdateEid(config.tsdb->pFS, config.cid); writer->ctx->offset = 0; writer->ctx->file = config.file; From b58a23df49b034490521d28faa51548caa1b2aba Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Dec 2023 13:57:13 +0800 Subject: [PATCH 138/236] fix:[TS-4391] rebalance cnt always 1 if msg lost --- include/common/tmsg.h | 6 - source/dnode/mnode/impl/inc/mndConsumer.h | 5 +- source/dnode/mnode/impl/src/mndConsumer.c | 184 +-------------------- source/dnode/mnode/impl/src/mndSubscribe.c | 170 +++++++++++++++---- 4 files changed, 151 insertions(+), 214 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bafe7583b7..c314d82036 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2544,12 +2544,6 @@ _err: return NULL; } -// this message is sent from mnode to mnode(read thread to write thread), -// so there is no need for serialization or deserialization -typedef struct { - SHashObj* rebSubHash; // SHashObj -} SMqDoRebalanceMsg; - typedef struct { int64_t streamId; int64_t checkpointId; diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index f075510428..59a22b76cd 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -46,9 +46,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); int32_t mndSetConsumerDropLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); -bool mndRebTryStart(); +const char *mndConsumerStatusName(int status); + bool mndRebCanStart(); -void mndRebEnd(); +bool mndRebTryStart(); void mndRebCntInc(); void mndRebCntDec(); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 11c929c898..5987f0ec34 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -29,13 +29,9 @@ #define MND_CONSUMER_RESERVE_SIZE 64 #define MND_MAX_GROUP_PER_TOPIC 100 -#define MND_CONSUMER_LOST_HB_CNT 6 -#define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200 static int32_t mqRebInExecCnt = 0; -static const char *mndConsumerStatusName(int status); - static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer); @@ -45,7 +41,6 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg); static int32_t mndProcessAskEpReq(SRpcMsg *pMsg); static int32_t mndProcessMqHbReq(SRpcMsg *pMsg); -static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg); @@ -63,7 +58,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TMQ_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); - mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); +// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg); @@ -97,42 +92,18 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* bool mndRebTryStart() { int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - mDebug("tq timer, rebalance counter old val:%d", old); + mInfo("rebalance counter old val:%d", old); return old == 0; } -bool mndRebCanStart() { - int32_t val = atomic_load_32(&mqRebInExecCnt); - if (val < 0) { - mError("rebalance trans end, rebalance counter:%d should not be less equalled than 0, ignore counter desc", val); - return false; - } - mInfo("tq timer, rebalance counter val:%d", val); - return val == 0; -} - -void mndRebEnd() { mndRebCntDec(); } - void mndRebCntInc() { int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1); - mInfo("rebalance trans start, rebalance counter:%d", val); + mInfo("rebalance cnt inc, value:%d", val); } void mndRebCntDec() { - while (1) { - int32_t val = atomic_load_32(&mqRebInExecCnt); - if (val <= 0) { - mError("rebalance trans end, rebalance counter:%d should not be less equalled than 0, ignore counter desc", val); - break; - } - - int32_t newVal = val - 1; - int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal); - if (oldVal == val) { - mInfo("rebalance trans end, rebalance counter:%d", newVal); - break; - } - } + int32_t val = atomic_sub_fetch_32(&mqRebInExecCnt, 1); + mInfo("rebalance cnt sub, value:%d", val); } static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { @@ -282,149 +253,6 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebInfo; } -static void freeRebalanceItem(void *param) { - SMqRebInfo *pInfo = param; - taosArrayDestroy(pInfo->newConsumers); - taosArrayDestroy(pInfo->removedConsumers); -} - -static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SSdb *pSdb = pMnode->pSdb; - SMqConsumerObj *pConsumer; - void *pIter = NULL; - - mInfo("start to process mq timer"); - - // rebalance cannot be parallel - if (!mndRebCanStart()) { - mInfo("mq rebalance already in progress, do nothing"); - return 0; - } - - SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); - if (pRebMsg == NULL) { - mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t)sizeof(SMqDoRebalanceMsg)); - return TSDB_CODE_OUT_OF_MEMORY; - } - - pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); - if (pRebMsg->rebSubHash == NULL) { - mError("failed to create rebalance hashmap"); - rpcFreeCont(pRebMsg); - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosHashSetFreeFp(pRebMsg->rebSubHash, freeRebalanceItem); - - // iterate all consumers, find all modification - while (1) { - pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); - if (pIter == NULL) { - break; - } - - int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); - int32_t status = atomic_load_32(&pConsumer->status); - - mInfo("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", - pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, - hbStatus); - - if (status == MQ_CONSUMER_STATUS_READY) { - if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close - mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); - } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { - taosRLockLatch(&pConsumer->lock); - int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); - for (int32_t i = 0; i < topicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); - mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); - } - taosRUnLockLatch(&pConsumer->lock); - }else{ - int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); - for (int32_t i = 0; i < newTopicNum; i++) { - char * topic = taosArrayGetP(pConsumer->currentTopics, i); - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); - if (pSub == NULL) { - continue; - } - taosRLockLatch(&pSub->lock); - - // 2.2 iterate all vg assigned to the consumer of that topic - SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); - int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); - - for (int32_t j = 0; j < vgNum; j++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - SVgObj * pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); - if (!pVgroup) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, pConsumer->cgroup, topic); - mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); - } - mndReleaseVgroup(pMnode, pVgroup); - } - taosRUnLockLatch(&pSub->lock); - mndReleaseSubscribe(pMnode, pSub); - } - } - } else if (status == MQ_CONSUMER_STATUS_LOST) { - if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { // clear consumer if lost a day - mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); - } - } else { // MQ_CONSUMER_STATUS_REBALANCE - taosRLockLatch(&pConsumer->lock); - - int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); - for (int32_t i = 0; i < newTopicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); - mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); - } - - int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics); - for (int32_t i = 0; i < removedTopicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); - mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); - } - - if (newTopicNum == 0 && removedTopicNum == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close - mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); - } - - taosRUnLockLatch(&pConsumer->lock); - } - - mndReleaseConsumer(pMnode, pConsumer); - } - - if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq send msg to rebalance"); - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_DO_REBALANCE, - .pCont = pRebMsg, - .contLen = sizeof(SMqDoRebalanceMsg), - }; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - } else { - taosHashCleanup(pRebMsg->rebSubHash); - rpcFreeCont(pRebMsg); - mInfo("mq timer finished, no need to re-balance"); - } - return 0; -} - static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t code = 0; SMnode *pMnode = pMsg->info.node; @@ -1263,7 +1091,7 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -static const char *mndConsumerStatusName(int status) { +const char *mndConsumerStatusName(int status) { switch (status) { case MQ_CONSUMER_STATUS_READY: return "ready"; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8cd7f6bc9c..b2370374b7 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -27,7 +27,8 @@ #define MND_SUBSCRIBE_VER_NUMBER 2 #define MND_SUBSCRIBE_RESERVE_SIZE 64 -#define MND_SUBSCRIBE_REBALANCE_CNT 3 +#define MND_CONSUMER_LOST_HB_CNT 6 +#define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200 static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); @@ -38,14 +39,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); - -static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { - SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; - return 0; -} +static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* hash); static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); @@ -68,7 +62,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_TMQ_SUBSCRIBE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_TMQ_DELETE_SUB_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DO_REBALANCE, mndProcessRebalanceReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP, mndProcessDropCgroupReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP_RSP, mndTransProcessRsp); @@ -727,21 +721,140 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu return 0; } -static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SMqDoRebalanceMsg *pReq = pMsg->pCont; - void *pIter = NULL; -// bool rebalanceOnce = false; // to ensure only once. - if (!mndRebTryStart()) { - mInfo("mq rebalance already in progress, do nothing"); - return 0; +static void freeRebalanceItem(void *param) { + SMqRebInfo *pInfo = param; + taosArrayDestroy(pInfo->newConsumers); + taosArrayDestroy(pInfo->removedConsumers); +} + +static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { + SMnode *pMnode = pMsg->info.node; + SSdb *pSdb = pMnode->pSdb; + SMqConsumerObj *pConsumer; + void *pIter = NULL; + + mInfo("start to process mq timer"); + + // iterate all consumers, find all modification + while (1) { + pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) { + break; + } + + int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); + int32_t status = atomic_load_32(&pConsumer->status); + + mInfo("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", + pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, + hbStatus); + + if (status == MQ_CONSUMER_STATUS_READY) { + if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { + taosRLockLatch(&pConsumer->lock); + int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < topicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + taosRUnLockLatch(&pConsumer->lock); + }else{ + int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < newTopicNum; i++) { + char * topic = taosArrayGetP(pConsumer->currentTopics, i); + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); + if (pSub == NULL) { + continue; + } + taosRLockLatch(&pSub->lock); + + // 2.2 iterate all vg assigned to the consumer of that topic + SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); + int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); + + for (int32_t j = 0; j < vgNum; j++) { + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); + SVgObj * pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); + if (!pVgroup) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + mndMakeSubscribeKey(key, pConsumer->cgroup, topic); + mndGetOrCreateRebSub(rebSubHash, key); + mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); + } + mndReleaseVgroup(pMnode, pVgroup); + } + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } + } + } else if (status == MQ_CONSUMER_STATUS_LOST) { + if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { // clear consumer if lost a day + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } + } else { + taosRLockLatch(&pConsumer->lock); + + int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < newTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); + taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); + } + + int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < removedTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + + if (newTopicNum == 0 && removedTopicNum == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } + + taosRUnLockLatch(&pConsumer->lock); + } + + mndReleaseConsumer(pMnode, pConsumer); } - mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); + return 0; +} + +static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { + int code = 0; + if (!mndRebTryStart()) { + mInfo("mq rebalance already in progress, do nothing"); + return code; + } + + SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + if (rebSubHash == NULL) { + mError("failed to create rebalance hashmap"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto END; + } + + taosHashSetFreeFp(rebSubHash, freeRebalanceItem); + + mndCheckConsumer(pMsg, rebSubHash); + mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(rebSubHash)); // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. + void *pIter = NULL; + SMnode *pMnode = pMsg->info.node; while (1) { - pIter = taosHashIterate(pReq->rebSubHash, pIter); + pIter = taosHashIterate(rebSubHash, pIter); if (pIter == NULL) { break; } @@ -760,12 +873,11 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { taosArrayDestroy(rebOutput.modifyConsumers); taosArrayDestroy(rebOutput.rebVgs); - taosHashCancelIterate(pReq->rebSubHash, pIter); + taosHashCancelIterate(rebSubHash, pIter); terrno = TSDB_CODE_OUT_OF_MEMORY; - mInfo("mq re-balance failed, due to out of memory"); - taosHashCleanup(pReq->rebSubHash); - mndRebEnd(); - return -1; + mError("mq re-balance failed, due to out of memory"); + code = -1; + goto END; } SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter; @@ -833,10 +945,12 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // reset flag mInfo("mq re-balance completed successfully"); - taosHashCleanup(pReq->rebSubHash); - mndRebEnd(); - return 0; +END: + taosHashCleanup(rebSubHash); + mndRebCntDec(); + + return code; } static int32_t sendDeleteSubToVnode(SMqSubscribeObj *pSub, STrans *pTrans){ From c83989754f448500aa91e4c486454a57f46c2dd9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Dec 2023 14:09:49 +0800 Subject: [PATCH 139/236] fix:[TS-4391] rebalance cnt always 1 if msg lost --- source/dnode/mnode/impl/inc/mndConsumer.h | 5 ----- source/dnode/mnode/impl/inc/mndSubscribe.h | 9 ++++----- source/dnode/mnode/impl/src/mndConsumer.c | 18 ------------------ source/dnode/mnode/impl/src/mndSubscribe.c | 18 ++++++++++++++++++ source/dnode/mnode/impl/src/mndTrans.c | 2 +- 5 files changed, 23 insertions(+), 29 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 59a22b76cd..8c89ddc825 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -48,11 +48,6 @@ int32_t mndSetConsumerDropLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *p const char *mndConsumerStatusName(int status); -bool mndRebCanStart(); -bool mndRebTryStart(); -void mndRebCntInc(); -void mndRebCntDec(); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index 10864da5fb..23b3a7d1fe 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -32,14 +32,13 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); -//static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { -// return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); -//} - -//int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic); int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub); +bool mndRebTryStart(); +void mndRebCntInc(); +void mndRebCntDec(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5987f0ec34..cf8b9e019a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -30,8 +30,6 @@ #define MND_MAX_GROUP_PER_TOPIC 100 -static int32_t mqRebInExecCnt = 0; - static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer); @@ -90,22 +88,6 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* return; } -bool mndRebTryStart() { - int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - mInfo("rebalance counter old val:%d", old); - return old == 0; -} - -void mndRebCntInc() { - int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1); - mInfo("rebalance cnt inc, value:%d", val); -} - -void mndRebCntDec() { - int32_t val = atomic_sub_fetch_32(&mqRebInExecCnt, 1); - mInfo("rebalance cnt sub, value:%d", val); -} - static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { SMqTopicObj *pTopic = NULL; int32_t code = 0; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b2370374b7..b7958c1484 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -30,6 +30,8 @@ #define MND_CONSUMER_LOST_HB_CNT 6 #define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200 +static int32_t mqRebInExecCnt = 0; + static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); @@ -830,6 +832,22 @@ static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { return 0; } +bool mndRebTryStart() { + int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); + mInfo("rebalance counter old val:%d", old); + return old == 0; +} + +void mndRebCntInc() { + int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1); + mInfo("rebalance cnt inc, value:%d", val); +} + +void mndRebCntDec() { + int32_t val = atomic_sub_fetch_32(&mqRebInExecCnt, 1); + mInfo("rebalance cnt sub, value:%d", val); +} + static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { int code = 0; if (!mndRebTryStart()) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9e478f3aa5..99553fc57a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndConsumer.h" +#include "mndSubscribe.h" #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" From 90a8fe558196d58198221a46b1c08f0e27203bf3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 29 Dec 2023 15:08:56 +0800 Subject: [PATCH 140/236] fix: fill prev not working when desc fill --- source/libs/executor/src/tfill.c | 13 +++-- tests/system-test/2-query/fill_with_group.py | 51 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 6c537d7b98..7b63bc8720 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -169,13 +169,14 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* setNotFillColumn(pFillInfo, pDstCol, index, i); } } else { - SGroupKeys* pKey = taosArrayGet(pFillInfo->prev.pRowVal, i); + SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + SGroupKeys* pKey = taosArrayGet(pRVal->pRowVal, i); if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { colDataSetNULL(pDstCol, index); continue; } - SGroupKeys* pKey1 = taosArrayGet(pFillInfo->prev.pRowVal, pFillInfo->tsSlotId); + SGroupKeys* pKey1 = taosArrayGet(pRVal->pRowVal, pFillInfo->tsSlotId); int64_t prevTs = *(int64_t*)pKey1->pData; int32_t srcSlotId = GET_DEST_SLOT_ID(pCol); @@ -346,9 +347,10 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t char* src = colDataGetData(pSrc, pFillInfo->index); if (!colDataIsNull_s(pSrc, pFillInfo->index)) { colDataSetVal(pDst, index, src, false); - saveColData(pFillInfo->prev.pRowVal, i, src, false); + SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + saveColData(pRVal->pRowVal, i, src, false); if (pFillInfo->srcTsSlotId == dstSlotId) { - pFillInfo->prev.key = *(int64_t*)src; + pRVal->key = *(int64_t*)src; } } else { // the value is null if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -361,7 +363,8 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t } else if (pFillInfo->type == TSDB_FILL_LINEAR) { bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); colDataSetVal(pDst, index, src, isNull); - saveColData(pFillInfo->prev.pRowVal, i, src, isNull); // todo: + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + saveColData(p, i, src, isNull); // todo: } else if (pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { colDataSetNULL(pDst, index); } else if (pFillInfo->type == TSDB_FILL_NEXT) { diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index b442647ff4..fce504d0f7 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -144,10 +144,61 @@ class TDTestCase: tdSql.query(sql) tdSql.checkRows(6) + def test_fill_with_order_by2(self): + ## window size: 5 minutes, with 6 rows in meters every 10 minutes + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart asc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, 10) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(linear) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + def run(self): self.prepareTestEnv() self.test_partition_by_with_interval_fill_prev_new_group_fill_error() self.test_fill_with_order_by() + self.test_fill_with_order_by2() def stop(self): tdSql.close() From 7cb12db8ce4b4eaa3e1446fdb0eca21c74376c8c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 29 Dec 2023 15:20:27 +0800 Subject: [PATCH 141/236] remove more useless code --- source/dnode/vnode/src/inc/tsdb.h | 18 - .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 796 ------------------ 2 files changed, 814 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ebaf0851e6..75f90af378 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -258,18 +258,6 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback); int32_t tsdbFSClose(STsdb *pTsdb); void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); // tsdbReaderWriter.c ============================================================================================== -// SDataFWriter -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); -int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx); -int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast); -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo); - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -281,12 +269,6 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *p int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); -// SDelFWriter -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx); -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx); -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e6f419362c..babf8c75fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -433,635 +433,6 @@ _exit: return code; } -// SDataFWriter ==================================================== -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { - int32_t code = 0; - int32_t flag; - int64_t n; - int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; - SDataFWriter *pWriter = NULL; - char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; - - // alloc - pWriter = taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pWriter->pTsdb = pTsdb; - pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, - .fid = pSet->fid, - .pHeadF = &pWriter->fHead, - .pDataF = &pWriter->fData, - .pSmaF = &pWriter->fSma, - .nSttF = pSet->nSttF}; - pWriter->fHead = *pSet->pHeadF; - pWriter->fData = *pSet->pDataF; - pWriter->fSma = *pSet->pSmaF; - for (int8_t iStt = 0; iStt < pSet->nSttF; iStt++) { - pWriter->wSet.aSttF[iStt] = &pWriter->fStt[iStt]; - pWriter->fStt[iStt] = *pSet->aSttF[iStt]; - } - - // head - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pHeadFD); - if (code) goto _err; - - code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fHead.size += TSDB_FHDR_SIZE; - - // data - if (pWriter->fData.size == 0) { - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_READ | TD_FILE_WRITE; - } - tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pDataFD); - if (code) goto _err; - if (pWriter->fData.size == 0) { - code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fData.size += TSDB_FHDR_SIZE; - } - - // sma - if (pWriter->fSma.size == 0) { - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_READ | TD_FILE_WRITE; - } - tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pSmaFD); - if (code) goto _err; - if (pWriter->fSma.size == 0) { - code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - pWriter->fSma.size += TSDB_FHDR_SIZE; - } - - // stt - ASSERT(pWriter->fStt[pSet->nSttF - 1].size == 0); - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbSttFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fStt[pSet->nSttF - 1], fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pSttFD); - if (code) goto _err; - code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fStt[pWriter->wSet.nSttF - 1].size += TSDB_FHDR_SIZE; - - *ppWriter = pWriter; - return code; - -_err: - tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; - return code; -} - -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - STsdb *pTsdb = NULL; - - if (*ppWriter == NULL) goto _exit; - - pTsdb = (*ppWriter)->pTsdb; - if (sync) { - code = tsdbFsyncFile((*ppWriter)->pHeadFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pDataFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pSmaFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pSttFD); - if (code) goto _err; - } - - tsdbCloseFile(&(*ppWriter)->pHeadFD); - tsdbCloseFile(&(*ppWriter)->pDataFD); - tsdbCloseFile(&(*ppWriter)->pSmaFD); - tsdbCloseFile(&(*ppWriter)->pSttFD); - - for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree((*ppWriter)->aBuf[iBuf]); - } - taosMemoryFree(*ppWriter); -_exit: - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { - int32_t code = 0; - int64_t n; - char hdr[TSDB_FHDR_SIZE]; - - // head ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutHeadFile(hdr, &pWriter->fHead); - code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // data ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutDataFile(hdr, &pWriter->fData); - code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // sma ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSmaFile(hdr, &pWriter->fSma); - code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // stt ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSttFile(hdr, &pWriter->fStt[pWriter->wSet.nSttF - 1]); - code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - // check - if (taosArrayGetSize(aBlockIdx) == 0) { - pHeadFile->offset = pHeadFile->size; - goto _exit; - } - - // prepare - size = 0; - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pHeadFile->offset = pHeadFile->size; - pHeadFile->size += size; - -_exit: - // tsdbTrace("vgId:%d, write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", - // TD_VID(pWriter->pTsdb->pVnode), - // pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); - return code; - -_err: - tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - ASSERT(mDataBlk->nItem > 0); - - // alloc - size = tPutMapData(NULL, mDataBlk); - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = tPutMapData(pWriter->aBuf[0], mDataBlk); - - // write - code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pBlockIdx->offset = pHeadFile->size; - pBlockIdx->size = size; - pHeadFile->size += size; - - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 - " size:%" PRId64 " nItem:%d", - TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, - pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); - return code; - -_err: - tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { - int32_t code = 0; - SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; - int64_t size = 0; - int64_t n; - - // check - if (taosArrayGetSize(aSttBlk) == 0) { - pSttFile->offset = pSttFile->size; - goto _exit; - } - - // size - size = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { - size += tPutSttBlk(NULL, taosArrayGet(aSttBlk, iBlockL)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // encode - n = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { - n += tPutSttBlk(pWriter->aBuf[0] + n, taosArrayGet(aSttBlk, iBlockL)); - } - - // write - code = tsdbWriteFile(pWriter->pSttFD, pSttFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pSttFile->offset = pSttFile->size; - pSttFile->size += size; - -_exit: - tsdbTrace("vgId:%d, tsdb write stt block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pSttFile->offset, size); - return code; - -_err: - tsdbError("vgId:%d, tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { - int32_t code = 0; - - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - - // encode - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; - - SColumnDataAgg sma = {.colId = pColData->cid}; - tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); - if (code) goto _err; - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); - } - - // write - if (pSmaInfo->size) { - code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); - if (code) goto _err; - - pSmaInfo->offset = pWriter->fSma.size; - pWriter->fSma.size += pSmaInfo->size; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - if (toLast) { - pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; - } else { - pBlkInfo->offset = pWriter->fData.size; - } - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; - - int32_t aBufN[4] = {0}; - code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); - if (code) goto _err; - - // write ================= - STsdbFD *pFD = toLast ? pWriter->pSttFD : pWriter->pDataFD; - - pBlkInfo->szKey = aBufN[3] + aBufN[2]; - pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - - int64_t offset = pBlkInfo->offset; - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[3], aBufN[3]); - if (code) goto _err; - offset += aBufN[3]; - - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[2], aBufN[2]); - if (code) goto _err; - offset += aBufN[2]; - - if (aBufN[1]) { - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[1], aBufN[1]); - if (code) goto _err; - offset += aBufN[1]; - } - - if (aBufN[0]) { - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], aBufN[0]); - if (code) goto _err; - } - - // update info - if (toLast) { - pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; - } else { - pWriter->fData.size += pBlkInfo->szBlock; - } - - // ================= SMA ==================== - if (pSmaInfo) { - code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); - if (code) goto _err; - } - -_exit: - tsdbTrace("vgId:%d, tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", - TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, - pBlkInfo->szBlock); - return code; - -_err: - tsdbError("vgId:%d, tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) { - int32_t code = 0; - int32_t lino = 0; - - STsdbFD *pFD = NULL; - if (pSmaInfo) { - pFD = pWriter->pDataFD; - pBlkInfo->offset = pWriter->fData.size; - } else { - pFD = pWriter->pSttFD; - pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; - } - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; - - // hdr - int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); - code = tRealloc(&pWriter->aBuf[0], n); - TSDB_CHECK_CODE(code, lino, _exit); - - tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); - - code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += n; - pBlkInfo->szBlock += n; - - // uid + ver + key - if (pDiskData->pUid) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szUid; - pBlkInfo->szBlock += pDiskData->hdr.szUid; - } - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szVer; - pBlkInfo->szBlock += pDiskData->hdr.szVer; - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szKey; - pBlkInfo->szBlock += pDiskData->hdr.szKey; - - // aBlockCol - if (pDiskData->hdr.szBlkCol) { - code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); - TSDB_CHECK_CODE(code, lino, _exit); - - n = 0; - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol); - } - ASSERT(n == pDiskData->hdr.szBlkCol); - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskData->hdr.szBlkCol; - } - - // aDiskCol - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - if (pDiskCol->pBit) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szBitmap; - } - - if (pDiskCol->pOff) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szOffset; - } - - if (pDiskCol->pVal) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szValue; - } - } - - if (pSmaInfo) { - pWriter->fData.size += pBlkInfo->szBlock; - } else { - pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; - goto _exit; - } - - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue; - if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue; - if (!pDiskCol->bCol.smaOn) continue; - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg)); - TSDB_CHECK_CODE(code, lino, _exit); - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg); - } - - if (pSmaInfo->size) { - pSmaInfo->offset = pWriter->fSma.size; - - code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->fSma.size += pSmaInfo->size; - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { - int32_t code = 0; - int64_t n; - int64_t size; - TdFilePtr pOutFD = NULL; - TdFilePtr PInFD = NULL; - int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; - char fNameFrom[TSDB_FILENAME_LEN]; - char fNameTo[TSDB_FILENAME_LEN]; - - // head - tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); - tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // data - tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); - tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // sma - tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); - tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // stt - for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { - tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); - tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; @@ -1478,173 +849,6 @@ _exit: return code; } -// SDelFWriter ==================================================== -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - char fname[TSDB_FILENAME_LEN]; - uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter = NULL; - int64_t n; - - // alloc - pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); - if (pDelFWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pDelFWriter->pTsdb = pTsdb; - pDelFWriter->fDel = *pFile; - - tsdbDelFileName(pTsdb, pFile, fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); - TSDB_CHECK_CODE(code, lino, _exit); - - // update header - code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); - TSDB_CHECK_CODE(code, lino, _exit); - - pDelFWriter->fDel.size = TSDB_FHDR_SIZE; - pDelFWriter->fDel.offset = 0; - - *ppWriter = pDelFWriter; - -_exit: - if (code) { - if (pDelFWriter) { - tsdbCloseFile(&pDelFWriter->pWriteH); - taosMemoryFree(pDelFWriter); - } - *ppWriter = NULL; - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); - } else { - *ppWriter = pDelFWriter; - } - return code; -} - -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - SDelFWriter *pWriter = *ppWriter; - STsdb *pTsdb = pWriter->pTsdb; - - // sync - if (sync) { - code = tsdbFsyncFile(pWriter->pWriteH); - if (code) goto _err; - } - - // close - tsdbCloseFile(&pWriter->pWriteH); - - for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree(pWriter->aBuf[iBuf]); - } - taosMemoryFree(pWriter); - - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - - // prepare - size = 0; - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pDelIdx->offset = pWriter->fDel.size; - pDelIdx->size = size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - SDelIdx *pDelIdx; - - // prepare - size = 0; - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pWriter->fDel.offset = pWriter->fDel.size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { - int32_t code = 0; - char hdr[TSDB_FHDR_SIZE] = {0}; - int64_t size = TSDB_FHDR_SIZE; - int64_t n; - - // build - tPutDelFile(hdr, &pWriter->fDel); - - // write - code = tsdbWriteFile(pWriter->pWriteH, 0, hdr, size); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} // SDelFReader ==================================================== struct SDelFReader { STsdb *pTsdb; From 7e1d59c565e408dc952b26ecdcd349c070cd3f9f Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 29 Dec 2023 16:22:41 +0800 Subject: [PATCH 142/236] sort --- source/libs/executor/inc/executorInt.h | 5 ++-- source/libs/executor/src/executil.c | 4 +++- source/libs/executor/src/operator.c | 2 +- source/libs/executor/src/scanoperator.c | 32 ++++++++++++++++++------- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 9acef69f9c..c09f615735 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -269,8 +269,9 @@ typedef struct STableScanInfo { SSDataBlock* pResBlock; SHashObj* pIgnoreTables; SSampleExecInfo sample; // sample execution info - int32_t tableStartIndex; // current group scan start - int32_t tableEndIndex; // current group scan end + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end + int32_t currentGroupIndex; // current group index of groupOffset int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 377de99fc0..4b3b1ffead 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2122,6 +2122,9 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool groupByTbname = groupbyTbname(group); size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); + if (!numOfTables) { + return code; + } if (group == NULL || groupByTbname) { for (int32_t i = 0; i < numOfTables; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); @@ -2143,7 +2146,6 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (code != TSDB_CODE_SUCCESS) { return code; } - if (pScanNode->groupOrderScan) pTableListInfo->numOfOuputGroups = taosArrayGetSize(pTableListInfo->pTableList); if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 69a8acb3d7..0b5e7f51ed 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -302,7 +302,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pTableListInfo->idInfo.suid = pTableScanNode->scan.suid; pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType; } else { - code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, + code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e990d3d975..27c9a76757 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -659,17 +659,30 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) { pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); - int32_t i = pInfo->tableStartIndex + 1; - for (; i < numOfTables; ++i) { - STableKeyInfo* pCur = tableListGetInfo(pInfo->base.pTableListInfo, i); - if (pCur->groupId != pStart->groupId) { - break; + STableListInfo* pTableListInfo = pInfo->base.pTableListInfo; + int32_t numOfTables = tableListGetSize(pTableListInfo); + STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableListInfo, pInfo->tableStartIndex); + + if (pTableListInfo->oneTableForEachGroup) { + pInfo->tableEndIndex = pInfo->tableStartIndex; + } else if (pTableListInfo->groupOffset) { + pInfo->currentGroupIndex++; + if (pInfo->currentGroupIndex + 1 < pTableListInfo->numOfOuputGroups) { + pInfo->tableEndIndex = pTableListInfo->groupOffset[pInfo->currentGroupIndex + 1] - 1; + } else { + pInfo->tableEndIndex = numOfTables - 1; } + } else { + int32_t i = pInfo->tableStartIndex + 1; + for (; i < numOfTables; ++i) { + STableKeyInfo* pCur = tableListGetInfo(pTableListInfo, i); + if (pCur->groupId != pStart->groupId) { + break; + } + } + pInfo->tableEndIndex = i - 1; } - pInfo->tableEndIndex = i - 1; if (!pInfo->needCountEmptyTable) { pInfo->countState = TABLE_COUNT_STATE_END; } else { @@ -677,7 +690,7 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i } *pKeyInfo = pStart; - *size = i - pInfo->tableStartIndex; + *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; } static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, @@ -1133,6 +1146,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->tableEndIndex = -1; + pInfo->currentGroupIndex = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; From d158ca74d841b0cf373725fed96010b8a7872f4a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Dec 2023 17:22:27 +0800 Subject: [PATCH 143/236] fix:[TS-4391] rebalance cnt always 1 if msg lost --- source/dnode/mnode/impl/src/mndConsumer.c | 15 --------------- source/dnode/mnode/impl/src/mndSubscribe.c | 14 ++++++++------ 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index cf8b9e019a..9d9473d883 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -220,21 +220,6 @@ FAIL: return -1; } -static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); - if (pRebInfo == NULL) { - pRebInfo = tNewSMqRebSubscribe(key); - if (pRebInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - taosHashPut(pHash, key, strlen(key) + 1, pRebInfo, sizeof(SMqRebInfo)); - taosMemoryFree(pRebInfo); - pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); - } - return pRebInfo; -} - static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t code = 0; SMnode *pMnode = pMsg->info.node; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b7958c1484..16ae7f2548 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -209,16 +209,18 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, } static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebInfo *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); - if (pRebSub == NULL) { - pRebSub = tNewSMqRebSubscribe(key); - if (pRebSub == NULL) { + SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); + if (pRebInfo == NULL) { + pRebInfo = tNewSMqRebSubscribe(key); + if (pRebInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebInfo)); + taosHashPut(pHash, key, strlen(key) + 1, pRebInfo, sizeof(SMqRebInfo)); + taosMemoryFree(pRebInfo); + pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); } - return pRebSub; + return pRebInfo; } static void doRemoveLostConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { From 8ac02adc0b94aae0e013c7914ec6182b24413a22 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 29 Dec 2023 17:42:23 +0800 Subject: [PATCH 144/236] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqRead.c | 3 ++- source/libs/wal/src/walRead.c | 5 ----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 41b1aa7bd1..bfa8cfdb53 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -195,7 +195,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal); int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal); - wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64", 0x%"PRIx64, + wDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64 + ", 0x%" PRIx64, vgId, offset, lastVer, committedVer, appliedVer, id); while (offset <= appliedVer) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c0435ca774..1660a0ecf0 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -259,11 +259,6 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { int64_t contLen; bool seeked = false; - wDebug("vgId:%d, try to fetch ver %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64", 0x%"PRIx64, - pRead->pWal->cfg.vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, - pRead->pWal->vers.appliedVer, pRead->readerId); - // TODO: valid ver if (ver > pRead->pWal->vers.commitVer) { return -1; From 7abeb5a27a3cd6077d0e4e0c1c659c00e41d3b26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 29 Dec 2023 17:44:10 +0800 Subject: [PATCH 145/236] fix(stream): update the stream error code. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- tests/system-test/8-stream/max_delay_session.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 25f51b85df..57f3ac174c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -792,7 +792,7 @@ static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { / if (pStream->targetStbUid == pStreamObj->targetStbUid) { mError("Cannot write the same stable as other stream:%s", pStream->name); sdbCancelFetch(pMnode->pSdb, pIter); - terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; + terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE; return terrno; } } diff --git a/tests/system-test/8-stream/max_delay_session.py b/tests/system-test/8-stream/max_delay_session.py index 46c4c5801d..7e9d365fc8 100644 --- a/tests/system-test/8-stream/max_delay_session.py +++ b/tests/system-test/8-stream/max_delay_session.py @@ -27,6 +27,8 @@ class TDTestCase: self.tb_stream_des_table = f'{self.tb_name}{self.tdCom.des_table_suffix}' self.tdCom.date_time = self.tdCom.dataDict["start_ts"] + time.sleep(1) + if watermark is not None: watermark_value = f'{self.tdCom.dataDict["watermark"]}s' else: From 87b7a9a980233880deda02ef05a0f3915560379c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 29 Dec 2023 17:51:35 +0800 Subject: [PATCH 146/236] fix: stream test case --- tests/script/tsim/stream/basic0.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 917a0a73ab..a525594861 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -33,6 +33,8 @@ if $rows != 3 then return -1 endi +sleep 1000 + sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) sql show stables From 1e8244f3c692cf181fa1a29d1c4d4b3a7493fa52 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 10:31:15 +0800 Subject: [PATCH 147/236] feat: add s3 minio ci case --- source/common/src/tglobal.c | 2 +- tests/army/enterprise/s3/s3_basic.json | 58 ++++++++++++++ tests/army/enterprise/s3/s3_basic.py | 105 +++++++++++++++++++++++++ tests/army/frame/caseBase.py | 4 + tests/army/frame/srvCtl.py | 5 ++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 tests/army/enterprise/s3/s3_basic.json create mode 100644 tests/army/enterprise/s3/s3_basic.py diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fdac3882c3..d6d1e1810d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -789,7 +789,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json new file mode 100644 index 0000000000..1c2b9274d2 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"1d", + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 4, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py new file mode 100644 index 0000000000..12c0720112 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.py @@ -0,0 +1,105 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * + +# +# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +# + +''' +s3EndPoint http://192.168.1.52:9000 +s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +s3BucketName ci-bucket +s3UploadDelaySec 60 +''' + + +class TDTestCase(TBase): + updatecfgDict = { + 's3EndPoint': 'http://192.168.1.52:9000', + 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', + 's3BucketName': 'ci-bucket', + 's3UploadDelaySec':'60' + } + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = etool.curFile(__file__, "mlevel_basic.json") + etool.runBenchmark(json=json) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 4 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + def doAction(self): + tdLog.info(f"do action.") + self.flushDb() + self.compactDb() + + # sleep 70s + time.sleep(70) + self.trimDb() + + loop = 0 + while len(sc.dnodeDataFiles()) > 0 and loop < 10: + time.sleep(10) + self.trimDb() + loop += 1 + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + # drop database and free s3 file + self.dropDb() + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 441196f050..6009111d43 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -72,6 +72,10 @@ class TBase: def flushDb(self): tdSql.execute(f"flush database {self.db}") + def dropDb(self): + tdSql.execute(f"drop database {self.db}") + + # # check db correct # diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index e7ef08cde9..99f503d4b6 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -24,4 +24,9 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # return dnode data files list + def dnodeDataFiles(self, idx): + files = [] + return files + sc = srvCtl() \ No newline at end of file From 64164f5684289af8a36ec52a2d41845359536551 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:39:39 +0800 Subject: [PATCH 148/236] fix: run s3_basic.py ok --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 16 ++++++++++++---- tests/army/frame/caseBase.py | 16 ++++++++-------- tests/army/frame/eos.py | 21 +++++++++++++++++++++ tests/army/frame/epath.py | 3 +++ tests/army/frame/srvCtl.py | 12 ++++++++++++ 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 1c2b9274d2..169da290bf 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-12d", + "start_timestamp":"now-15d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 12c0720112..84c91209b3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -17,6 +17,7 @@ import time import taos import frame import frame.etool +import frame.eos from frame.log import * from frame.cases import * @@ -24,6 +25,7 @@ from frame.sql import * from frame.caseBase import * from frame.srvCtl import * from frame import * +from frame.eos import * # # 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa @@ -42,13 +44,16 @@ class TDTestCase(TBase): 's3EndPoint': 'http://192.168.1.52:9000', 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', 's3BucketName': 'ci-bucket', + 's3BlockSize': '10240', + 's3BlockCacheSize': '320', + 's3PageCacheSize': '10240', 's3UploadDelaySec':'60' } def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run - json = etool.curFile(__file__, "mlevel_basic.json") + json = etool.curFile(__file__, "s3_basic.json") etool.runBenchmark(json=json) tdSql.execute(f"use {self.db}") @@ -63,12 +68,15 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s + tdLog.info(f"wait 70s ...") time.sleep(70) self.trimDb() - + + rootPath = sc.clusterRootPath() + cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" loop = 0 - while len(sc.dnodeDataFiles()) > 0 and loop < 10: - time.sleep(10) + while len(eos.runRetList(cmd)) > 0 and loop < 40: + time.sleep(5) self.trimDb() loop += 1 diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 6009111d43..ad6e744fe6 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -63,17 +63,17 @@ class TBase: # db action # - def trimDb(self): - tdSql.execute(f"trim database {self.db}") + def trimDb(self, show = False): + tdSql.execute(f"trim database {self.db}", show = show) - def compactDb(self): - tdSql.execute(f"compact database {self.db}") + def compactDb(self, show = False): + tdSql.execute(f"compact database {self.db}", show = show) - def flushDb(self): - tdSql.execute(f"flush database {self.db}") + def flushDb(self, show = False): + tdSql.execute(f"flush database {self.db}", show = show) - def dropDb(self): - tdSql.execute(f"drop database {self.db}") + def dropDb(self, show = False): + tdSql.execute(f"drop database {self.db}", show = show) # diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index dcb63ff3aa..b24be74393 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -20,11 +20,16 @@ import os import time import datetime import platform +import subprocess # if windows platform return True def isWin(): return platform.system().lower() == 'windows' +# +# execute programe +# + # wait util execute file finished def exe(file): return os.system(file) @@ -34,3 +39,19 @@ def exeNoWait(file): print("exe no wait") +# run return output and error +def run(command): + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait(3) + + output = process.stdout.read().decode(encoding="gbk") + error = process.stderr.read().decode(encoding="gbk") + + return output, error + + +# return list after run +def runRetList(command): + lines = [] + output,error = run(command) + return output.splitlines() diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py index edff9c78a4..c7796befbc 100644 --- a/tests/army/frame/epath.py +++ b/tests/army/frame/epath.py @@ -51,3 +51,6 @@ def binPath(): def binFile(filename): return binPath() + filename + + + diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 99f503d4b6..75f0d79513 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -16,6 +16,8 @@ import os import time import datetime +from frame.server.dnodes import * + class srvCtl: def __init__(self): # record server information @@ -24,9 +26,19 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # + # about path + # + + # get cluster root path like /root/TDinternal/sim/ + def clusterRootPath(self): + return tdDnodes.path + # return dnode data files list def dnodeDataFiles(self, idx): files = [] return files + + sc = srvCtl() \ No newline at end of file From 26023864f14ab5a6b6926bca68a6a43c2cb12a8a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:48:36 +0800 Subject: [PATCH 149/236] fix: add show path information --- tests/army/enterprise/s3/s3_basic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 84c91209b3..85beb195c3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -70,14 +70,15 @@ class TDTestCase(TBase): # sleep 70s tdLog.info(f"wait 70s ...") time.sleep(70) - self.trimDb() + self.trimDb(True) rootPath = sc.clusterRootPath() cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: time.sleep(5) - self.trimDb() + self.trimDb(True) loop += 1 # run From 1600f0654d6c3f46aae1242199aa61f7af223a2a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 15:16:29 +0800 Subject: [PATCH 150/236] fix: s3 for minio finished --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 6 +++--- tests/army/frame/srvCtl.py | 2 +- tests/parallel_test/cases.task | 11 +++++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 169da290bf..e56a18e757 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-15d", + "start_timestamp":"now-13d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 85beb195c3..389b8a52c7 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -68,12 +68,12 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s - tdLog.info(f"wait 70s ...") - time.sleep(70) + tdLog.info(f"wait 65s ...") + time.sleep(65) self.trimDb(True) rootPath = sc.clusterRootPath() - cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 75f0d79513..155c83094d 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -32,7 +32,7 @@ class srvCtl: # get cluster root path like /root/TDinternal/sim/ def clusterRootPath(self): - return tdDnodes.path + return tdDnodes.getDnodesRootDir() # return dnode data files list def dnodeDataFiles(self, idx): diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b5cde02d25..e7395a26f1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -5,10 +5,17 @@ #unit-test ,,y,unit-test,bash test.sh -#army-test +# +# army-test +# ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 -#system test + + +# +# system test +# ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py From 4f56af59c588377d9370fb9e3c46fe10901df6e2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 16:44:52 +0800 Subject: [PATCH 151/236] add -B set bi mode option --- tools/shell/inc/shellInt.h | 1 + tools/shell/src/shellArguments.c | 6 ++++++ tools/shell/src/shellEngine.c | 9 +++++++++ 3 files changed, 16 insertions(+) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 1c885b151c..ba3dadc646 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -66,6 +66,7 @@ typedef struct { char file[PATH_MAX]; char password[TSDB_USET_PASSWORD_LEN]; bool is_gen_auth; + bool is_bi_mode; bool is_raw_time; bool is_version; bool is_dump_config; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 4817b23029..1930a16f46 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -44,6 +44,7 @@ #define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server." #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." +#define SHELL_BI_MODE "Set BI mode" #define SHELL_VERSION "Print program version." #ifdef WEBSOCKET @@ -59,6 +60,7 @@ void shellPrintHelp() { printf("Usage: taos [OPTION...] \r\n\r\n"); printf("%s%s%s%s\r\n", indent, "-a,", indent, SHELL_AUTH); printf("%s%s%s%s\r\n", indent, "-A,", indent, SHELL_GEN_AUTH); + printf("%s%s%s%s\r\n", indent, "-B,", indent, SHELL_BI_MODE); printf("%s%s%s%s\r\n", indent, "-c,", indent, SHELL_CFG_DIR); printf("%s%s%s%s\r\n", indent, "-C,", indent, SHELL_DMP_CFG); printf("%s%s%s%s\r\n", indent, "-d,", indent, SHELL_DB); @@ -127,6 +129,7 @@ static struct argp_option shellOptions[] = { {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, + {"bimode", 'B', "BIMODE", 0, SHELL_BI_MODE}, {0}, }; @@ -173,6 +176,9 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'A': pArgs->is_gen_auth = true; break; + case 'B': + pArgs->is_bi_mode = true; + break; case 'c': #ifdef WEBSOCKET pArgs->cloud = false; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index e8a5b04178..40b0236d07 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1291,6 +1291,15 @@ int32_t shellExecute() { shellSetConn(shell.conn, runOnce); shellReadHistory(); + if(shell.args.is_bi_mode) { + // need set bi mode +#ifdef WEBSOCKET + //ws_taos_set_conn_mode(shell.ws_conn, TAOS_CONN_MODE_BI, 1); +#else + taos_set_conn_mode(shell.conn, TAOS_CONN_MODE_BI, 1); +#endif + } + if (runOnce) { if (pArgs->commands != NULL) { printf("%s%s\r\n", shell.info.promptHeader, pArgs->commands); From 861cb8572adcbd56c72834b3660ece57bdb52ad0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 17:15:08 +0800 Subject: [PATCH 152/236] fix: taos-tools can not build passed --- source/libs/parser/src/parTranslater.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..3030ec5734 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1104,11 +1104,9 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p return DEAL_RES_CONTINUE; } -#ifndef TD_ENTERPRISE bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { return false; } -#endif static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { if (NULL == pCxt->pCurrStmt || @@ -3178,9 +3176,7 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { return TSDB_CODE_SUCCESS; } -#ifndef TD_ENTERPRISE int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } -#endif static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { SNode* pNode = NULL; @@ -5743,11 +5739,9 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions return code; } -#ifndef TD_ENTERPRISE int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { return TSDB_CODE_SUCCESS; } -#endif static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { if (NULL != strchr(pStmt->tableName, '.')) { From c6549b5ff65c7afccd9294f87693d175559a563c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 21:24:58 +0800 Subject: [PATCH 153/236] fix: format tweak --- tools/shell/src/shellArguments.c | 2 +- tools/shell/src/shellEngine.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 1930a16f46..71fa777d63 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -129,7 +129,7 @@ static struct argp_option shellOptions[] = { {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, - {"bimode", 'B', "BIMODE", 0, SHELL_BI_MODE}, + {"bimode", 'B', 0, 0, SHELL_BI_MODE}, {0}, }; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 40b0236d07..60b1dfab41 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1293,6 +1293,7 @@ int32_t shellExecute() { if(shell.args.is_bi_mode) { // need set bi mode + printf("Set BI mode is true.\n"); #ifdef WEBSOCKET //ws_taos_set_conn_mode(shell.ws_conn, TAOS_CONN_MODE_BI, 1); #else From 62b2a5320e6e35d909048e7745aa88490cd4c3c3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 21:29:49 +0800 Subject: [PATCH 154/236] fix: prune assert --- tools/shell/src/shellCommand.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 8c91ff53e2..c6459c4590 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -62,7 +62,6 @@ int32_t shellCountPrefixOnes(uint8_t c) { } void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - ASSERT(pos > 0); if (pos <= 0) return; TdWchar wc; @@ -82,7 +81,6 @@ void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - ASSERT(pos >= 0); if(pos < 0) return; TdWchar wc; @@ -91,7 +89,6 @@ void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; TdWchar wc; @@ -138,7 +135,6 @@ void shellInsertStr(SShellCmd *cmd, char *str, int32_t size) { } void shellBackspaceChar(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { @@ -159,7 +155,6 @@ void shellBackspaceChar(SShellCmd *cmd) { } void shellClearLineBefore(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -174,7 +169,6 @@ void shellClearLineBefore(SShellCmd *cmd) { } void shellClearLineAfter(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -184,7 +178,6 @@ void shellClearLineAfter(SShellCmd *cmd) { } void shellDeleteChar(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { @@ -203,7 +196,6 @@ void shellDeleteChar(SShellCmd *cmd) { } void shellMoveCursorLeft(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { @@ -218,7 +210,6 @@ void shellMoveCursorLeft(SShellCmd *cmd) { } void shellMoveCursorRight(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { @@ -233,7 +224,6 @@ void shellMoveCursorRight(SShellCmd *cmd) { } void shellPositionCursorHome(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { @@ -254,7 +244,6 @@ void positionCursorMiddle(SShellCmd *cmd) { } void shellPositionCursorEnd(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { @@ -290,7 +279,6 @@ void shellPositionCursor(int32_t step, int32_t direction) { } void shellUpdateBuffer(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); @@ -306,7 +294,6 @@ void shellUpdateBuffer(SShellCmd *cmd) { } bool shellIsReadyGo(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return false; char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); @@ -334,7 +321,6 @@ void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { } void shellResetCommand(SShellCmd *cmd, const char s[]) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); From 3302e22c791f560035653650bba1cd7a14083322 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 22:39:53 +0800 Subject: [PATCH 155/236] case: create db with wa set small --- tests/system-test/0-others/splitVGroupRep1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index 0b75a3e6e1..f799c9dbd8 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 1 wal_retention_size 1" self.exeDouble(sql) # create super talbe st From fcc8185e452d37bf38e85ca3cf66cb4e5e2aa3d2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 11:35:39 +0800 Subject: [PATCH 156/236] coverage: add snapshot.json case --- tests/army/community/cluster/snapshot.json | 60 +++++++++++++++++ tests/army/community/cluster/snapshot.py | 76 ++++++++++++++++++++++ tests/army/frame/caseBase.py | 5 -- tests/parallel_test/cases.task | 2 +- 4 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 tests/army/community/cluster/snapshot.json create mode 100644 tests/army/community/cluster/snapshot.py diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json new file mode 100644 index 0000000000..56a3630bca --- /dev/null +++ b/tests/army/community/cluster/snapshot.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 3, + "duration":"1d", + "wal_retention_period": 1, + "wal_retention_size": 1, + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 10000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 32} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py new file mode 100644 index 0000000000..984ea0bece --- /dev/null +++ b/tests/army/community/cluster/snapshot.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + jfile = etool.curFile(__file__, "snapshot.json") + etool.runBenchmark(json=jfile) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 10 + self.insert_rows = 100000 + self.timestamp_step = 10000 + + def doAction(self): + tdLog.info(f"do action.") + self.flushDb() + self.trimDb() + self.compactDb() + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index ad6e744fe6..ece3d2ccc3 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -42,11 +42,6 @@ class TBase: self.db = "db" self.stb = "stb" - # variant in taosBenchmark json - self.childtable_count = 2 - self.insert_rows = 1000000 - self.timestamp_step = 1000 - # sql self.sqlSum = f"select sum(ic) from {self.stb}" self.sqlMax = f"select max(ic) from {self.stb}" diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e7395a26f1..093fec78ab 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,7 +10,7 @@ # ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 - +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 # From 673d3bc376ad8955bd4b3b9163853eebb983b915 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 31 Dec 2023 13:15:05 +0800 Subject: [PATCH 157/236] arrangeTableGroup --- source/libs/executor/src/executil.c | 70 +++++++++++++++++++++++++++++ source/libs/executor/src/operator.c | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 4b3b1ffead..f3e2e99332 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2116,6 +2116,70 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TSDB_CODE_SUCCESS; } +static int32_t arrangeTableGroup(STableListInfo* pTableListInfo) { + int32_t code = TSDB_CODE_SUCCESS; + size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); + SArray* pDup = NULL; + SHashObj* pHashObj = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (!pHashObj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t num = 1; + // first: get the number of tables per group + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, i); + int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); + // update each group's table count + if (!pVal) { + taosHashPut(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId), &num, sizeof(num)); + } else { + (*pVal)++; + } + } + pTableListInfo->numOfOuputGroups = taosHashGetSize(pHashObj); + pTableListInfo->oneTableForEachGroup = (pTableListInfo->numOfOuputGroups == numOfTables); + + if (pTableListInfo->numOfOuputGroups > 1 && pTableListInfo->numOfOuputGroups < numOfTables) { + pDup = taosArrayDup(pTableListInfo->pTableList, NULL); + pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups); + if (pDup == NULL || pTableListInfo->groupOffset == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + int32_t idx = 0, sum = 0; + + void* pIter = taosHashIterate(pHashObj, NULL); + while (pIter != NULL) { + // record each group's offset + pTableListInfo->groupOffset[idx] = sum; + int32_t* pData = (int32_t*)pIter; + sum += *pData; + // change value to record group item's first position + *pData = pTableListInfo->groupOffset[idx++]; + pIter = taosHashIterate(pHashObj, pIter); + } + + // second: arrange the tables and put the items with same groupId together + STableKeyInfo* pStart = taosArrayGet(pTableListInfo->pTableList, 0); + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pInfo = taosArrayGet(pDup, i); + int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); + if (*pVal != i) { + pStart[*pVal] = *pInfo; + } + (*pVal)++; // update to next item's position + } + } + +_exit: + taosHashCleanup(pHashObj); + taosArrayDestroy(pDup); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pTableListInfo->groupOffset); + } + return code; +} + int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, bool groupSort, uint8_t* digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; @@ -2149,6 +2213,12 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; + if (pTableScanNode->needCountEmptyTable) { + // only put together tables with the same groupid + arrangeTableGroup(pTableListInfo); + } } } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 0b5e7f51ed..69a8acb3d7 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -302,7 +302,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pTableListInfo->idInfo.suid = pTableScanNode->scan.suid; pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType; } else { - code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, + code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; From aed127ab0b92e4b99773ff6315717b573e569a20 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 16:53:57 +0800 Subject: [PATCH 158/236] coverage: snapshot add balance operator --- tests/army/community/cluster/snapshot.json | 2 +- tests/army/community/cluster/snapshot.py | 16 ++++ tests/army/frame/caseBase.py | 91 +++++++++++++++++++++- tools/shell/src/shellAuto.c | 8 +- 4 files changed, 109 insertions(+), 8 deletions(-) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json index 56a3630bca..64bb2aaf3c 100644 --- a/tests/army/community/cluster/snapshot.json +++ b/tests/army/community/cluster/snapshot.json @@ -36,7 +36,7 @@ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, { "type": "double", "name": "dc"}, - { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, { "type": "int", "name": "ic" }, { "type": "bigint", "name": "bi" }, diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 984ea0bece..40783ad4ce 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -13,11 +13,13 @@ import sys import time +import random import taos import frame import frame.etool + from frame.log import * from frame.cases import * from frame.sql import * @@ -43,7 +45,21 @@ class TDTestCase(TBase): def doAction(self): tdLog.info(f"do action.") self.flushDb() + + # split vgroups + self.splitVGroups() self.trimDb() + + # balance vgroups + self.balanceVGroupLeader() + + # replica to 1 + self.alterReplica(1) + + vgids = self.getVGroup() + selid = random.choice(vgids) + self.balanceVGroupLeaderOn(selid) + self.compactDb() # run diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index ece3d2ccc3..05ccf975a7 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +import random from frame.log import * from frame.sql import * @@ -47,7 +48,8 @@ class TBase: self.sqlMax = f"select max(ic) from {self.stb}" self.sqlMin = f"select min(ic) from {self.stb}" self.sqlAvg = f"select avg(ic) from {self.stb}" - + self.sqlFirst = f"select first(ts) from {self.stb}" + self.sqlLast = f"select last(ts) from {self.stb}" # stop def stop(self): @@ -70,6 +72,50 @@ class TBase: def dropDb(self, show = False): tdSql.execute(f"drop database {self.db}", show = show) + def splitVGroups(self): + vgids = self.getVGroup(self.db) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + + def alterReplica(self, replica): + sql = f"alter database {self.db} replica {replica}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + def balanceVGroup(self): + sql = f"balance vgroup" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + def balanceVGroupLeader(self): + sql = f"balance vgroup leader" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + + def balanceVGroupLeaderOn(self, vgId): + sql = f"balance vgroup leader on {vgId}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + # # check db correct @@ -90,12 +136,13 @@ class TBase: tdSql.checkAgg(sql, 0) # save agg result - def snapshotAgg(self): - + def snapshotAgg(self): self.sum = tdSql.getFirstValue(self.sqlSum) self.avg = tdSql.getFirstValue(self.sqlAvg) self.min = tdSql.getFirstValue(self.sqlMin) self.max = tdSql.getFirstValue(self.sqlMax) + self.first = tdSql.getFirstValue(self.sqlFirst) + self.last = tdSql.getFirstValue(self.sqlLast) # check agg def checkAggCorrect(self): @@ -103,3 +150,41 @@ class TBase: tdSql.checkAgg(self.sqlAvg, self.avg) tdSql.checkAgg(self.sqlMin, self.min) tdSql.checkAgg(self.sqlMax, self.max) + tdSql.checkAgg(self.sqlFirst, self.first) + tdSql.checkAgg(self.sqlLast, self.last) + + +# +# get db information +# + + # get vgroups + def getVGroup(self, db_name): + vgidList = [] + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + res = tdSql.getResult(sql) + rows = len(res) + for i in range(rows): + vgidList.append(res[i][0]) + + return vgidList + + + +# +# util +# + + # wait transactions count to zero , return False is translation not finished + def waitTransactionZero(self, seconds = 300, interval = 1): + # wait end + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("split vgroup finished.") + return True + #tdLog.info(f"i={i} wait ...") + time.sleep(interval) + + return False diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index bd5329d810..e9b9b9e944 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -84,8 +84,8 @@ SWords shellCommands[] = { {"alter topic", 0, 0, NULL}, {"alter user ;", 0, 0, NULL}, #ifdef TD_ENTERPRISE - {"balance vgroup;", 0, 0, NULL}, - {"balance vgroup leader ", 0, 0, NULL}, + {"balance vgroup ;", 0, 0, NULL}, + {"balance vgroup leader on ", 0, 0, NULL}, #endif // 20 @@ -531,8 +531,8 @@ void showHelp() { printf( "\n\n\ ----- special commands on enterpise version ----- \n\ - balance vgroup; \n\ - balance vgroup leader \n\ + balance vgroup ;\n\ + balance vgroup leader on \n\ compact database ; \n\ redistribute vgroup dnode ;\n\ split vgroup ;"); From d318c8f9831db04533bd2e4821e49ffe60d29aa5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:04:18 +0800 Subject: [PATCH 159/236] fix: add data correct check point --- tests/army/community/cluster/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 40783ad4ce..66f9d9f598 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -49,14 +49,16 @@ class TDTestCase(TBase): # split vgroups self.splitVGroups() self.trimDb() + self.checkAggCorrect() # balance vgroups self.balanceVGroupLeader() # replica to 1 self.alterReplica(1) + self.checkAggCorrect() - vgids = self.getVGroup() + vgids = self.getVGroup(self.db) selid = random.choice(vgids) self.balanceVGroupLeaderOn(selid) From 6dba2a7aefe52a1dfd9529b17002670f7e1b4570 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:11:47 +0800 Subject: [PATCH 160/236] fix: add replica to 3 --- tests/army/community/cluster/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 66f9d9f598..b177ba3b29 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -57,12 +57,14 @@ class TDTestCase(TBase): # replica to 1 self.alterReplica(1) self.checkAggCorrect() + self.compactDb() + self.alterReplica(3) vgids = self.getVGroup(self.db) selid = random.choice(vgids) self.balanceVGroupLeaderOn(selid) - self.compactDb() + # run def run(self): From 902529352dd361b2d9625ec52e3057d1ea10c36a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:16:54 +0800 Subject: [PATCH 161/236] fix: transaction tips --- tests/army/frame/caseBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 05ccf975a7..8d2c1e6d18 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -182,7 +182,7 @@ class TBase: sql ="show transactions;" rows = tdSql.query(sql) if rows == 0: - tdLog.info("split vgroup finished.") + tdLog.info("transaction count became zero.") return True #tdLog.info(f"i={i} wait ...") time.sleep(interval) From 28238fa7dc6792f426567836597d2edfbab6fc53 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 18:29:37 +0800 Subject: [PATCH 162/236] coverage: no call function with BUILD_NO_CALL MACRO define --- source/libs/stream/src/streamBackendRocksdb.c | 28 +++++++++++++++++-- source/libs/stream/src/streamMeta.c | 2 ++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5485c13616..a5a4bd4dc1 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -665,6 +665,7 @@ void streamBackendHandleCleanup(void* arg) { return; } +#ifdef BUILD_NO_CALL int32_t getLatestCheckpoint(void* arg, int64_t* checkpoint) { SStreamMeta* pMeta = arg; taosWLockLatch(&pMeta->chkpDirLock); @@ -738,6 +739,7 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { taosArrayDestroy(chkpDel); return 0; } +#endif /* * checkpointSave |--cp1--|--cp2--|--cp3--|--cp4--|--cp5--| * chkpInUse: |--cp2--|--cp4--| @@ -855,6 +857,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { return 0; } +#ifdef BUILD_NO_CALL int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { return 0; // SArray* pHandle = taosArrayInit(16, POINTER_BYTES); @@ -891,6 +894,7 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* // *ppHandle = ppCf; // return nCf; } +#endif int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { SArray* pHandle = taosArrayInit(8, POINTER_BYTES); @@ -1006,6 +1010,7 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { return code; } +#ifdef BUILD_NO_CALL int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { // if (arg == NULL) return 0; @@ -1029,6 +1034,7 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { // } // taosWUnLockLatch(&pMeta->chkpDirLock); } +#endif /* 0 @@ -1099,7 +1105,9 @@ void streamBackendDelCompare(void* backend, void* arg) { taosMemoryFree(node); } } +#ifdef BUILD_NO_CALL void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } +#endif void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); if (inst->pHandle) { @@ -2170,6 +2178,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t taosMemoryFree(cfOpts); return 0; } +#ifdef BUILD_NO_CALL int streamStateOpenBackend(void* backend, SStreamState* pState) { taosAcquireRef(streamBackendId, pState->streamBackendRid); SBackendWrapper* handle = backend; @@ -2279,6 +2288,7 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { wrapper->remove |= remove; // update by other pState taosReleaseRef(streamBackendCfWrapperId, pState->pTdbState->backendCfWrapperId); } +#endif void streamStateDestroyCompar(void* arg) { SCfComparator* comp = (SCfComparator*)arg; for (int i = 0; i < comp->numOfComp; i++) { @@ -2669,7 +2679,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { STREAM_STATE_DEL_ROCKSDB(pState, "state", &maxStateKey); return pCur; } - +#ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -2719,6 +2729,7 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) { STREAM_STATE_DEL_ROCKSDB(pState, "func", key); return 0; } +#endif // session cf int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { @@ -3128,6 +3139,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const streamStateFreeCur(pCur); return NULL; } +#ifdef BUILD_NO_CALL int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { stDebug("streamStateSessionGetKeyByRange_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -3185,6 +3197,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes streamStateFreeCur(pCur); return -1; } +#endif int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen) { @@ -3317,6 +3330,7 @@ _end: return res; } +#ifdef BUILD_NO_CALL // partag cf int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen) { int code = 0; @@ -3329,6 +3343,7 @@ int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void STREAM_STATE_GET_ROCKSDB(pState, "partag", &groupId, tagVal, tagLen); return code; } +#endif // parname cfg int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { int code = 0; @@ -3342,11 +3357,13 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi return code; } +#ifdef BUILD_NO_CALL int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } +#endif int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); @@ -3400,6 +3417,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co rocksdb_iter_destroy(pIter); return code; } +#ifdef BUILD_NO_CALL void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -3443,6 +3461,7 @@ char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) { return ret; } +#endif // batch func void* streamStateCreateBatch() { rocksdb_writebatch_t* pBatch = rocksdb_writebatch_create(); @@ -3796,11 +3815,12 @@ void dbChkpDestroy(SDbChkp* pChkp) { taosMemoryFree(pChkp->pManifest); taosMemoryFree(pChkp); } - +#ifdef BUILD_NO_CALL int32_t dbChkpInit(SDbChkp* p) { if (p == NULL) return 0; return 0; } +#endif int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { taosThreadRwlockRdlock(&p->rwLock); int32_t code = -1; @@ -3945,6 +3965,7 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, return code; } +#ifdef BUILD_NO_CALL int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path) { int32_t code = -1; @@ -3974,4 +3995,5 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) { taosThreadRwlockUnlock(&bm->rwLock); return code; -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d09d370a36..a658c46c28 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -419,6 +419,7 @@ _err: } // todo refactor: the lock shoud be restricted in one function +#ifdef BUILD_NO_CALL void streamMetaInitBackend(SStreamMeta* pMeta) { pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); if (pMeta->streamBackend == NULL) { @@ -440,6 +441,7 @@ void streamMetaInitBackend(SStreamMeta* pMeta) { pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); streamBackendLoadCheckpointInfo(pMeta); } +#endif void streamMetaClear(SStreamMeta* pMeta) { // remove all existed tasks in this vnode From 40205541a67f8db225254d111aef6390abd9a650 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:19:23 +0800 Subject: [PATCH 163/236] fix: check in client when create index --- source/libs/parser/src/parTranslater.c | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bdee4d54ac..42aebb3806 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6929,11 +6929,45 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde } static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + SName name; + STableMeta* pMeta = NULL; + int32_t code = + getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + if (code) { + taosMemoryFree(pMeta); + return code; + } + if (TSDB_SUPER_TABLE != pMeta->tableType) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + } + + if (LIST_LENGTH(pStmt->pCols) != 1) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only 1 tag is allowed"); + } + + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pCols) { + SColumnNode* p = (SColumnNode*)pNode; + const SSchema* pSchema = getTagSchema(pMeta, p->colName); + if (!pSchema) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, p->colName); + } + if (IS_IDX_ON(pSchema)) { + code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + goto _exit; + } + } + SCreateTagIndexReq createTagIdxReq = {0}; - int32_t code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); + code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); } +_exit: + taosMemoryFree(pMeta); return code; } From 0d26c186f0d3f11818264498e9e5b40b97626881 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:23:19 +0800 Subject: [PATCH 164/236] chore: code optimization --- source/libs/parser/src/parTranslater.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42aebb3806..d980c0f5ae 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6949,11 +6949,10 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS SNode* pNode = NULL; FOREACH(pNode, pStmt->pCols) { - SColumnNode* p = (SColumnNode*)pNode; - const SSchema* pSchema = getTagSchema(pMeta, p->colName); + const SSchema* pSchema = getTagSchema(pMeta, ((SColumnNode*)pNode)->colName); if (!pSchema) { taosMemoryFree(pMeta); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, p->colName); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, ((SColumnNode*)pNode)->colName); } if (IS_IDX_ON(pSchema)) { code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; From ecf3a7b6d350f5e3f40ae9736f84e1a6f2b8c9d3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:28:30 +0800 Subject: [PATCH 165/236] fix: code optimization --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d980c0f5ae..c3b9f9fc85 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6939,12 +6939,12 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS } if (TSDB_SUPER_TABLE != pMeta->tableType) { taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only super table is allowed"); } if (LIST_LENGTH(pStmt->pCols) != 1) { taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only 1 tag is allowed"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only one tag is allowed"); } SNode* pNode = NULL; From e79516ade302ddf508230fe950d7b48fd6dbba6d Mon Sep 17 00:00:00 2001 From: fullhonest Date: Tue, 2 Jan 2024 10:02:30 +0800 Subject: [PATCH 166/236] Fix TD-27939: error for sub query is empty when where 1=2 --- source/libs/parser/src/parCalcConst.c | 7 ++++++ tests/system-test/2-query/select_null.py | 31 +++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 441f4da3b1..af24624067 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -334,6 +334,13 @@ static int32_t calcConstSelectWithoutFrom(SCalcConstContext* pCxt, SSelectStmt* static int32_t calcConstSelectFrom(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { int32_t code = calcConstFromTable(pCxt, pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + ((STempTableNode*)pSelect->pFromTable)->pSubquery != NULL && + QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery) && + ((SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery)->isEmptyResult){ + pSelect->isEmptyResult = true; + return code; + } if (TSDB_CODE_SUCCESS == code) { code = calcConstProjections(pCxt, pSelect, subquery); } diff --git a/tests/system-test/2-query/select_null.py b/tests/system-test/2-query/select_null.py index e20f6131c9..8411a33a1f 100755 --- a/tests/system-test/2-query/select_null.py +++ b/tests/system-test/2-query/select_null.py @@ -401,7 +401,29 @@ class TDTestCase: tdSql.execute(sql) sql = "select * from %s.`12345` order by `567` desc limit 2;"%(database) tdSql.error(sql) - + + def td_27939(self,database): + sql = "create table %s.`test1eq2` (`ts` timestamp, id int);"%(database) + tdSql.execute(sql) + + sql = "insert into %s.test1eq2 values (now,1);"%(database) + tdSql.execute(sql) + + sql = "insert into %s.`test1eq2` values (now,2);"%(database) + tdSql.execute(sql) + + sql = "select * from %s.`test1eq2` where 1=2;"%(database) + tdSql.query(sql) + tdSql.checkRows(0) + + sql = "select * from (select * from %s.`test1eq2` where 1=2);"%(database) + tdSql.query(sql) + tdSql.checkRows(0) + + sql = "drop table %s.`test1eq2` ;"%(database) + tdSql.execute(sql) + + def run(self): startTime = time.time() @@ -418,6 +440,8 @@ class TDTestCase: self.ts_3110("%s" %self.db) self.ts_23505("%s" %self.db) self.ts_3036("%s" %self.db) + + self.td_27939("%s" %self.db) tdSql.query("flush database %s" %self.db) @@ -430,8 +454,9 @@ class TDTestCase: self.ts_3110("%s" %self.db) self.ts_23505("%s" %self.db) self.ts_3036("%s" %self.db) - - + + self.td_27939("%s" %self.db) + self.test_select_as_chinese_characters(); endTime = time.time() print("total time %ds" % (endTime - startTime)) From 0e9a1b22edf3a71aab960b48f7d02b77b2336ec0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 2 Jan 2024 10:10:44 +0800 Subject: [PATCH 167/236] fix:error msg is not specific in create stream --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bdee4d54ac..c9345dd00e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7446,7 +7446,7 @@ static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStream } } if (!found) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, ((SColumnDefNode*)pTag)->colName); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, getTagNameForCreateStreamTag(pTag)); } } return TSDB_CODE_SUCCESS; From c787535693f3d2d411f2c92e28c59d3c421018c2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 2 Jan 2024 10:11:22 +0800 Subject: [PATCH 168/236] fix session state issue --- source/libs/stream/src/streamSessionState.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index c71edccb99..765403b1aa 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -117,7 +117,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* pFileStore = getStateFileStore(pFileState); void* p = NULL; int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); - if (code_file == TSDB_CODE_SUCCESS) { + if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); code = code_file; qDebug("===stream===0 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); From be0d35a5af93f21e1ee774b440d0a093577432f3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 2 Jan 2024 10:20:04 +0800 Subject: [PATCH 169/236] fix: last table scan reported invalid param when col was dropped which should report invalid col --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- tests/system-test/2-query/last_cache_scan.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 1f54f13592..4ca7881814 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -120,7 +120,7 @@ static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInf } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing i += 1; } else { - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; } } diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 39271318ba..23433277c2 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -413,8 +413,7 @@ class TDTestCase: tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) - - + tdSql.query_success_failed('select last(c2), c2, c3 from meters', queryTimes=10, expectErrInfo="Invalid column name: c2") def run(self): self.prepareTestEnv() From d21741269170f92deebd51684db588154ab1d7bb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 10:33:25 +0800 Subject: [PATCH 170/236] fix(stream): avoid race condition in starting threads. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 2991e3cef5..f576345f64 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -545,7 +545,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = /*(HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; streamTaskHandleEvent(p->status.pSM, event); } else if (!restored) { tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 048092131d..c41aea255d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -571,7 +571,20 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta); - tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); + + { + streamMetaWLock(pMeta); + if (pMeta->startInfo.taskStarting == 1) { + pMeta->startInfo.restartCount += 1; + tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, + pMeta->startInfo.restartCount); + streamMetaWUnLock(pMeta); + } else { + pMeta->startInfo.taskStarting = 1; + streamMetaWUnLock(pMeta); + tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); + } + } } } else { vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); From b3d8f4e9d6914c6d8746da174a4bcd6dc2859104 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 10:39:01 +0800 Subject: [PATCH 171/236] fix(stream): fix a typo. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index f576345f64..a3d860fd78 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -545,7 +545,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = TASK_EVENT_INIT; streamTaskHandleEvent(p->status.pSM, event); } else if (!restored) { tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); From 4368bfb20a0fb4737b91011a0772dfe4ab841879 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 11:40:29 +0800 Subject: [PATCH 172/236] fix(stream): fix dead lock. --- source/dnode/vnode/src/vnd/vnodeSync.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index c41aea255d..84ef36df74 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -587,10 +587,9 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } } } else { + streamMetaWUnLock(pMeta); vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); } - - streamMetaWUnLock(pMeta); } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { From afc2a346e5719a9529a038b7d6a36d31461a623d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 2 Jan 2024 14:16:34 +0800 Subject: [PATCH 173/236] reset update info --- source/libs/executor/inc/executorInt.h | 1 + .../libs/executor/src/streameventwindowoperator.c | 1 + .../libs/executor/src/streamtimewindowoperator.c | 15 +++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e3e504cdbc..a2ced573f7 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -845,6 +845,7 @@ void resetWinRange(STimeWindow* winRange); bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts); int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval); void resetUnCloseSessionWinInfo(SSHashObj* winMap); +void reloadAggSupFromDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup); int32_t encodeSSessionKey(void** buf, SSessionKey* key); void* decodeSSessionKey(void* buf, SSessionKey* key); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index bd247eba07..85f15cc368 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -654,6 +654,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..946b02f2d4 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2759,6 +2759,18 @@ void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, } } +void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup) { + SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; + + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + reloadAggSupFromDownStream(downstream->pDownstream[0], pAggSup); + return; + } + + SStreamScanInfo* pScanInfo = downstream->info; + pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; +} + void streamSessionSemiReloadState(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -2790,6 +2802,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } void streamSessionReloadState(SOperatorInfo* pOperator) { @@ -2842,6 +2855,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -3724,6 +3738,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, From 85ba9a4a5fe2332dd3fc889ee71e8f4e0b96fc20 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 2 Jan 2024 15:45:03 +0800 Subject: [PATCH 174/236] remainGroups --- source/libs/executor/inc/executil.h | 1 + source/libs/executor/inc/executorInt.h | 7 +- source/libs/executor/src/executil.c | 103 +++++++----------------- source/libs/executor/src/scanoperator.c | 51 ++++++++---- 4 files changed, 72 insertions(+), 90 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 946d3311a4..640ed2f2f9 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -105,6 +105,7 @@ typedef struct STableListInfo { int32_t* groupOffset; // keep the offset value for each group in the tableList SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid + SHashObj* remainGroups; // remaining group has not yet processed the empty group STableListIdInfo idInfo; // this maybe the super table or ordinary table } STableListInfo; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index c09f615735..bee378effc 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -218,9 +218,10 @@ enum { }; typedef enum ETableCountState { - TABLE_COUNT_STATE_NONE = 0, // before start scan - TABLE_COUNT_STATE_SCAN = 1, // scanning - TABLE_COUNT_STATE_END = 2, // finish or noneed to process + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // cur group scanning + TABLE_COUNT_STATE_PROCESSED = 2, // cur group processed + TABLE_COUNT_STATE_END = 3, // finish or noneed to process } ETableCountState; typedef struct SAggSupporter { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f3e2e99332..fe3528cce6 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -456,7 +456,7 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C } int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest, - SStorageAPI* pAPI) { + SStorageAPI* pAPI, bool initRemainGroups) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -590,6 +590,15 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf goto end; } + if (initRemainGroups) { + pTableListInfo->remainGroups = + taosHashInit(rows, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pTableListInfo->remainGroups == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + for (int i = 0; i < rows; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); @@ -631,6 +640,14 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf int32_t len = (int32_t)(pStart - (char*)keyBuf); info->groupId = calcGroupId(keyBuf, len); + if (initRemainGroups) { + // groupId ~ table uid + taosHashPut(pTableListInfo->remainGroups, &(info->groupId), sizeof(info->groupId), &(info->uid), sizeof(info->uid)); + } + } + + if (initRemainGroups) { + pTableListInfo->numOfOuputGroups = taosHashGetSize(pTableListInfo->remainGroups); } if (tsTagFilterCache) { @@ -2025,6 +2042,7 @@ STableListInfo* tableListCreate() { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pListInfo->remainGroups = NULL; pListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); if (pListInfo->pTableList == NULL) { @@ -2054,7 +2072,7 @@ void* tableListDestroy(STableListInfo* pTableListInfo) { taosMemoryFreeClear(pTableListInfo->groupOffset); taosHashCleanup(pTableListInfo->map); - + taosHashCleanup(pTableListInfo->remainGroups); pTableListInfo->pTableList = NULL; pTableListInfo->map = NULL; taosMemoryFree(pTableListInfo); @@ -2068,6 +2086,7 @@ void tableListClear(STableListInfo* pTableListInfo) { taosArrayClear(pTableListInfo->pTableList); taosHashClear(pTableListInfo->map); + taosHashClear(pTableListInfo->remainGroups); taosMemoryFree(pTableListInfo->groupOffset); pTableListInfo->numOfOuputGroups = 1; pTableListInfo->oneTableForEachGroup = false; @@ -2116,70 +2135,6 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TSDB_CODE_SUCCESS; } -static int32_t arrangeTableGroup(STableListInfo* pTableListInfo) { - int32_t code = TSDB_CODE_SUCCESS; - size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); - SArray* pDup = NULL; - SHashObj* pHashObj = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (!pHashObj) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t num = 1; - // first: get the number of tables per group - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, i); - int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); - // update each group's table count - if (!pVal) { - taosHashPut(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId), &num, sizeof(num)); - } else { - (*pVal)++; - } - } - pTableListInfo->numOfOuputGroups = taosHashGetSize(pHashObj); - pTableListInfo->oneTableForEachGroup = (pTableListInfo->numOfOuputGroups == numOfTables); - - if (pTableListInfo->numOfOuputGroups > 1 && pTableListInfo->numOfOuputGroups < numOfTables) { - pDup = taosArrayDup(pTableListInfo->pTableList, NULL); - pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups); - if (pDup == NULL || pTableListInfo->groupOffset == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - int32_t idx = 0, sum = 0; - - void* pIter = taosHashIterate(pHashObj, NULL); - while (pIter != NULL) { - // record each group's offset - pTableListInfo->groupOffset[idx] = sum; - int32_t* pData = (int32_t*)pIter; - sum += *pData; - // change value to record group item's first position - *pData = pTableListInfo->groupOffset[idx++]; - pIter = taosHashIterate(pHashObj, pIter); - } - - // second: arrange the tables and put the items with same groupId together - STableKeyInfo* pStart = taosArrayGet(pTableListInfo->pTableList, 0); - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = taosArrayGet(pDup, i); - int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); - if (*pVal != i) { - pStart[*pVal] = *pInfo; - } - (*pVal)++; // update to next item's position - } - } - -_exit: - taosHashCleanup(pHashObj); - taosArrayDestroy(pDup); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableListInfo->groupOffset); - } - return code; -} - int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, bool groupSort, uint8_t* digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; @@ -2206,19 +2161,21 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI); + bool initRemainGroups = false; + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; + if (pTableScanNode->needCountEmptyTable) { + initRemainGroups = true; + } + } + + code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI, initRemainGroups); if (code != TSDB_CODE_SUCCESS) { return code; } if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { - STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable) { - // only put together tables with the same groupid - arrangeTableGroup(pTableListInfo); - } } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 27c9a76757..8999340ddc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -673,14 +673,7 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i pInfo->tableEndIndex = numOfTables - 1; } } else { - int32_t i = pInfo->tableStartIndex + 1; - for (; i < numOfTables; ++i) { - STableKeyInfo* pCur = tableListGetInfo(pTableListInfo, i); - if (pCur->groupId != pStart->groupId) { - break; - } - } - pInfo->tableEndIndex = i - 1; + pInfo->tableEndIndex = numOfTables - 1; } if (!pInfo->needCountEmptyTable) { @@ -693,6 +686,17 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; } +void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { + if (pInfo->countState == TABLE_COUNT_STATE_END) { + return; + } + if (pInfo->base.pTableListInfo->oneTableForEachGroup) { + pInfo->countState = TABLE_COUNT_STATE_PROCESSED; + } else { + taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); + } +} + static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, const STableKeyInfo* tbInfo) { blockDataEmpty(pBlock); @@ -802,7 +806,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; + markGroupProcessed(pTableScanInfo, p->info.id.groupId); return p; } @@ -831,7 +835,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; + markGroupProcessed(pTableScanInfo, p->info.id.groupId); return p; } @@ -849,11 +853,30 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { - // output once for this group + STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo; + if (pTableListInfo->oneTableForEachGroup) { // group by tbname + if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) { + pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED; + STableKeyInfo* pStart = + (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); + return getBlockForEmptyTable(pOperator, pStart); + } + } else { // group by tag + int32_t numOfTables = tableListGetSize(pTableListInfo); + if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) { + // get empty group, mark processed & rm from hash + void* pIte = taosHashIterate(pTableListInfo->remainGroups, NULL); + if (pIte != NULL) { + size_t keySize = 0; + uint64_t* pGroupId = taosHashGetKey(pIte, &keySize); + STableKeyInfo info = {.uid = *(uint64_t*)pIte, .groupId = *pGroupId}; + taosHashCancelIterate(pTableListInfo->remainGroups, pIte); + markGroupProcessed(pTableScanInfo, *pGroupId); + return getBlockForEmptyTable(pOperator, &info); + } + } + } pTableScanInfo->countState = TABLE_COUNT_STATE_END; - STableKeyInfo* pStart = - (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); - return getBlockForEmptyTable(pOperator, pStart); } return NULL; From 1a265a2e9dfbf1c38f4262b9f8e5b839bca71b6e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:26:46 +0800 Subject: [PATCH 175/236] test: modify testcase of hot refresh config --- tests/pytest/util/common.py | 16 ++++++++++++++++ .../test_hot_refresh_configurations.py | 18 ++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 9c45c09715..2e7bd2a89c 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -506,7 +506,23 @@ class TDCom: if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] break + if platform.system().lower() == 'windows': + win_sep = "\\" + buildPath = buildPath.replace(win_sep,'/') + return buildPath + + def getTaosdPath(self, dnodeID="dnode1"): + buildPath = self.getBuildPath() + print("123456",buildPath) + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + taosdPath = buildPath + "/../sim/" + dnodeID + tdLog.info("taosdPath: %s" % taosdPath) + return taosdPath + def getClientCfgPath(self): buildPath = self.getBuildPath() diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 09e634974b..d281b9c1f7 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -201,6 +201,17 @@ class TDTestCase: tdSql.error(f'alter {dnode} "{name} {v}";') def run(self): + + # reset log + taosdPath = tdCom.getTaosdPath() + print(taosdPath,"123") + taosdLogAbsoluteFilename = taosdPath + "/log/" + "taosdlog*" + print(taosdLogAbsoluteFilename) + tdSql.execute("alter all dnodes 'resetlog';") + r = subprocess.Popen("cat {} | grep 'reset log file'".format(taosdLogAbsoluteFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = r.communicate() + assert('reset log file' in stdout.decode()) + for key in self.configration_dic: if "cli" == key: for item in self.configration_dic[key]: @@ -214,12 +225,7 @@ class TDTestCase: self.svr_check(item["name"], item["alias"], item["except_values"], True) else: raise Exception(f"unknown key: {key}") - # reset log - path = os.sep.join([tdDnodes.getDnodesRootDir(), "dnode1", "log", "taosdlog.*"]) - tdSql.execute("alter all dnodes 'resetlog';") - r = subprocess.Popen("cat {} | grep 'reset log file'".format(path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = r.communicate() - assert('reset log file' in stdout.decode()) + def stop(self): tdSql.close() From 480f571c2bda07f02a7b77cd3b4d2beff8763b29 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:29:01 +0800 Subject: [PATCH 176/236] test: modify testcase of hot refresh config --- tests/pytest/util/common.py | 1 - .../system-test/0-others/test_hot_refresh_configurations.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 2e7bd2a89c..010f45a573 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -514,7 +514,6 @@ class TDCom: def getTaosdPath(self, dnodeID="dnode1"): buildPath = self.getBuildPath() - print("123456",buildPath) if (buildPath == ""): tdLog.exit("taosd not found!") else: diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index d281b9c1f7..71f6290469 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -203,10 +203,7 @@ class TDTestCase: def run(self): # reset log - taosdPath = tdCom.getTaosdPath() - print(taosdPath,"123") - taosdLogAbsoluteFilename = taosdPath + "/log/" + "taosdlog*" - print(taosdLogAbsoluteFilename) + taosdLogAbsoluteFilename = tdCom.getTaosdPath() + "/log/" + "taosdlog*" tdSql.execute("alter all dnodes 'resetlog';") r = subprocess.Popen("cat {} | grep 'reset log file'".format(taosdLogAbsoluteFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = r.communicate() From 7333fe54052ac97ac306c727499cf3a3162353bd Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:35:01 +0800 Subject: [PATCH 177/236] test: add script loop.bat --- tests/system-test/loop.bat | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/system-test/loop.bat diff --git a/tests/system-test/loop.bat b/tests/system-test/loop.bat new file mode 100644 index 0000000000..41e3901b3f --- /dev/null +++ b/tests/system-test/loop.bat @@ -0,0 +1,13 @@ +@echo off +set LoopNumner=%1 +set cmd=%2 + +echo "Loop Number: %LoopNumner%" +echo "command : %cmd:"=%" + +For /l %%i in (1,1,%LoopNumner% ) do ( + echo "execute times:"%%i + %cmd:"=% +) + +% case loop.bat 100 "python3 .\test.py -f 2-query\query_cols_tags_and_or.py" % \ No newline at end of file From 2387d5bade30ab3630059ea237bc6197437fe14e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 17:10:12 +0800 Subject: [PATCH 178/236] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 1 + source/libs/stream/inc/streamsm.h | 2 ++ source/libs/stream/src/streamExec.c | 13 ++++++++++--- source/libs/stream/src/streamTaskSm.c | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 0c09e0e1ea..a6dac7f5ba 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -792,6 +792,7 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, void* pFn); int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event); void streamTaskRestoreStatus(SStreamTask* pTask); diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index 7be655fbed..ea0522bd5a 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -34,6 +34,8 @@ typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); typedef struct SAttachedEventInfo { ETaskStatus status; // required status that this event can be handled EStreamTaskEvent event; // the delayed handled event + void* pParam; + void* pFn; } SAttachedEventInfo; typedef struct STaskStateTrans { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1f7bb56ec1..b9839dfc0c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -356,9 +356,16 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); } else { - ASSERT(status == TASK_STATUS__READY|| status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); - streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); - stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, id); + ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); + int32_t code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id, + pStreamTask->id.idStr, tstrerror(code)); + streamMetaReleaseTask(pMeta, pStreamTask); + return code; + } else { + stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, id); + } } // wait for the stream task to handle all in the inputQ, and to be idle diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 4bd6483f7f..d785932109 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -278,10 +278,10 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt ETaskStatus s = streamTaskGetStatus(pTask, NULL); taosThreadMutexUnlock(&pTask->lock); - if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) { + if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event)); return TSDB_CODE_SUCCESS; - } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP) { // this event has been handled already + } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT) { stDebug("s-task:%s not handle event:%s yet, wait for 100ms and recheck", id, GET_EVT_NAME(event)); taosMsleep(100); } else { From 93fe9558289052437385d5b24fe339593dcfd944 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 17:48:05 +0800 Subject: [PATCH 179/236] test: modify testcase of windows --- tests/script/win-test-file | 2 +- tests/system-test/win-test-file | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/script/win-test-file b/tests/script/win-test-file index d9ff09f468..ada8574165 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -15,7 +15,6 @@ ./test.sh -f tsim/dnode/balance2.sim ./test.sh -f tsim/vnode/replica3_repeat.sim ./test.sh -f tsim/parser/col_arithmetic_operation.sim -./test.sh -f tsim/trans/create_db.sim ./test.sh -f tsim/dnode/balance3.sim ./test.sh -f tsim/vnode/replica3_many.sim ./test.sh -f tsim/stable/metrics_idx.sim @@ -201,6 +200,7 @@ ./test.sh -f tsim/query/show_db_table_kind.sim ./test.sh -f tsim/query/bi_star_table.sim ./test.sh -f tsim/query/bi_tag_scan.sim +./test.sh -f tsim/query/bi_tbname_col.sim ./test.sh -f tsim/query/tag_scan.sim ./test.sh -f tsim/query/nullColSma.sim ./test.sh -f tsim/query/bug3398.sim diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index aefdb1e824..0db83e71a3 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -149,7 +149,7 @@ python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -python3 ./test.py -f 7-tmq/tmq_taosx.py +# python3 ./test.py -f 7-tmq/tmq_taosx.py python3 ./test.py -f 7-tmq/tmq_replay.py python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py python3 ./test.py -f 7-tmq/tmq_offset.py @@ -163,6 +163,7 @@ python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 @@ -196,6 +197,7 @@ python3 ./test.py -f 0-others/ttlChangeOnWrite.py python3 ./test.py -f 0-others/compress_tsz1.py python3 ./test.py -f 0-others/compress_tsz2.py python3 ./test.py -f 0-others/view/non_marterial_view/test_view.py +python3 ./test.py -f 0-others/test_show_table_distributed.py python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py python3 ./test.py -N 3 -f 0-others/walRetention.py @@ -204,6 +206,7 @@ python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 python3 ./test.py -f 0-others/delete_check.py python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +python3 ./test.py -f 1-insert/insert_double.py python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/alter_replica.py -N 3 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py @@ -252,6 +255,10 @@ python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 python3 ./test.py -f 1-insert/precisionUS.py python3 ./test.py -f 1-insert/precisionNS.py python3 ./test.py -f 1-insert/test_ts4219.py +python3 ./test.py -f 1-insert/ts-4272.py +python3 ./test.py -f 1-insert/test_ts4295.py +python3 ./test.py -f 1-insert/test_td27388.py +python3 ./test.py -f 1-insert/insert_timestamp.py python3 ./test.py -f 0-others/show.py python3 ./test.py -f 0-others/show_tag_index.py python3 ./test.py -f 0-others/information_schema.py @@ -795,6 +802,7 @@ python3 ./test.py -f 2-query/blockSMA.py -Q 4 python3 ./test.py -f 2-query/projectionDesc.py -Q 4 python3 ./test.py -f 2-query/odbc.py python3 ./test.py -f 2-query/fill_with_group.py +python3 ./test.py -f 2-query/state_window.py -Q 3 python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 python3 ./test.py -f 99-TDcase/TD-20582.py python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 From 5bbf12c2c9240df4677fa3ba00325ce01b8d4b38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 18:35:17 +0800 Subject: [PATCH 180/236] fix(tsdb): check if stt blocks are clean before building composed data blocks. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 1f54f13592..0aef1efc7b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1713,6 +1713,9 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* } else if ((!dataInDataFile) && dataInSttFile) { // no data ile block exists return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + } else if (pBlockScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) { + // opt model for count data in stt file, which is not overlap with data blocks in files. + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else { // row in both stt file blocks and data file blocks TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); From 866fac624fa9985bd3b0d90c371cae19bdc83bc4 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Tue, 2 Jan 2024 19:26:59 +0800 Subject: [PATCH 181/236] Add test cases for TS-4348 and TD-27939 --- tests/system-test/2-query/ts-4348-td-27939.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/system-test/2-query/ts-4348-td-27939.py diff --git a/tests/system-test/2-query/ts-4348-td-27939.py b/tests/system-test/2-query/ts-4348-td-27939.py new file mode 100644 index 0000000000..2a861899d5 --- /dev/null +++ b/tests/system-test/2-query/ts-4348-td-27939.py @@ -0,0 +1,46 @@ +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + tdSql.execute("drop database if exists ts_4338;") + tdSql.execute("create database ts_4338;") + tdSql.execute("drop table if exists ts_4338.t;") + tdSql.execute("create database if not exists ts_4338;") + tdSql.execute("create table ts_4338.t (ts timestamp, i8 tinyint);") + tdSql.execute("insert into ts_4338.t (ts, i8) values (now(), 1) (now()+1s, 2);") + + def run(self): + # TS-4348 + tdSql.query(f'select i8 from ts_4338.t;') + tdSql.checkRows(2) + + tdSql.query(f'select i8 from ts_4338.t where 1 = 1;') + tdSql.checkRows(2) + + tdSql.query(f'select i8 from ts_4338.t where i8 = 1;') + tdSql.checkRows(1) + + tdSql.query(f'select * from (select * from ts_4338.t where i8 = 3);') + tdSql.checkRows(0) + + # TD-27939 + tdSql.query(f'select * from (select * from ts_4338.t where 1 = 100);') + tdSql.checkRows(0) + + tdSql.query(f'select * from (select * from (select * from ts_4338.t where 1 = 200));') + tdSql.checkRows(0) + + tdSql.execute("drop database if exists ts_4338;") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From b48df60fdd157ac058f1d4e4e2731304210c5872 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 20:18:10 +0800 Subject: [PATCH 182/236] test: add testcase that merge same ver stt file after split db --- tests/system-test/0-others/splitVGroupRep1.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index f799c9dbd8..1745b34cc4 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 1 wal_retention_size 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 0 wal_retention_size 1" self.exeDouble(sql) # create super talbe st @@ -221,14 +221,14 @@ class TDTestCase: sql1 = sql.replace('@db_name', self.db1) tdLog.info(sql1) start1 = time.time() - rows1 = tdSql.query(sql1) + rows1 = tdSql.query(sql1,queryTimes=2) spend1 = time.time() - start1 res1 = copy.deepcopy(tdSql.queryResult) sql2 = sql.replace('@db_name', self.db2) tdLog.info(sql2) start2 = time.time() - tdSql.query(sql2) + tdSql.query(sql2,queryTimes=2) spend2 = time.time() - start2 res2 = tdSql.queryResult @@ -269,7 +269,7 @@ class TDTestCase: def checkResult(self): # check vgroupid sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'" - tdSql.query(sql) + tdSql.query(sql,queryTimes=2) tdSql.checkRows(self.vgroups2) # check child table count same @@ -412,8 +412,16 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - + tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver3.stt to db2 and db1 ") + for dbname in [self.db2, self.db1]: + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') tdLog.info("check db1 and db2 same after creating ...") + self.checkResult() for i in range(3): @@ -423,6 +431,15 @@ class TDTestCase: end = time.time() self.vgroups2 += 1 + # insert the same data per tables into splited vgroups + tdLog.info("insert the same data per tables into splited vgroups(3,4)") + for dbname in [self.db2, self.db1]: + for tableid in range(self.childCnt): + tdSql.execute(f'insert into {dbname}.t{tableid} values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.ta values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999",0);') + tdSql.execute(f'flush database {dbname}') + # check two db query result same self.checkResult() spend = "%.3f"%(end-start) From 7b805d2b002f49496d591bb2a2276f05442cc465 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 20:19:09 +0800 Subject: [PATCH 183/236] test: add testcase that merge same ver stt file after split db --- tests/system-test/0-others/splitVGroupRep1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index 1745b34cc4..f7dab4fcf6 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -412,7 +412,7 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver3.stt to db2 and db1 ") + tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver4.stt to db2 and db1 ") for dbname in [self.db2, self.db1]: tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') tdSql.execute(f'flush database {dbname}') From 453d3e8a3069f860b9aa0072986a0aa039084dd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 10:50:38 +0800 Subject: [PATCH 184/236] fix(stream): fix dead lock on windows. --- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 84ef36df74..34ae6623a7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -567,13 +567,13 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) if (vnodeIsRoleLeader(pVnode)) { // start to restore all stream tasks if (tsDisableStream) { + streamMetaWUnLock(pMeta); vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta); { - streamMetaWLock(pMeta); if (pMeta->startInfo.taskStarting == 1) { pMeta->startInfo.restartCount += 1; tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, From c0186a214d9dc11faf56fedd2b483ce228ff91b5 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 3 Jan 2024 14:50:05 +0800 Subject: [PATCH 185/236] test: add testcase for TD-28106 of copy file in snapshot --- tests/parallel_test/cases.task | 6 +- tests/script/win-test-file | 1 + tests/system-test/0-others/splitVGroup.py | 464 ++++++++++++++++++ .../{splitVGroupRep1.py => splitVGroupWal.py} | 8 +- tests/system-test/win-test-file | 8 +- 5 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 tests/system-test/0-others/splitVGroup.py rename tests/system-test/0-others/{splitVGroupRep1.py => splitVGroupWal.py} (99%) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 093fec78ab..edf28b6518 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -260,8 +260,10 @@ e ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py diff --git a/tests/script/win-test-file b/tests/script/win-test-file index ada8574165..744415c89f 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -189,6 +189,7 @@ ./test.sh -f tsim/query/unionall_as_table.sim ./test.sh -f tsim/query/multi_order_by.sim ./test.sh -f tsim/query/sys_tbname.sim +./test.sh -f tsim/query/sort-pre-cols.sim ./test.sh -f tsim/query/groupby.sim ./test.sh -f tsim/query/groupby_distinct.sim ./test.sh -f tsim/query/event.sim diff --git a/tests/system-test/0-others/splitVGroup.py b/tests/system-test/0-others/splitVGroup.py new file mode 100644 index 0000000000..c49713fc6d --- /dev/null +++ b/tests/system-test/0-others/splitVGroup.py @@ -0,0 +1,464 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time +import copy +import string + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # random string + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into @db_name.t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + + # insert nomal talbe + for i in range(20): + self.ts += 1000 + name = self.random_string(20) + sql = f"insert into @db_name.ta values({self.ts}, {i}, {self.ts%100000}, '{name}', false)" + self.exeDouble(sql) + + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + def exeDouble(self, sql): + # dbname replace + sql1 = sql.replace("@db_name", self.db1) + + if len(sql1) > 100: + tdLog.info(sql1[:100]) + else: + tdLog.info(sql1) + tdSql.execute(sql1) + + sql2 = sql.replace("@db_name", self.db2) + if len(sql2) > 100: + tdLog.info(sql2[:100]) + else: + tdLog.info(sql2) + tdSql.execute(sql2) + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000 + self.childCnt = 4 + self.childRow = 10000 + self.batchSize = 50000 + self.vgroups1 = 1 + self.vgroups2 = 1 + self.db1 = "db1" + self.db2 = "db2" + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db wal_retention_period 0 + sql = f"create database @db_name vgroups {self.vgroups1} replica {self.replicaVar} wal_retention_period 0 wal_retention_size 1" + self.exeDouble(sql) + + # create super talbe st + sql = f"create table @db_name.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + self.exeDouble(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table @db_name.t{i} using @db_name.st tags({i}) " + self.exeDouble(sql) + + # create normal table + sql = f"create table @db_name.ta(ts timestamp, c1 int, c2 bigint, c3 binary(32), c4 bool)" + self.exeDouble(sql) + + # insert data + self.insertData() + + # update + self.ts = 1680000000000 + 20000 + self.childRow = 1000 + + + # delete data + sql = "delete from @db_name.st where ts > 1680000019000 and ts < 1680000062000" + self.exeDouble(sql) + sql = "delete from @db_name.st where ts > 1680000099000 and ts < 1680000170000" + self.exeDouble(sql) + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.time() % 10000 + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # check query result same + def queryDouble(self, sql): + # sql + sql1 = sql.replace('@db_name', self.db1) + tdLog.info(sql1) + start1 = time.time() + rows1 = tdSql.query(sql1,queryTimes=2) + spend1 = time.time() - start1 + res1 = copy.deepcopy(tdSql.queryResult) + + sql2 = sql.replace('@db_name', self.db2) + tdLog.info(sql2) + start2 = time.time() + tdSql.query(sql2,queryTimes=2) + spend2 = time.time() - start2 + res2 = tdSql.queryResult + + rowlen1 = len(res1) + rowlen2 = len(res2) + errCnt = 0 + + if rowlen1 != rowlen2: + tdLog.exit(f"both row count not equal. rowlen1={rowlen1} rowlen2={rowlen2} ") + return False + + for i in range(rowlen1): + row1 = res1[i] + row2 = res2[i] + collen1 = len(row1) + collen2 = len(row2) + if collen1 != collen2: + tdLog.exit(f"both col count not equal. collen1={collen1} collen2={collen2}") + return False + for j in range(collen1): + if row1[j] != row2[j]: + tdLog.info(f"error both column value not equal. row={i} col={j} col1={row1[j]} col2={row2[j]} .") + errCnt += 1 + + if errCnt > 0: + tdLog.exit(f" db2 column value different with db2. different count ={errCnt} ") + + # warning performance + diff = (spend2 - spend1)*100/spend1 + tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff)) + if spend2 > spend1 and diff > 20: + tdLog.info("warning: the diff for performance after spliting is over 20%") + + return True + + + # check result + def checkResult(self): + # check vgroupid + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'" + tdSql.query(sql,queryTimes=2) + tdSql.checkRows(self.vgroups2) + + # check child table count same + sql = "select table_name from information_schema.ins_tables where db_name='@db_name' order by table_name" + self.queryDouble(sql) + + # check row value is ok + sql = "select * from @db_name.st order by ts, tbname" + self.queryDouble(sql) + + # where + sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts, tbname" + self.queryDouble(sql) + + # max + sql = "select max(c1) from @db_name.st" + self.queryDouble(sql) + + # min + sql = "select min(c2) from @db_name.st" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.st" + self.queryDouble(sql) + + # normal table + + # count + sql = "select count(*) from @db_name.ta" + self.queryDouble(sql) + + # all rows + sql = "select * from @db_name.ta" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.ta" + self.queryDouble(sql) + + + # get vgroup list + def getVGroup(self, db_name): + vgidList = [] + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + res = tdSql.getResult(sql) + rows = len(res) + for i in range(rows): + vgidList.append(res[i][0]) + + return vgidList; + + # split vgroup on db2 + def splitVGroup(self, db_name): + vgids = self.getVGroup(db_name) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.execute(sql) + + # wait end + seconds = 300 + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("split vgroup finished.") + return True + #tdLog.info(f"i={i} wait split vgroup ...") + time.sleep(1) + + tdLog.exit(f"split vgroup transaction is not finished after executing {seconds}s") + return False + + # split error + def expectSplitError(self, dbName): + vgids = self.getVGroup(dbName) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.error(sql) + + # expect split ok + def expectSplitOk(self, dbName): + # split vgroup + vgList1 = self.getVGroup(dbName) + self.splitVGroup(dbName) + vgList2 = self.getVGroup(dbName) + vgNum1 = len(vgList1) + 1 + vgNum2 = len(vgList2) + if vgNum1 != vgNum2: + tdLog.exit(f" vglist len={vgNum1} is not same for expect {vgNum2}") + return + + # split empty database + def splitEmptyDB(self): + dbName = "emptydb" + vgNum = 2 + # create database + sql = f"create database {dbName} vgroups {vgNum} replica {self.replicaVar }" + tdLog.info(sql) + tdSql.execute(sql) + + # split vgroup + self.expectSplitOk(dbName) + + + # forbid + def checkForbid(self): + # stream + tdLog.info("check forbid split having stream...") + tdSql.execute("create database streamdb;") + tdSql.execute("use streamdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create stream ma into sta as select count(*) from ta interval(1s);") + self.expectSplitError("streamdb") + tdSql.execute("drop stream ma;") + self.expectSplitOk("streamdb") + + # topic + tdLog.info("check forbid split having topic...") + tdSql.execute("create database topicdb wal_retention_period 10;") + tdSql.execute("use topicdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create topic toa as select * from ta;") + + #self.expectSplitError("topicdb") + tdSql.execute("drop topic toa;") + self.expectSplitOk("topicdb") + + # compact and check db2 + def compactAndCheck(self): + tdLog.info("compact db2 and check result ...") + # compact + tdSql.execute(f"compact database {self.db2};") + # check result + self.checkResult() + + # run + def run(self): + # prepare env + self.prepareEnv() + tdLog.info("generate at least two stt files of the same fileset (e.g. v4f1944) for db2 and db1 ") + for dbname in [self.db2, self.db1]: + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdLog.info("check db1 and db2 same after creating ...") + + self.checkResult() + + for i in range(3): + # split vgroup on db2 + start = time.time() + self.splitVGroup(self.db2) + end = time.time() + self.vgroups2 += 1 + + # insert the same data per tables into splited vgroups + tdLog.info("insert the same data per tables into splited vgroups(3,4)") + for dbname in [self.db2, self.db1]: + for tableid in range(self.childCnt): + tdSql.execute(f'insert into {dbname}.t{tableid} values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.ta values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999",0);') + tdSql.execute(f'flush database {dbname}') + + # check two db query result same + self.checkResult() + spend = "%.3f"%(end-start) + tdLog.info(f"split vgroup i={i} passed. spend = {spend}s") + + # split empty db + self.splitEmptyDB() + + # check topic and stream forib + self.checkForbid() + + # compact database + self.compactAndCheck() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupWal.py similarity index 99% rename from tests/system-test/0-others/splitVGroupRep1.py rename to tests/system-test/0-others/splitVGroupWal.py index f7dab4fcf6..dfb2281e5e 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupWal.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 0 wal_retention_size 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica {self.replicaVar}" self.exeDouble(sql) # create super talbe st @@ -277,11 +277,11 @@ class TDTestCase: self.queryDouble(sql) # check row value is ok - sql = "select * from @db_name.st order by ts" + sql = "select * from @db_name.st order by ts, tbname" self.queryDouble(sql) # where - sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts" + sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts, tbname" self.queryDouble(sql) # max @@ -369,7 +369,7 @@ class TDTestCase: dbName = "emptydb" vgNum = 2 # create database - sql = f"create database {dbName} vgroups {vgNum} replica 1" + sql = f"create database {dbName} vgroups {vgNum} replica {self.replicaVar}" tdLog.info(sql) tdSql.execute(sql) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index e261e18456..1ab0fee7bf 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -1,4 +1,5 @@ python3 ./test.py -f 2-query/tbname_vgroup.py +python3 ./test.py -f 2-query/compact-col.py python3 ./test.py -f 2-query/stbJoin.py python3 ./test.py -f 2-query/stbJoin.py -Q 2 python3 ./test.py -f 2-query/stbJoin.py -Q 3 @@ -205,8 +206,10 @@ python3 ./test.py -f 0-others/test_show_table_distributed.py python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py python3 ./test.py -N 3 -f 0-others/walRetention.py -python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 -python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 +python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 1 +python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 1 +python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3 +python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 python3 ./test.py -f 0-others/delete_check.py python3 ./test.py -f 0-others/test_hot_refresh_configurations.py @@ -467,6 +470,7 @@ python3 ./test.py -f 2-query/concat2.py python3 ./test.py -f 2-query/json_tag.py python3 ./test.py -f 2-query/nestedQueryInterval.py python3 ./test.py -f 2-query/systable_func.py +python3 ./test.py -f 2-query/test_ts4382.py python3 ./test.py -f 2-query/stablity.py python3 ./test.py -f 2-query/stablity_1.py python3 ./test.py -f 2-query/elapsed.py From 35b8683262ecfbdcd3a7af15fd80bd74a39136b3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 3 Jan 2024 14:52:24 +0800 Subject: [PATCH 186/236] test: add testcase for TD-28106 of copy file in snapshot --- tests/system-test/0-others/splitVGroupWal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupWal.py b/tests/system-test/0-others/splitVGroupWal.py index dfb2281e5e..d11d63b5a9 100644 --- a/tests/system-test/0-others/splitVGroupWal.py +++ b/tests/system-test/0-others/splitVGroupWal.py @@ -412,7 +412,7 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver4.stt to db2 and db1 ") + tdLog.info("generate at least two stt files of the same fileset (e.g. v4f1944) for db2 and db1 ") for dbname in [self.db2, self.db1]: tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') tdSql.execute(f'flush database {dbname}') From e67c6764664b8f37504b7cc86f61555bade36601 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 3 Jan 2024 15:19:27 +0800 Subject: [PATCH 187/236] TS-4423: add test case --- tests/parallel_test/cases.task | 1 + .../system-test/1-insert/stt_blocks_check.py | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/system-test/1-insert/stt_blocks_check.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 093fec78ab..0e826304c1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -109,6 +109,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_stb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/stt_blocks_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py diff --git a/tests/system-test/1-insert/stt_blocks_check.py b/tests/system-test/1-insert/stt_blocks_check.py new file mode 100644 index 0000000000..5de6414453 --- /dev/null +++ b/tests/system-test/1-insert/stt_blocks_check.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from random import randint +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.boundary import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def stt_block_check(self): + tdSql.prepare() + tdSql.execute('use db') + + tdSql.execute('create table meters (ts timestamp, c1 int, c2 float) tags(t1 int)') + tdSql.execute("create table d0 using meters tags(1)") + + ts = 1704261670000 + + sql = "insert into d0 values " + for i in range(100): + sql = sql + f"({ts + i}, 1, 0.1)" + tdSql.execute(sql) + tdSql.execute("flush database db") + + ts = 1704261670099 + + sql = "insert into d0 values " + for i in range(100): + sql = sql + f"({ts + i}, 1, 0.1)" + tdSql.execute(sql) + tdSql.execute("flush database db") + + tdSql.execute(f"insert into d0 values({ts + 100}, 2, 1.0)") + tdSql.execute("flush database db") + + time.sleep(2) + + tdSql.query("show table distributed db.meters") + tdSql.query("select count(*) from db.meters") + tdSql.checkData(0, 0, 200) + + def run(self): + self.stt_block_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 589b046ecb366471018830268defd73b2735ce93 Mon Sep 17 00:00:00 2001 From: charles Date: Wed, 3 Jan 2024 17:05:55 +0800 Subject: [PATCH 188/236] update alter_database test case for arm64 ci --- tests/system-test/1-insert/alter_database.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 8de43b8ee8..6a831b88ff 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -4,7 +4,8 @@ import time import socket import os import threading - +import psutil +import platform from util.log import * from util.sql import * from util.cases import * @@ -17,6 +18,14 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.buffer_boundary = [3, 4097, 8193, 12289, 16384] + # remove the value > free_memory, 70% is the weight to calculate the max value + if platform.system() == "Linux" and platform.machine() == "aarch64": + mem = psutil.virtual_memory() + free_memory = mem.free * 0.7 / 1024 / 1024 + for item in self.buffer_boundary: + if item > free_memory: + self.buffer_boundary.remove(item) + self.buffer_error = [self.buffer_boundary[0] - 1, self.buffer_boundary[-1]+1] # pages_boundary >= 64 From b818faaa5eba3bf6fc28a4e059ea81d86a3b54c3 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 3 Jan 2024 17:51:27 +0800 Subject: [PATCH 189/236] remainGroups --- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/scanoperator.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fe3528cce6..2e93dbd803 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2164,7 +2164,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool initRemainGroups = false; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable) { + if (pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { initRemainGroups = true; } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8999340ddc..c2a194d150 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -690,7 +690,7 @@ void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { if (pInfo->countState == TABLE_COUNT_STATE_END) { return; } - if (pInfo->base.pTableListInfo->oneTableForEachGroup) { + if (pInfo->base.pTableListInfo->oneTableForEachGroup || pInfo->base.pTableListInfo->groupOffset) { pInfo->countState = TABLE_COUNT_STATE_PROCESSED; } else { taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); @@ -854,14 +854,14 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo; - if (pTableListInfo->oneTableForEachGroup) { // group by tbname + if (pTableListInfo->oneTableForEachGroup || pTableListInfo->groupOffset) { // group by tbname, group by tag + sort if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) { pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED; STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); return getBlockForEmptyTable(pOperator, pStart); } - } else { // group by tag + } else { // group by tag + no sort int32_t numOfTables = tableListGetSize(pTableListInfo); if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) { // get empty group, mark processed & rm from hash From de631e20a5469ff80e638e2e99e742bd6b924aff Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 3 Jan 2024 19:57:22 +0800 Subject: [PATCH 190/236] fix: refactor logdebugs --- include/common/tglobal.h | 4 +- include/util/tconfig.h | 2 +- include/util/tdef.h | 7 ++- include/util/tlog.h | 1 + source/common/src/tglobal.c | 98 +++++++++++++++++++++------------- source/dnode/mgmt/exe/dmMain.c | 2 +- source/util/src/tconfig.c | 37 ++++++++++++- source/util/src/tlog.c | 1 + utils/tsim/inc/simInt.h | 1 - utils/tsim/src/simSystem.c | 2 - 10 files changed, 110 insertions(+), 45 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 887f4cb6dc..f006779a48 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -228,8 +228,8 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer); struct SConfig *taosGetCfg(); -void taosSetAllDebugFlag(int32_t flag, bool rewrite); -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite); +void taosSetAllDebugFlag(int32_t flag); +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 07abdd223d..f2a9446700 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -94,7 +94,7 @@ typedef struct SConfigItem { int64_t imax; double fmax; }; - SArray *array; // SDiskCfg + SArray *array; // SDiskCfg/SLogVar } SConfigItem; typedef struct { diff --git a/include/util/tdef.h b/include/util/tdef.h index 51b0b63da2..875a6f5738 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -260,7 +260,7 @@ typedef enum ELogicConditionType { #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 32 -#define TSDB_LABEL_LEN 12 +#define TSDB_LABEL_LEN 12 #define TSDB_JOB_STATUS_LEN 32 #define TSDB_CLUSTER_ID_LEN 40 @@ -272,6 +272,7 @@ typedef enum ELogicConditionType { #define TSDB_SHOW_SCHEMA_JSON_LEN TSDB_MAX_COLUMNS * 256 #define TSDB_SLOW_QUERY_SQL_LEN 512 #define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_LOG_VAR_LEN 32 #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 @@ -503,6 +504,10 @@ typedef struct { int32_t primary; } SDiskCfg; +typedef struct { + char name[TSDB_LOG_VAR_LEN]; +} SLogVar; + #define TMQ_SEPARATOR ':' enum { diff --git a/include/util/tlog.h b/include/util/tlog.h index 6d393bfefb..11ac0e1fae 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -67,6 +67,7 @@ extern int32_t smaDebugFlag; extern int32_t idxDebugFlag; extern int32_t tdbDebugFlag; extern int32_t sndDebugFlag; +extern int32_t simDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d6d1e1810d..79d21955d4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -423,7 +423,7 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; @@ -497,7 +497,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; @@ -962,6 +962,7 @@ static void taosSetClientLogCfg(SConfig *pCfg) { rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; } static void taosSetServerLogCfg(SConfig *pCfg) { @@ -1278,7 +1279,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetServerLogCfg(pCfg); } - taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false); + taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1356,6 +1357,8 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); if (taosSetFileHandlesLimit() != 0) return -1; + taosSetAllDebugFlag(cfgGetItem(tsCfg, "debugFlag")->i32); + cfgDumpCfg(tsCfg, tsc, false); if (taosCheckGlobalCfg() != 0) { @@ -1399,7 +1402,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, *pVar = flag; if (isDebugflag) { - taosSetDebugFlag(pOptions[d].optionVar, optName, flag, true); + taosSetDebugFlag(pOptions[d].optionVar, optName, flag); } terrno = TSDB_CODE_SUCCESS; } break; @@ -1447,8 +1450,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { } if (strncasecmp(name, "debugFlag", 9) == 0) { - int32_t flag = pItem->i32; - taosSetAllDebugFlag(flag, true); + taosSetAllDebugFlag(pItem->i32); return 0; } @@ -1460,7 +1462,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, - {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, + {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, }; static OptionNameAndVar options[] = { @@ -1524,8 +1526,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { switch (lowcaseName[0]) { case 'd': { if (strcasecmp("debugFlag", name) == 0) { - int32_t flag = pItem->i32; - taosSetAllDebugFlag(flag, true); + taosSetAllDebugFlag(pItem->i32); matched = true; } break; @@ -1691,7 +1692,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { {"cDebugFlag", &cDebugFlag}, {"dDebugFlag", &dDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"jniDebugFlag", &jniDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, - {"uDebugFlag", &uDebugFlag}, + {"uDebugFlag", &uDebugFlag}, {"simDebugFlag", &simDebugFlag}, }; static OptionNameAndVar options[] = { @@ -1736,9 +1737,9 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer) { return taosCfgDynamicOptionsForClient(pCfg, name); } -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite) { +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal) { SConfigItem *pItem = cfgGetItem(tsCfg, flagName); - if (pItem != NULL && (rewrite || pItem->i32 == 0)) { + if (pItem != NULL) { pItem->i32 = flagVal; } if (pFlagPtr != NULL) { @@ -1746,33 +1747,58 @@ void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, } } -void taosSetAllDebugFlag(int32_t flag, bool rewrite) { +static int taosLogVarComp(void const *lp, void const *rp) { + SLogVar *lpVar = (SLogVar *)lp; + SLogVar *rpVar = (SLogVar *)rp; + return strcasecmp(lpVar->name, rpVar->name); +} + +static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag, SArray *noNeedToSetVars) { + if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) { + return; + } + taosSetDebugFlag(pFlagPtr, name, flag); +} + +void taosSetAllDebugFlag(int32_t flag) { if (flag <= 0) return; - taosSetDebugFlag(NULL, "debugFlag", flag, rewrite); - taosSetDebugFlag(NULL, "simDebugFlag", flag, rewrite); - taosSetDebugFlag(NULL, "tmrDebugFlag", flag, rewrite); - taosSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, rewrite); - taosSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, rewrite); - taosSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, rewrite); - taosSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, rewrite); - taosSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, rewrite); - taosSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, rewrite); - taosSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, rewrite); - taosSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, rewrite); - taosSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, rewrite); - taosSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, rewrite); - taosSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, rewrite); - taosSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, rewrite); - taosSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, rewrite); - taosSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, rewrite); - taosSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, rewrite); - taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, rewrite); - taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite); - taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); - taosSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, rewrite); - taosSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, rewrite); + SArray *noNeedToSetVars = NULL; + SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); + if (pItem != NULL) { + pItem->i32 = flag; + noNeedToSetVars = pItem->array; + } + + taosCheckAndSetDebugFlag(&simDebugFlag, "simDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tmrDebugFlag, "tmrDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, noNeedToSetVars); + + taosCheckAndSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, noNeedToSetVars); + + taosCheckAndSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, noNeedToSetVars); + + taosArrayClear(noNeedToSetVars); // reset array + uInfo("all debug flag are set to %d", flag); + if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist } int8_t taosGranted() { return atomic_load_8(&tsGrant); } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 3c08714218..756ac8167e 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -68,7 +68,7 @@ static struct { int64_t startTime; } global = {0}; -static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143, true); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143); } static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } static void dmStopDnode(int signum, void *sigInfo, void *context) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index a78d930326..d656f0c14e 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -307,6 +307,34 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, return 0; } +static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { + SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag"); + if (resetArray) { + // reset + if (pDebugFlagItem == NULL) return -1; + + // logflag names that should 'not' be set by 'debugFlag' + if (pDebugFlagItem->array == NULL) { + pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); + if (pDebugFlagItem->array == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + taosArrayClear(pDebugFlagItem->array); + return 0; + } + + // update + if (pDebugFlagItem == NULL) return -1; + if (pDebugFlagItem->array != NULL) { + SLogVar logVar = {0}; + strncpy(logVar.name, name, TSDB_LOG_VAR_LEN - 1); + taosArrayPush(pDebugFlagItem->array, &logVar); + } + return 0; +} + int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { GRANT_CFG_SET; SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -661,7 +689,6 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; - if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) { if (src[j] == 0) src[j] = ' '; @@ -931,6 +958,14 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + + size_t len = strlen(name); + const char *debugFlagStr = "debugFlag"; + const size_t debugFlagLen = strlen(debugFlagStr); + if (len >= debugFlagLen && strcasecmp(name + len - debugFlagLen, debugFlagStr) == 0) { + code = cfgUpdateDebugFlagItem(pConfig, name, len == debugFlagLen); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } } taosCloseFile(&pFile); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e113a95fcc..bd6c37a7b5 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -111,6 +111,7 @@ int32_t udfDebugFlag = 131; int32_t smaDebugFlag = 131; int32_t idxDebugFlag = 131; int32_t sndDebugFlag = 131; +int32_t simDebugFlag = 131; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h index 02c7540cde..f2360277e0 100644 --- a/utils/tsim/inc/simInt.h +++ b/utils/tsim/inc/simInt.h @@ -184,7 +184,6 @@ extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; extern SCommand simCmdList[]; extern int32_t simScriptPos; extern int32_t simScriptSucceed; -extern int32_t simDebugFlag; extern char simScriptDir[]; extern bool abortExecution; extern bool useValgrind; diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index 98f9217fd6..dcf5d6ab12 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -21,7 +21,6 @@ SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; int32_t simScriptPos = -1; int32_t simScriptSucceed = 0; -int32_t simDebugFlag = 143; void simCloseTaosdConnect(SScript *script); char simScriptDir[PATH_MAX] = {0}; @@ -32,7 +31,6 @@ int32_t simInitCfg() { taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); SConfig *pCfg = taosGetCfg(); - simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX); return 0; } From 860146d425dcea53420439ee2f2f591d7fb56400 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 3 Jan 2024 23:59:48 +0800 Subject: [PATCH 191/236] initRemainGroups --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2e93dbd803..e3f634f406 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2164,7 +2164,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool initRemainGroups = false; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { + if (tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { initRemainGroups = true; } } From f4ad4ea998d9e3b150172c39a7f1b809899c7823 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 10:01:05 +0800 Subject: [PATCH 192/236] enh: use learner in split vgroup --- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 20fb7a4d31..6eb5d7e592 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2707,8 +2707,16 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro if (newVg1.replica == 1) { if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER; + + newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER; + + newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; + + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; } else if (newVg1.replica == 3) { SVnodeGid del1 = {0}; if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER; From b85f36bcea67d833b65194333fec446413912ca3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 12:00:49 +0800 Subject: [PATCH 193/236] enh: allow to drop an empty dnode, no matter online or not --- source/dnode/mnode/impl/src/mndDnode.c | 29 +++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e224aceec2..6585e70533 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1094,6 +1094,32 @@ _OVER: return code; } +static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) { + bool isEmpty = false; + SMnodeObj *pMObj = NULL; + SQnodeObj *pQObj = NULL; + SSnodeObj *pSObj = NULL; + + pQObj = mndAcquireQnode(pMnode, dnodeId); + if (pQObj) goto _OVER; + + pSObj = mndAcquireSnode(pMnode, dnodeId); + if (pSObj) goto _OVER; + + pMObj = mndAcquireMnode(pMnode, dnodeId); + if (pMObj) goto _OVER; + + int32_t numOfVnodes = mndGetVnodesNum(pMnode, dnodeId); + if (numOfVnodes > 0) goto _OVER; + + isEmpty = true; +_OVER: + mndReleaseMnode(pMnode, pMObj); + mndReleaseQnode(pMnode, pQObj); + mndReleaseSnode(pMnode, pSObj); + return isEmpty; +} + static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -1155,7 +1181,8 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (!isonline && !force) { + bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id); + if (!isonline && !force && !isEmpty) { terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); From bd64a78094f08aa5ad9bd3c0f1d03b7ab5e6acef Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 4 Jan 2024 18:53:37 +0800 Subject: [PATCH 194/236] fix:[TD-28155] set consumer lost if all current topic is 0 --- source/dnode/mnode/impl/src/mndConsumer.c | 24 ++++++++++------------- source/dnode/mnode/impl/src/mndTrans.c | 5 ++++- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 9d9473d883..a365a44d56 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -749,18 +749,6 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { return 0; } -static void updateConsumerStatus(SMqConsumerObj *pConsumer) { - int32_t status = pConsumer->status; - - if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS_REBALANCE) { - pConsumer->status = MQ_CONSUMER_STATUS_READY; - } else if (status == MQ_CONSUMER_STATUS_READY) { - pConsumer->status = MQ_CONSUMER_STATUS_LOST; - } - } -} - // remove from new topic static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); @@ -881,7 +869,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - updateConsumerStatus(pOldConsumer); + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && + taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && + taosArrayGetSize(pOldConsumer->currentTopics) != 0){ + pOldConsumer->status = MQ_CONSUMER_STATUS_READY; + } // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = taosGetTimestampMs(); @@ -905,7 +897,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - updateConsumerStatus(pOldConsumer); + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && + taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && + taosArrayGetSize(pOldConsumer->currentTopics) == 0){ + pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; + } pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 99553fc57a..9075f0145c 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1119,7 +1119,10 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode, topHalf)) return TSDB_CODE_MND_TRANS_CTX_SWITCH; + if (mndCannotExecuteTransAction(pMnode, topHalf)) { + terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH; + return TSDB_CODE_MND_TRANS_CTX_SWITCH; + } int64_t signature = pTrans->id; signature = (signature << 32); From 4ea2cad0bb6824e315c68be890e52422b818fceb Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 4 Jan 2024 18:59:55 +0800 Subject: [PATCH 195/236] comment snode_restart_with_checkpoint.py --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1761189b19..ac21f0fbb2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -33,7 +33,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py #,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 -,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py From a811a60ad497220fad7410afb779ba0e93cabbd5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 16:07:58 +0800 Subject: [PATCH 196/236] enh: restore sync log store from snap on fsm state incomplete --- source/libs/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 52557f7b9c..9352d5662e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -790,7 +790,7 @@ int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) { SyncIndex commitIndex = snapshot.lastApplyIndex; SyncIndex firstVer = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); - if (lastVer < commitIndex || firstVer > commitIndex + 1) { + if ((lastVer < commitIndex || firstVer > commitIndex + 1) || pNode->fsmState == SYNC_FSM_STATE_INCOMPLETE) { if (pNode->pLogStore->syncLogRestoreFromSnapshot(pNode->pLogStore, commitIndex)) { sError("vgId:%d, failed to restore log store from snapshot since %s. lastVer:%" PRId64 ", snapshotVer:%" PRId64, pNode->vgId, terrstr(), lastVer, commitIndex); From 81d8b9b98b3775848c6398e136c260125f43fa3b Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 5 Jan 2024 00:35:29 +0800 Subject: [PATCH 197/236] test: add test case for create index --- source/libs/parser/src/parTranslater.c | 11 ++++------- tests/system-test/0-others/tag_index_basic.py | 8 ++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 40b869d2b7..c51e96d1b6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6930,10 +6930,11 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde } static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + int32_t code = 0; SName name; STableMeta* pMeta = NULL; - int32_t code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + + code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -6955,10 +6956,6 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS taosMemoryFree(pMeta); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, ((SColumnNode*)pNode)->colName); } - if (IS_IDX_ON(pSchema)) { - code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; - goto _exit; - } } SCreateTagIndexReq createTagIdxReq = {0}; @@ -6966,7 +6963,7 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); } -_exit: +_exit: taosMemoryFree(pMeta); return code; } diff --git a/tests/system-test/0-others/tag_index_basic.py b/tests/system-test/0-others/tag_index_basic.py index c1e1d521d2..a55508e229 100644 --- a/tests/system-test/0-others/tag_index_basic.py +++ b/tests/system-test/0-others/tag_index_basic.py @@ -88,6 +88,13 @@ class TDTestCase: tdLog.info(f" create {count} child tables ok.") + def create_tagidx_check(self, stbname): + err_dict = {"NULL","",",","\"","\"\"","undef","t1,t2","t12,t12"} + for errs in err_dict: + sql = (f'create index idx_err_check on {stbname} (%s)'% (errs)) + tdLog.info(f' sql={sql}') + tdSql.error(f'{sql}') + tdLog.info(f' create tagidx check ok.') # create stable and child tables def create_tagidx(self, stbname): @@ -231,6 +238,7 @@ class TDTestCase: count = 1000 # do self.create_table(stable, tbname, count) + self.create_tagidx_check(stable) self.create_tagidx(stable) self.insert_data(tbname) self.show_tagidx(stable) From d3aee612e0cd4418c18c90b2c99f4acf3a4a8169 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 5 Jan 2024 13:08:16 +0800 Subject: [PATCH 198/236] chore: code optimization --- source/libs/parser/src/parTranslater.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c51e96d1b6..ea1a5f7d1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6939,10 +6939,6 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS taosMemoryFree(pMeta); return code; } - if (TSDB_SUPER_TABLE != pMeta->tableType) { - taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only super table is allowed"); - } if (LIST_LENGTH(pStmt->pCols) != 1) { taosMemoryFree(pMeta); From 9eced80e49303929c8e21c1b80a2aca8a39c07b8 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Fri, 5 Jan 2024 14:33:42 +0800 Subject: [PATCH 199/236] updated cases.task --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ac21f0fbb2..7eedf505a5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -542,6 +542,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py From 8ddff2eecbe476c84c1a7aba2364e7fb1dd0bc96 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 5 Jan 2024 07:02:16 +0000 Subject: [PATCH 200/236] fix/TD-28073 --- source/dnode/mnode/impl/src/mndCompact.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 6471fa817d..101022a44f 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -26,6 +26,7 @@ #include "mndTrans.h" #define MND_COMPACT_VER_NUMBER 1 +#define MND_COMPACT_ID_LEN 11 static int32_t mndProcessCompactTimer(SRpcMsg *pReq); @@ -451,7 +452,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){ code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[10] = {0}; + char obj[MND_COMPACT_ID_LEN] = {0}; sprintf(obj, "%d", pCompact->compactId); auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); From 0b9e3c9871d530fb9b19720284ed879f457a02c3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 5 Jan 2024 16:20:55 +0800 Subject: [PATCH 201/236] fix stream state buff issue --- source/libs/executor/src/executil.c | 4 +- source/libs/stream/src/streamSessionState.c | 2 +- tests/script/tsim/stream/state1.sim | 43 +++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c1ddd6e4f4..b713b9b112 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2274,10 +2274,10 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr) { if (!pBlock) { - qDebug("%s===stream===%s: Block is Null", taskIdStr, flag); + qDebug("%s===stream===%s %s: Block is Null", taskIdStr, flag, opStr); return; } else if (pBlock->info.rows == 0) { - qDebug("%s===stream===%s: Block is Empty. type:%d", taskIdStr, flag, pBlock->info.type); + qDebug("%s===stream===%s %s: Block is Empty. block type %d", taskIdStr, flag, opStr, pBlock->info.type); return; } if (qDebugFlag & DEBUG_DEBUG) { diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 765403b1aa..5f19b47e1f 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -557,7 +557,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch void* pFileStore = getStateFileStore(pFileState); void* p = NULL; int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); - if (code_file == TSDB_CODE_SUCCESS) { + if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim index 775528bd4e..e6f7a81851 100644 --- a/tests/script/tsim/stream/state1.sim +++ b/tests/script/tsim/stream/state1.sim @@ -152,6 +152,49 @@ endi print step 2 over +print step 3 + +sql create database test3 vgroups 1; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0); + +print create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); +sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); + +sleep 1000 + +sql insert into t1 values(1648791203000,2,2,3,1.0); + +sleep 500 + +sql insert into t1 values(1648791214000,1,2,3,1.0); + +$loop_count = 0 +loop6: + +sleep 1000 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select * from streamt3 +sql select * from streamt3; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +print step 3 over + print state1 end system sh/stop_dnodes.sh From db9655c54999f7830ab224ab397b014768a8e74f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 5 Jan 2024 17:07:24 +0800 Subject: [PATCH 202/236] test: save case.sql when case is failed and set default saved sqlfile True --- tests/parallel_test/run.sh | 4 ++++ tests/pytest/util/sql.py | 4 ++-- tests/system-test/1-insert/influxdb_line_taosc_insert.py | 2 +- tests/system-test/1-insert/insert_double.py | 2 +- tests/system-test/1-insert/insert_perf.py | 2 +- tests/system-test/1-insert/mutil_stage.py | 2 +- .../1-insert/opentsdb_telnet_line_taosc_insert.py | 2 +- tests/system-test/1-insert/time_range_wise.py | 2 +- tests/system-test/2-query/abs.py | 2 +- tests/system-test/2-query/and_or_for_byte.py | 2 +- tests/system-test/2-query/apercentile.py | 2 +- tests/system-test/2-query/case_when.py | 2 +- tests/system-test/2-query/check_tsdb.py | 2 +- tests/system-test/2-query/count.py | 2 +- tests/system-test/2-query/countAlwaysReturnValue.py | 2 +- tests/system-test/2-query/db.py | 2 +- tests/system-test/2-query/fill_with_group.py | 2 +- tests/system-test/2-query/func_to_char_timestamp.py | 2 +- tests/system-test/2-query/function_null.py | 2 +- tests/system-test/2-query/histogram.py | 2 +- tests/system-test/2-query/interval_limit_opt.py | 2 +- tests/system-test/2-query/interval_limit_opt_2.py | 2 +- tests/system-test/2-query/interval_unit.py | 2 +- tests/system-test/2-query/irate.py | 2 +- tests/system-test/2-query/join.py | 2 +- tests/system-test/2-query/join2.py | 2 +- tests/system-test/2-query/json_tag.py | 2 +- tests/system-test/2-query/json_tag_large_tables.py | 2 +- tests/system-test/2-query/last_cache_scan.py | 5 +++-- tests/system-test/2-query/like.py | 2 +- tests/system-test/2-query/limit.py | 2 +- tests/system-test/2-query/odbc.py | 2 +- tests/system-test/2-query/partition_by_col.py | 2 +- tests/system-test/2-query/partition_by_col_agg.py | 2 +- tests/system-test/2-query/qnodeCluster.py | 2 +- tests/system-test/2-query/query_cols_tags_and_or.py | 2 +- tests/system-test/2-query/state_window.py | 2 +- tests/system-test/2-query/substr.py | 2 +- tests/system-test/2-query/ts_3405_3398_3423.py | 2 +- tests/system-test/2-query/tsbsQuery.py | 2 +- tests/system-test/2-query/ttl_comment.py | 2 +- tests/system-test/2-query/varbinary.py | 2 +- tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py | 2 +- tests/system-test/7-tmq/dataFromTsdbNWal.py | 2 +- tests/system-test/7-tmq/stbTagFilter-1ctb.py | 2 +- tests/system-test/7-tmq/stbTagFilter-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- tests/system-test/7-tmq/tmqClientConsLog.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py | 2 +- .../7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py | 2 +- .../7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1.py | 2 +- tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py | 2 +- tests/system-test/7-tmq/tmqDataPrecisionUnit.py | 2 +- tests/system-test/7-tmq/tmqDelete-1ctb.py | 2 +- tests/system-test/7-tmq/tmqDelete-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart1.py | 2 +- tests/system-test/7-tmq/tmqDropConsumer.py | 2 +- tests/system-test/7-tmq/tmqDropNtb-snapshot0.py | 2 +- tests/system-test/7-tmq/tmqDropNtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqDropStbCtb.py | 2 +- tests/system-test/7-tmq/tmqMaxGroupIds.py | 2 +- tests/system-test/7-tmq/tmqMaxTopic.py | 2 +- tests/system-test/7-tmq/tmqMultiConsumer.py | 2 +- tests/system-test/7-tmq/tmqOffset.py | 2 +- tests/system-test/7-tmq/tmqSeekAndCommit.py | 2 +- tests/system-test/7-tmq/tmqUpdate-1ctb.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqUpdateWithConsume.py | 2 +- tests/system-test/7-tmq/tmqVnodeReplicate.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-column.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-db.py | 2 +- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py | 2 +- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-stb.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-db.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform.py | 2 +- tests/system-test/7-tmq/tmq_per.py | 2 +- tests/system-test/8-stream/snode_restart.py | 2 +- tests/system-test/8-stream/snode_restart_with_checkpoint.py | 2 +- tests/system-test/8-stream/vnode_restart.py | 2 +- tests/system-test/99-TDcase/TD-17255.py | 2 +- 95 files changed, 101 insertions(+), 96 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index f05e0dfc83..cf21349513 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -200,6 +200,7 @@ function run_thread() { if [ -z "$case_file" ]; then continue fi + case_sql_file="$exec_dir/${case_file}.sql" case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" count=$(( count + 1 )) local case_path=`dirname "$case_file"` @@ -324,12 +325,15 @@ function run_thread() { cmd="$tarcmd sh -c \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" $cmd local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" + local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file)" scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" if [ -z ${passwords[index]} ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" $cmd + cmd="$scpcmd:${remote_case_sql_file} $log_dir/${case_file}.sql" + $cmd # backup source code (disabled) source_tar_dir=$log_dir/TDengine_${hosts[index]} source_tar_file=TDengine.tar.gz diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index f8a49983e1..000040d958 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -48,9 +48,9 @@ class TDSql: self.queryCols = 0 self.affectedRows = 0 - def init(self, cursor, log=False): + def init(self, cursor, log=True): self.cursor = cursor - + print(f"sqllog is :{log}") if (log): caller = inspect.getframeinfo(inspect.stack()[1][0]) self.cursor.log(caller.filename + ".sql") diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 3e9bad4efd..85a0afed48 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self._conn = conn def createDb(self, name="test", db_update_tag=0): diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index b47b22ab44..b7af7237db 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -15,7 +15,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) self.database = "db1" tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_db(self): tdSql.execute(f"drop database if exists {self.database}") diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py index d96a031458..dac2afb3b2 100644 --- a/tests/system-test/1-insert/insert_perf.py +++ b/tests/system-test/1-insert/insert_perf.py @@ -30,7 +30,7 @@ class TDTestCase: self.tb3 = "t3" self.once = 1000 tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_db(self): tdSql.execute(f"drop database if exists {self.database}") diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 970554c6c1..ebaa1e7dc3 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -60,7 +60,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.taos_cfg_path = tdDnodes.dnodes[0].cfgPath self.taos_data_dir = tdDnodes.dnodes[0].dataDir diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index d419aee12c..2d9e34c287 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -33,7 +33,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self._conn = conn self.smlChildTableName_value = "id" diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 14b717d1da..a3df0eb6b0 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -137,7 +137,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.precision = "ms" self.sma_count = 0 self.sma_created_index = [] diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d64d550bc4..0eabd91535 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -17,7 +17,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 15e9110b3b..5b2fb51998 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -17,7 +17,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/apercentile.py b/tests/system-test/2-query/apercentile.py index e190bf9374..ee6835bd10 100644 --- a/tests/system-test/2-query/apercentile.py +++ b/tests/system-test/2-query/apercentile.py @@ -21,7 +21,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) self.rowNum = 10 self.ts = 1537146000000 self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/case_when.py b/tests/system-test/2-query/case_when.py index eebbcc3fa3..1ccd2b5076 100755 --- a/tests/system-test/2-query/case_when.py +++ b/tests/system-test/2-query/case_when.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) + tdSql.init(conn.cursor()) self.testcasePath = os.path.split(__file__)[0] self.testcaseFilename = os.path.split(__file__)[-1] diff --git a/tests/system-test/2-query/check_tsdb.py b/tests/system-test/2-query/check_tsdb.py index b3ff259cc5..c49c4f1481 100644 --- a/tests/system-test/2-query/check_tsdb.py +++ b/tests/system-test/2-query/check_tsdb.py @@ -15,7 +15,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_datas(self, dbname="db"): tdSql.execute( diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 3ba651622a..c06ee28d02 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -6,7 +6,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) self.setsql = TDSetSql() self.rowNum = 10 self.ts = 1537146000000 diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index 244964e486..f22fa3e01e 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_data(self, dbname="db"): tdSql.execute( diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index a48b4cf3bb..07514a0950 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): tdSql.execute("create database if not exists dbms precision 'ms'") diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index fce504d0f7..05be2ec9d2 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/func_to_char_timestamp.py b/tests/system-test/2-query/func_to_char_timestamp.py index d955e00a82..034d025fcd 100644 --- a/tests/system-test/2-query/func_to_char_timestamp.py +++ b/tests/system-test/2-query/func_to_char_timestamp.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/function_null.py b/tests/system-test/2-query/function_null.py index 44b86ee543..e5056b7c56 100644 --- a/tests/system-test/2-query/function_null.py +++ b/tests/system-test/2-query/function_null.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/histogram.py b/tests/system-test/2-query/histogram.py index cc6e8fe7e9..a69fb505d1 100644 --- a/tests/system-test/2-query/histogram.py +++ b/tests/system-test/2-query/histogram.py @@ -147,7 +147,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __create_hsg(self, sma:Hsgschema): return f"""{sma.histogram_flag}({sma.col}, '{sma.bin_type}', '{sma.bin_desc}', {sma.normalized})""" diff --git a/tests/system-test/2-query/interval_limit_opt.py b/tests/system-test/2-query/interval_limit_opt.py index 492f453de5..aa1702fe3c 100644 --- a/tests/system-test/2-query/interval_limit_opt.py +++ b/tests/system-test/2-query/interval_limit_opt.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/interval_limit_opt_2.py b/tests/system-test/2-query/interval_limit_opt_2.py index cadb32b388..cfabe46dca 100644 --- a/tests/system-test/2-query/interval_limit_opt_2.py +++ b/tests/system-test/2-query/interval_limit_opt_2.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/interval_unit.py b/tests/system-test/2-query/interval_unit.py index 9430c74cff..03917bfaf7 100644 --- a/tests/system-test/2-query/interval_unit.py +++ b/tests/system-test/2-query/interval_unit.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d', precision: str='ms'): if dropFlag == 1: diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index d882c9b8fb..82841541f0 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 120cc2eb30..c5c8f6c730 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -66,7 +66,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 7ddc5e57ba..1006033eae 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 8db92f38a2..33209620e1 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): # tdSql.prepare() diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py index ea3963640a..86bd6c2d82 100644 --- a/tests/system-test/2-query/json_tag_large_tables.py +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -36,7 +36,7 @@ class TDTestCase: self.testcaseFilename = os.path.split(__file__)[-1] # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): # tdSql.prepare() diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 23433277c2..9b6dca56fd 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -376,7 +376,7 @@ class TDTestCase: tdSql.checkCols(1) p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"]) p.check_returncode() - tdSql.query_success_failed("select ts, last_row(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") + tdSql.query_success_failed("select ts, last(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) tdSql.checkRows(1) tdSql.checkCols(3) @@ -413,7 +413,8 @@ class TDTestCase: tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) - tdSql.query_success_failed('select last(c2), c2, c3 from meters', queryTimes=10, expectErrInfo="Invalid column name: c2") + + def run(self): self.prepareTestEnv() diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py index 95e2daf6d3..65d0a36b24 100644 --- a/tests/system-test/2-query/like.py +++ b/tests/system-test/2-query/like.py @@ -13,7 +13,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def initDB(self): tdSql.execute("drop database if exists db") diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index 961cff5087..e6736704f7 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1): if dropFlag == 1: diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index 8fbad93995..7daf5fd730 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -13,7 +13,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def check_ins_cols(self): tdSql.execute("create database if not exists db") diff --git a/tests/system-test/2-query/partition_by_col.py b/tests/system-test/2-query/partition_by_col.py index 230b7582d9..feb8693e3e 100644 --- a/tests/system-test/2-query/partition_by_col.py +++ b/tests/system-test/2-query/partition_by_col.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/partition_by_col_agg.py b/tests/system-test/2-query/partition_by_col_agg.py index 1bc7a2414a..405cff9663 100644 --- a/tests/system-test/2-query/partition_by_col_agg.py +++ b/tests/system-test/2-query/partition_by_col_agg.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index d15ef89eb8..851b5bb1d1 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): tsql.execute("use %s" %dbName) diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index b8d4e3c314..d86b5d07e7 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -20,7 +20,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) ## add for TD-6672 tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def insertData(self, tb_name): insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1, 2, 3, 4)', diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py index 2bf363f0dc..7dedeb88f1 100644 --- a/tests/system-test/2-query/state_window.py +++ b/tests/system-test/2-query/state_window.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 302711b13d..ad5a2bba55 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) def __substr_condition(self): # sourcery skip: extract-method substr_condition = [] diff --git a/tests/system-test/2-query/ts_3405_3398_3423.py b/tests/system-test/2-query/ts_3405_3398_3423.py index dcd8fb5a85..814d9de394 100644 --- a/tests/system-test/2-query/ts_3405_3398_3423.py +++ b/tests/system-test/2-query/ts_3405_3398_3423.py @@ -11,7 +11,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): """This test case is used to verify the query performance for the merge scans process of diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 0e2975cdba..5a49f653ec 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_ctable(self,tsql=None, dbName='db',stbName='stb',ctbPrefix='ctb',ctbNum=1): tsql.execute("use %s" %dbName) diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py index 3ae602fa23..ba24579611 100644 --- a/tests/system-test/2-query/ttl_comment.py +++ b/tests/system-test/2-query/ttl_comment.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): dbname="db" diff --git a/tests/system-test/2-query/varbinary.py b/tests/system-test/2-query/varbinary.py index 8f8021c8ad..c2e2693762 100644 --- a/tests/system-test/2-query/varbinary.py +++ b/tests/system-test/2-query/varbinary.py @@ -10,7 +10,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def test(self): tdLog.info(" test") diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 808a4935e3..cf6aafc374 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 8386c22cd0..e0fbe89c36 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 1e6011d5db..15fd4ed875 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 5043c46f00..c7af55daae 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 5d0af636b2..e3fe9ae0b6 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py index d708e6642c..ab0c5f6a72 100644 --- a/tests/system-test/7-tmq/tmqClientConsLog.py +++ b/tests/system-test/7-tmq/tmqClientConsLog.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index f372a2b742..2af61f4bbb 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index c7f95f6f41..d3087c0455 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 26dacf514d..275d80ca18 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index d6f100041b..0572499079 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 11fc7dbcc0..7b31019572 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -26,7 +26,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 8ed4a6df97..a05b6a23ec 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 117c3ce637..85adb50e88 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 2864240441..facbb6c228 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index a717c4966d..ede17130df 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index d8606efe58..fa7aad9dae 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 05aa82c929..efc186c3bf 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index dcaa6ceb7c..f66605f5f4 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py index dd2420dcfb..d94f8df35c 100644 --- a/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py +++ b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqDataPrecisionUnit.py b/tests/system-test/7-tmq/tmqDataPrecisionUnit.py index f050116a1b..13b0fcbe5d 100644 --- a/tests/system-test/7-tmq/tmqDataPrecisionUnit.py +++ b/tests/system-test/7-tmq/tmqDataPrecisionUnit.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.db_name = "tmq_db" self.topic_name = "tmq_topic" diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index aa9c8d25d0..117bd66340 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index 7a47cd6025..0dc2dc35f0 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 0ac8482163..f4a63de53e 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 2bde32800b..b398178c61 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDropConsumer.py b/tests/system-test/7-tmq/tmqDropConsumer.py index e3e9906ecf..5208d14069 100644 --- a/tests/system-test/7-tmq/tmqDropConsumer.py +++ b/tests/system-test/7-tmq/tmqDropConsumer.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index c8bcdd6235..2c87e4083c 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) # drop some ntbs def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 3fc5a2fdc7..7a35867ce7 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) # drop some ntbs def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index c9e34136cc..eacacd913b 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqMaxGroupIds.py b/tests/system-test/7-tmq/tmqMaxGroupIds.py index 5049ee5bd7..eba15a4bef 100644 --- a/tests/system-test/7-tmq/tmqMaxGroupIds.py +++ b/tests/system-test/7-tmq/tmqMaxGroupIds.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 05b699ca00..4f25c39627 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def modifyMaxTopics(self, tmqMaxTopicNum): # single dnode diff --git a/tests/system-test/7-tmq/tmqMultiConsumer.py b/tests/system-test/7-tmq/tmqMultiConsumer.py index cc217e0c4c..e9727042e2 100644 --- a/tests/system-test/7-tmq/tmqMultiConsumer.py +++ b/tests/system-test/7-tmq/tmqMultiConsumer.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqOffset.py b/tests/system-test/7-tmq/tmqOffset.py index 500c6f53e4..991d1185a9 100644 --- a/tests/system-test/7-tmq/tmqOffset.py +++ b/tests/system-test/7-tmq/tmqOffset.py @@ -31,7 +31,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqSeekAndCommit.py b/tests/system-test/7-tmq/tmqSeekAndCommit.py index 253edcd10d..65ee438a41 100644 --- a/tests/system-test/7-tmq/tmqSeekAndCommit.py +++ b/tests/system-test/7-tmq/tmqSeekAndCommit.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.db_name = "tmq_db" self.topic_name = "tmq_topic" diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 920e8e77e4..216f67ffe4 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index 2f1d3e2631..e51ac9f029 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index 6b8c10de27..3988565154 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index 3975013e74..86e49d378a 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index 6a9bb0ae92..e88e16eec0 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqVnodeReplicate.py b/tests/system-test/7-tmq/tmqVnodeReplicate.py index 0ee11781ed..3e40cb13ad 100644 --- a/tests/system-test/7-tmq/tmqVnodeReplicate.py +++ b/tests/system-test/7-tmq/tmqVnodeReplicate.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 1fe2b5809a..74213c4536 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index f66acf4fcd..b9eac4b692 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py index 68fb07b813..f6b523e5f4 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 6140e8a544..d4c76c4f61 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index 18b80b7f8d..e79a0ca77d 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py index c203350322..a96d52ee2b 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py index 5c61908d96..cef90ff65f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-db.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 64cdf6d153..95703c52ca 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index 3698297618..811b72c35f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmq_per.py b/tests/system-test/7-tmq/tmq_per.py index f3701dacab..bd319aa3a4 100644 --- a/tests/system-test/7-tmq/tmq_per.py +++ b/tests/system-test/7-tmq/tmq_per.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/8-stream/snode_restart.py b/tests/system-test/8-stream/snode_restart.py index 3657163ab0..e80ff54578 100644 --- a/tests/system-test/8-stream/snode_restart.py +++ b/tests/system-test/8-stream/snode_restart.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/8-stream/snode_restart_with_checkpoint.py b/tests/system-test/8-stream/snode_restart_with_checkpoint.py index d7bfd7b407..9eb8c09ca3 100644 --- a/tests/system-test/8-stream/snode_restart_with_checkpoint.py +++ b/tests/system-test/8-stream/snode_restart_with_checkpoint.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/8-stream/vnode_restart.py b/tests/system-test/8-stream/vnode_restart.py index a53432b77a..be0ee81787 100644 --- a/tests/system-test/8-stream/vnode_restart.py +++ b/tests/system-test/8-stream/vnode_restart.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 0f83468754..fa28372ae3 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") From f2a1d24ab1a5a6c7b19a1ffdbafef4ebc44846b0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 5 Jan 2024 19:28:40 +0800 Subject: [PATCH 203/236] fix:[TD-28155] set consumer lost if all current topic is 0 --- source/dnode/mnode/impl/src/mndConsumer.c | 24 +++++++++++++--------- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index a365a44d56..4db000287c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -749,6 +749,18 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { return 0; } +static void updateConsumerStatus(SMqConsumerObj *pConsumer) { + int32_t status = pConsumer->status; + + if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { + if (status == MQ_CONSUMER_STATUS_REBALANCE) { + pConsumer->status = MQ_CONSUMER_STATUS_READY; + } else if (status == MQ_CONSUMER_STATUS_READY && taosArrayGetSize(pConsumer->currentTopics) == 0) { + pConsumer->status = MQ_CONSUMER_STATUS_LOST; + } + } +} + // remove from new topic static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); @@ -869,11 +881,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && - taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && - taosArrayGetSize(pOldConsumer->currentTopics) != 0){ - pOldConsumer->status = MQ_CONSUMER_STATUS_READY; - } + updateConsumerStatus(pOldConsumer); // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = taosGetTimestampMs(); @@ -897,11 +905,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && - taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && - taosArrayGetSize(pOldConsumer->currentTopics) == 0){ - pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; - } + updateConsumerStatus(pOldConsumer); pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 16ae7f2548..62b671a12f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -737,8 +737,6 @@ static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { SMqConsumerObj *pConsumer; void *pIter = NULL; - mInfo("start to process mq timer"); - // iterate all consumers, find all modification while (1) { pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); @@ -852,6 +850,8 @@ void mndRebCntDec() { static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { int code = 0; + mInfo("start to process mq timer"); + if (!mndRebTryStart()) { mInfo("mq rebalance already in progress, do nothing"); return code; From 0aef021d885e313c1df2b51ac7acf47c0cbce44b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 6 Jan 2024 21:01:34 +0800 Subject: [PATCH 204/236] fix: backslash add G only valid in string end --- tools/shell/src/shellEngine.c | 21 ++++++++++++++++++++- tools/shell/src/shellWebsocket.c | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 60b1dfab41..cec4a9683c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -195,6 +195,25 @@ int32_t shellRunCommand(char *command, bool recordHistory) { return shellRunSingleCommand(cmd); } + +char * strendG(const char* pstr) { + if(pstr == NULL) { + return NULL; + } + + size_t len = strlen(pstr); + if(len < 3) { + return NULL; + } + + char * p = pstr + len - 2 + if (strcmp(p, "\\G") == 0) { + return p; + } + + return NULL; +} + void shellRunSingleCommandImp(char *command) { int64_t st, et; char *sptr = NULL; @@ -213,7 +232,7 @@ void shellRunSingleCommandImp(char *command) { } } - if ((sptr = strstr(command, "\\G")) != NULL) { + if ((sptr = strendG(command)) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index bff2ef7592..8c3a46aa05 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -238,6 +238,7 @@ static int shellDumpWebsocket(WS_RES *wres, char *fname, return numOfRows; } +char * strendG(const char* pstr); void shellRunSingleCommandWebsocketImp(char *command) { int64_t st, et; char *sptr = NULL; @@ -256,7 +257,7 @@ void shellRunSingleCommandWebsocketImp(char *command) { } } - if ((sptr = strstr(command, "\\G")) != NULL) { + if ((sptr = strendG(command)) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. } From 96947e24d702d9ae28c473a723d5a249f6286127 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 6 Jan 2024 23:24:36 +0800 Subject: [PATCH 205/236] fix: modify build error --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index cec4a9683c..563481ede7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,7 +206,7 @@ char * strendG(const char* pstr) { return NULL; } - char * p = pstr + len - 2 + char * p = (char *)pstr + len - 2; if (strcmp(p, "\\G") == 0) { return p; } From 772a644de0a4262b6a0b5a9da90b496e2cd8c137 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Jan 2024 23:41:52 +0800 Subject: [PATCH 206/236] fix: end must G; --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 563481ede7..d3a2dfdf79 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -202,7 +202,7 @@ char * strendG(const char* pstr) { } size_t len = strlen(pstr); - if(len < 3) { + if(len < 4) { return NULL; } @@ -210,6 +210,11 @@ char * strendG(const char* pstr) { if (strcmp(p, "\\G") == 0) { return p; } + + p = (char *)pstr + len - 3; + if (strcmp(p, "\\G;") == 0 ){ + return p; + } return NULL; } From b38a8102655d8211977b2c753e8e1e0df2133338 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Jan 2024 23:43:55 +0800 Subject: [PATCH 207/236] fix: end must G; fix --- tools/shell/src/shellEngine.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d3a2dfdf79..060e28282e 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,12 +206,7 @@ char * strendG(const char* pstr) { return NULL; } - char * p = (char *)pstr + len - 2; - if (strcmp(p, "\\G") == 0) { - return p; - } - - p = (char *)pstr + len - 3; + char * p = (char *)pstr + len - 3; if (strcmp(p, "\\G;") == 0 ){ return p; } From fbfa1c9abe901f6fefc3d9e180461ff53d11110f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 7 Jan 2024 15:10:57 +0800 Subject: [PATCH 208/236] fix: use \G have no ; end --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 563481ede7..02a3084d4f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -202,7 +202,7 @@ char * strendG(const char* pstr) { } size_t len = strlen(pstr); - if(len < 3) { + if(len < 4) { return NULL; } @@ -211,6 +211,11 @@ char * strendG(const char* pstr) { return p; } + p = (char *)pstr + len - 3; + if (strcmp(p, "\\G;") == 0) { + return p; + } + return NULL; } From d3b13bbb84d41c1c72bea877509da431c5c9a1f3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 7 Jan 2024 15:15:56 +0800 Subject: [PATCH 209/236] fix: git add G no ; --- tools/shell/src/shellEngine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f077c7817..f6f834f3d7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,8 +206,8 @@ char * strendG(const char* pstr) { return NULL; } - char * p = (char *)pstr + len - 3; - if (strcmp(p, "\\G;") == 0 ){ + char * p = (char *)pstr + len - 2; + if (strcmp(p, "\\G") == 0 ){ return p; } From 96460914923c2263accf2c2c774cfd00b52929db Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 7 Jan 2024 16:18:24 +0800 Subject: [PATCH 210/236] fix: sort error due to out of space --- source/libs/executor/src/tsort.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 205cd7d3ef..ee1d831a24 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -878,6 +878,9 @@ static int32_t blockCompareTsFn(const void* pLeft, const void* pRight, void* par static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, SArray* aPgId) { int32_t pageId = -1; void* pPage = getNewBufPage(pHandle->pBuf, &pageId); + if (pPage == NULL) { + return terrno; + } taosArrayPush(aPgId, &pageId); int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); @@ -944,16 +947,15 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO totalRows += blk->info.rows; } - SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); - SMultiwayMergeTreeInfo* pTree = NULL; code = tMergeTreeCreate(&pTree, taosArrayGetSize(aBlk), &sup, blockCompareTsFn); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(sup.aRowIdx); taosMemoryFree(sup.aTs); - return code; } + + SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); int32_t nRows = 0; int32_t nMergedRows = 0; bool mergeLimitReached = false; @@ -969,7 +971,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) { SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; - appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pTree); + taosArrayDestroy(aPgId); + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); + return code; + } nMergedRows += pHandle->pDataBlock->info.rows; blockDataCleanup(pHandle->pDataBlock); blkPgSz = pgHeaderSz; @@ -1001,7 +1010,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO if (!mergeLimitReached) { SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; - appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(aPgId); + taosMemoryFree(pTree); + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); + return code; + } nMergedRows += pHandle->pDataBlock->info.rows; if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { mergeLimitReached = true; From 1a02bb4232a36f12bf13f3b88f1066c1673d4a7f Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 7 Jan 2024 18:25:30 +0800 Subject: [PATCH 211/236] doc: fix insert stb syntax --- docs/en/12-taos-sql/05-insert.md | 2 +- docs/zh/12-taos-sql/05-insert.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index 57e15746cc..462e7fc0ae 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -157,7 +157,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c Automatically creating table and the table name is specified through the `tbname` column ```sql -INSERT INTO meters(tbname, location, groupId, ts, current, phase) +INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index 1f31698d88..fa330cabae 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -157,7 +157,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c 自动建表, 表名通过tbname列指定 ```sql -INSERT INTO meters(tbname, location, groupId, ts, current, phase) +INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) From c5932c9d1c6226863b0e960ec7174a284044bf87 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 8 Jan 2024 01:13:53 +0800 Subject: [PATCH 212/236] fix: wildcard character in both target and pattern --- source/util/src/tcompare.c | 22 ++++++++++------ tests/system-test/2-query/like.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 2baa21f5d9..53fdc96c37 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1102,10 +1102,13 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t c1 = str[j++]; ++nMatchChar; - if (c == '\\' && pattern[i] == c1 && - (c1 == '_' || c1 == '%')) { - i++; - continue; + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + if (c1 != pattern[i]) { + return TSDB_PATTERN_NOMATCH; + } else { + i++; + continue; + } } if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { @@ -1176,10 +1179,13 @@ int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, c1 = str[j++]; nMatchChar++; - if (c == '\\' && pattern[i] == c1 && - (c1 == '_' || c1 == '%')) { - i++; - continue; + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + if (c1 != pattern[i]) { + return TSDB_PATTERN_NOMATCH; + } else { + i++; + continue; + } } if (c == c1 || towlower(c) == towlower(c1) || (c == pInfo->umatchOne && c1 != 0)) { diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py index 95e2daf6d3..88c1e3b8dd 100644 --- a/tests/system-test/2-query/like.py +++ b/tests/system-test/2-query/like.py @@ -126,6 +126,47 @@ class TDTestCase: tdSql.query("select * from db.t4x where c1 like '\%\_%\%%'") tdSql.checkRows(1) + def like_wildcard_test2(self): + tdSql.execute("create table db.t5x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t5x values(now(), 'a\%c')") + tdSql.execute("insert into db.t5x values(now+1s, 'a\%bbbc')") + tdSql.execute("insert into db.t5x values(now()+2s, 'a%c')") + + tdSql.query("select * from db.t5x where c1 like 'a\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a%c") + + tdSql.execute("create table db.t6x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t6x values(now(), '\%c')") + tdSql.execute("insert into db.t6x values(now+1s, '\%bbbc')") + tdSql.execute("insert into db.t6x values(now()+2s, '%c')") + + tdSql.query("select * from db.t6x where c1 like '\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "%c") + + tdSql.execute("create table db.t7x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t7x values(now(), 'a\_c')") + tdSql.execute("insert into db.t7x values(now+1s, 'a\_bbbc')") + tdSql.execute("insert into db.t7x values(now()+2s, 'a_c')") + + tdSql.query("select * from db.t7x where c1 like 'a\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a_c") + + tdSql.execute("create table db.t8x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t8x values(now(), '\_c')") + tdSql.execute("insert into db.t8x values(now+1s, '\_bbbc')") + tdSql.execute("insert into db.t8x values(now()+2s, '_c')") + + tdSql.query("select * from db.t8x where c1 like '\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "_c") + def run(self): tdLog.printNoPrefix("==========start like_wildcard_test run ...............") tdSql.prepare(replica = self.replicaVar) @@ -135,6 +176,7 @@ class TDTestCase: self.like_wildcard_test() self.like_cnc_wildcard_test() self.like_multi_wildcard_test() + self.like_wildcard_test2() tdLog.printNoPrefix("==========end like_wildcard_test run ...............") self.stopTest() From 6b38e5b8fb083b8e784ae9cee778e00e40e732cb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 8 Jan 2024 10:44:19 +0800 Subject: [PATCH 213/236] feat: websocket support bi with appending &conn_mode=1 string to dsn --- tools/shell/src/shellEngine.c | 4 +--- tools/shell/src/shellWebsocket.c | 11 ++++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index f6f834f3d7..23424cea98 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1318,9 +1318,7 @@ int32_t shellExecute() { if(shell.args.is_bi_mode) { // need set bi mode printf("Set BI mode is true.\n"); -#ifdef WEBSOCKET - //ws_taos_set_conn_mode(shell.ws_conn, TAOS_CONN_MODE_BI, 1); -#else +#ifndef WEBSOCKET taos_set_conn_mode(shell.conn, TAOS_CONN_MODE_BI, 1); #endif } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 8c3a46aa05..2d3aa27a3b 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -30,7 +30,16 @@ int shell_conn_ws_server(bool first) { fprintf(stdout, "trying to connect %s****** ", cuttedDsn); fflush(stdout); for (int i = 0; i < shell.args.timeout; i++) { - shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + if(shell.args.is_bi_mode) { + size_t len = strlen(shell.args.dsn); + char * dsn = taosMemoryMalloc(len + 32); + sprintf(dsn, "%s&conn_mode=1", shell.args.dsn); + shell.ws_conn = ws_connect_with_dsn(dsn); + taosMemoryFree(dsn); + } else { + shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + } + if (NULL == shell.ws_conn) { int errNo = ws_errno(NULL); if (0xE001 == errNo) { From b4f869d051d62d6b68755e80ce8193691ca9b8c1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 8 Jan 2024 11:19:54 +0800 Subject: [PATCH 214/236] case: s3_basic.py except arm64 --- tests/army/enterprise/s3/s3_basic.py | 3 +++ tests/army/frame/eos.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 389b8a52c7..112cb65b15 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -84,6 +84,9 @@ class TDTestCase(TBase): # run def run(self): tdLog.debug(f"start to excute {__file__}") + if eos.isArm64Cpu(): + tdLog.success(f"{__file__} arm64 ignore executed") + return # insert data self.insertData() diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index b24be74393..1c00f94380 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -22,10 +22,31 @@ import datetime import platform import subprocess +# +# platform +# + # if windows platform return True def isWin(): return platform.system().lower() == 'windows' +def isArm64Cpu(): + system = platform.system() + + if system == 'Linux': + machine = platform.machine().lower() + + # Check for ARM64 architecture on Linux systems + return machine in ['aarch64', 'armv8l'] + elif system == 'Darwin' or system == 'Windows': + processor = platform.processor().lower() + + # Check for ARM64 architecture on macOS and Windows systems + return processor in ['arm64', 'aarch64'] + else: + print("Unsupported operating system") + return False + # # execute programe # From 8cdd28131f642a5d087b04e6d3817db8c4377a0e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 12:29:09 +0800 Subject: [PATCH 215/236] test: save case.sql when case is failed and set default saved sqlfile True --- tests/parallel_test/run.sh | 330 +++++++++++++++++++----------------- tests/system-test/pytest.sh | 12 +- 2 files changed, 176 insertions(+), 166 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index cf21349513..ef9413a9d5 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -15,61 +15,61 @@ function usage() { ent=0 while getopts "m:t:b:l:o:w:eh" opt; do case $opt in - m) - config_file=$OPTARG - ;; - t) - t_file=$OPTARG - ;; - b) - branch=$OPTARG - ;; - l) - log_dir=$OPTARG - ;; - e) - ent=1 - ;; - o) - timeout_param="-o $OPTARG" - ;; - w) - web_server=$OPTARG - ;; - h) - usage - exit 0 - ;; - \?) - echo "Invalid option: -$OPTARG" - usage - exit 0 - ;; + m) + config_file=$OPTARG + ;; + t) + t_file=$OPTARG + ;; + b) + branch=$OPTARG + ;; + l) + log_dir=$OPTARG + ;; + e) + ent=1 + ;; + o) + timeout_param="-o $OPTARG" + ;; + w) + web_server=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; esac done #config_file=$1 -if [ -z $config_file ]; then +if [ -z "$config_file" ]; then usage exit 1 fi -if [ ! -f $config_file ]; then +if [ ! -f "$config_file" ]; then echo "$config_file not found" usage exit 1 fi #t_file=$2 -if [ -z $t_file ]; then +if [ -z "$t_file" ]; then usage exit 1 fi -if [ ! -f $t_file ]; then +if [ ! -f "$t_file" ]; then echo "$t_file not found" usage exit 1 fi -date_tag=`date +%Y%m%d-%H%M%S` +date_tag=$(date +%Y%m%d-%H%M%S) test_log_dir=${branch}_${date_tag} -if [ -z $log_dir ]; then +if [ -z "$log_dir" ]; then log_dir="log/${test_log_dir}" else log_dir="$log_dir/${test_log_dir}" @@ -82,42 +82,41 @@ workdirs=() threads=() i=0 -while [ 1 ]; do - host=`jq .[$i].host $config_file` +while true; do + host=$(jq .[$i].host "$config_file") if [ "$host" = "null" ]; then break fi - username=`jq .[$i].username $config_file` + username=$(jq .[$i].username "$config_file") if [ "$username" = "null" ]; then break fi - password=`jq .[$i].password $config_file` + password=$(jq .[$i].password "$config_file") if [ "$password" = "null" ]; then password="" fi - workdir=`jq .[$i].workdir $config_file` + workdir=$(jq .[$i].workdir "$config_file") if [ "$workdir" = "null" ]; then break fi - thread=`jq .[$i].thread $config_file` + thread=$(jq .[$i].thread "$config_file") if [ "$thread" = "null" ]; then break fi - hosts[i]=`echo $host|sed 's/\"$//'|sed 's/^\"//'` - usernames[i]=`echo $username|sed 's/\"$//'|sed 's/^\"//'` - passwords[i]=`echo $password|sed 's/\"$//'|sed 's/^\"//'` - workdirs[i]=`echo $workdir|sed 's/\"$//'|sed 's/^\"//'` + hosts[i]=$(echo "$host" | sed 's/\"$//' | sed 's/^\"//') + usernames[i]=$(echo "$username" | sed 's/\"$//' | sed 's/^\"//') + passwords[i]=$(echo "$password" | sed 's/\"$//' | sed 's/^\"//') + workdirs[i]=$(echo "$workdir" | sed 's/\"$//' | sed 's/^\"//') threads[i]=$thread - i=$(( i + 1 )) + i=$((i + 1)) done - function prepare_cases() { - cat $t_file >>$task_file + cat "$t_file" >>"$task_file" local i=0 - while [ $i -lt $1 ]; do - echo "%%FINISHED%%" >>$task_file - i=$(( i + 1 )) + while [ $i -lt "$1" ]; do + echo "%%FINISHED%%" >>"$task_file" + i=$((i + 1)) done } @@ -125,7 +124,7 @@ function clean_tmp() { # clean tmp dir local index=$1 local ssh_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then ssh_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local cmd="${ssh_script} rm -rf ${workdirs[index]}/tmp" @@ -136,7 +135,7 @@ function run_thread() { local index=$1 local thread_no=$2 local runcase_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then runcase_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local count=0 @@ -147,8 +146,9 @@ function run_thread() { local cmd="${runcase_script} ${script}" # script="echo" - while [ 1 ]; do - local line=`flock -x $lock_file -c "head -n1 $task_file;sed -i \"1d\" $task_file"` + while true; do + local line + line=$(flock -x "$lock_file" -c "head -n1 $task_file;sed -i \"1d\" $task_file") if [ "x$line" = "x%%FINISHED%%" ]; then # echo "$index . $thread_no EXIT" break @@ -156,15 +156,17 @@ function run_thread() { if [ -z "$line" ]; then continue fi - echo "$line"|grep -q "^#" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "^#"; then continue fi - local case_redo_time=`echo "$line"|cut -d, -f2` + local case_redo_time + case_redo_time=$(echo "$line" | cut -d, -f2) if [ -z "$case_redo_time" ]; then case_redo_time=${DEFAULT_RETRY_TIME:-2} fi - local case_build_san=`echo "$line"|cut -d, -f3` + local case_build_san + case_build_san=$(echo "$line" | cut -d, -f3) if [ "${case_build_san}" == "y" ]; then case_build_san="y" DEBUGPATH="debugSan" @@ -175,140 +177,150 @@ function run_thread() { usage exit 1 fi - local exec_dir=`echo "$line"|cut -d, -f4` - local case_cmd=`echo "$line"|cut -d, -f5` + local exec_dir + exec_dir=$(echo "$line" | cut -d, -f4) + local case_cmd + case_cmd=$(echo "$line" | cut -d, -f5) local case_file="" - echo "$case_cmd"|grep -q "\.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sh"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sh" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^python3" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^python3"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^./pytest.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^./pytest.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "\.sim" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sim"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sim"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sim" | awk '{print $NF}') fi if [ -z "$case_file" ]; then - case_file=`echo "$case_cmd"|awk '{print $NF}'` + case_file=$(echo "$case_cmd" | awk '{print $NF}') fi if [ -z "$case_file" ]; then continue fi case_sql_file="$exec_dir/${case_file}.sql" case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" - count=$(( count + 1 )) - local case_path=`dirname "$case_file"` - if [ ! -z "$case_path" ]; then - mkdir -p $log_dir/$case_path + count=$((count + 1)) + local case_path + case_path=$(dirname "$case_file") + if [ -n "$case_path" ]; then + mkdir -p "$log_dir"/"$case_path" fi cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 local case_log_file=$log_dir/${case_file}.txt - start_time=`date +%s` - local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` - case_index=`printf "%5d" $case_index` - local case_info=`echo "$line"|cut -d, -f 3,4,5` + start_time=$(date +%s) + local case_index + case_index=$(flock -x "$lock_file" -c "sh -c \"echo \$(( \$( cat $index_file ) + 1 )) | tee $index_file\"") + case_index=$(printf "%5d" "$case_index") + local case_info + case_info=$(echo "$line" | cut -d, -f 3,4,5) while [ ${redo_count} -lt 6 ]; do - if [ -f $case_log_file ]; then - cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt + if [ -f "$case_log_file" ]; then + cp "$case_log_file" "$log_dir"/"$case_file".${redo_count}.redotxt fi - echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$case_log_file - local current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >"$case_log_file" + local current_time + current_time=$(date "+%Y-%m-%d %H:%M:%S") echo -e "$case_index \e[33m START >>>>> \e[0m ${case_info} \e[33m[$current_time]\e[0m" - echo "$current_time" >>$case_log_file - local real_start_time=`date +%s` + echo "$current_time" >>"$case_log_file" + local real_start_time + real_start_time=$(date +%s) # $cmd 2>&1 | tee -a $case_log_file # ret=${PIPESTATUS[0]} - $cmd >>$case_log_file 2>&1 + $cmd >>"$case_log_file" 2>&1 ret=$? - local real_end_time=`date +%s` - local time_elapsed=$(( real_end_time - real_start_time )) - echo "execute time: ${time_elapsed}s" >>$case_log_file - current_time=`date "+%Y-%m-%d %H:%M:%S"` - echo "${hosts[index]} $current_time exit code:${ret}" >>$case_log_file + local real_end_time + real_end_time=$(date +%s) + local time_elapsed + time_elapsed=$((real_end_time - real_start_time)) + echo "execute time: ${time_elapsed}s" >>"$case_log_file" + current_time=$(date "+%Y-%m-%d %H:%M:%S") + echo "${hosts[index]} $current_time exit code:${ret}" >>"$case_log_file" if [ $ret -eq 0 ]; then break fi redo=0 - grep -q "wait too long for taosd start" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "wait too long for taosd start" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "ssh_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "ssh_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: read: Connection reset by peer" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: read: Connection reset by peer" "$case_log_file"; then redo=1 fi - grep -q "Database not ready" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Database not ready" "$case_log_file"; then redo=1 fi - grep -q "Unable to establish connection" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Unable to establish connection" "$case_log_file"; then redo=1 fi - if [ $redo_count -lt $case_redo_time ]; then + if [ $redo_count -lt "$case_redo_time" ]; then redo=1 fi if [ $redo -eq 0 ]; then break fi - redo_count=$(( redo_count + 1 )) + redo_count=$((redo_count + 1)) done - end_time=`date +%s` - echo >>$case_log_file - total_time=$(( end_time - start_time )) - echo "${hosts[index]} total time: ${total_time}s" >>$case_log_file + end_time=$(date +%s) + echo >>"$case_log_file" + total_time=$((end_time - start_time)) + echo "${hosts[index]} total time: ${total_time}s" >>"$case_log_file" # echo "$thread_no ${line} DONE" if [ $ret -eq 0 ]; then echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m" - flock -x $lock_file -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" + flock -x "$lock_file" -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" else - if [ ! -z ${web_server} ]; then - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" + if [ -n "${web_server}" ]; then + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" else - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" fi - mkdir -p $log_dir/${case_file}.coredump + mkdir -p "${log_dir}"/"${case_file}".coredump local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump" local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" $cmd # 2>/dev/null - local corefile=`ls $log_dir/${case_file}.coredump/` + local corefile + corefile=$(ls "$log_dir/${case_file}.coredump/") echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m" echo "=========================log============================" - cat $case_log_file + cat "$case_log_file" echo "=====================================================" echo -e "\e[34m log file: $case_log_file \e[0m" - if [ ! -z "${web_server}" ]; then + if [ -n "${web_server}" ]; then echo "${web_server}/$test_log_dir/${case_file}.txt" fi - if [ ! -z "$corefile" ]; then + if [ -n "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" # if [ $ent -ne 0 ]; then # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" # fi - mkdir $build_dir 2>/dev/null + mkdir "$build_dir" 2>/dev/null if [ $? -eq 0 ]; then # scp build binary cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" @@ -319,15 +331,15 @@ function run_thread() { # get remote sim dir local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then tarcmd="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi cmd="$tarcmd sh -c \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" $cmd local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" - local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file)" + local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file}" scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" @@ -341,11 +353,9 @@ function run_thread() { source_tar_dir=$log_dir/TDinternal_${hosts[index]} source_tar_file=TDinternal.tar.gz fi - mkdir $source_tar_dir 2>/dev/null + mkdir "$source_tar_dir" 2>/dev/null if [ $? -eq 0 ]; then cmd="$scpcmd:${workdirs[index]}/$source_tar_file $source_tar_dir" - # echo "$cmd" - # $cmd fi fi done @@ -361,12 +371,12 @@ function run_thread() { i=0 while [ $i -lt ${#hosts[*]} ]; do clean_tmp $i & - i=$(( i + 1 )) + i=$((i + 1)) done wait -mkdir -p $log_dir -rm -rf $log_dir/* +mkdir -p "$log_dir" +rm -rf "${log_dir:?}"/* task_file=$log_dir/$$.task lock_file=$log_dir/$$.lock index_file=$log_dir/case_index.txt @@ -374,71 +384,71 @@ stat_file=$log_dir/stat.txt failed_case_file=$log_dir/failed.txt success_case_file=$log_dir/success.txt -echo "0" >$index_file +echo "0" >"$index_file" i=0 j=0 while [ $i -lt ${#hosts[*]} ]; do - j=$(( j + threads[i] )) - i=$(( i + 1 )) + j=$((j + threads[i])) + i=$((i + 1)) done prepare_cases $j i=0 while [ $i -lt ${#hosts[*]} ]; do j=0 - while [ $j -lt ${threads[i]} ]; do + while [ $j -lt "${threads[i]}" ]; do run_thread $i $j & - j=$(( j + 1 )) + j=$((j + 1)) done - i=$(( i + 1 )) + i=$((i + 1)) done wait -rm -f $lock_file -rm -f $task_file +rm -f "$lock_file" +rm -f "$task_file" # docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f echo "=====================================================================" echo "log dir: $log_dir" -total_cases=`cat $index_file` +total_cases=$(cat "$index_file") failed_cases=0 -if [ -f $failed_case_file ]; then - if [ ! -z "$web_server" ]; then - failed_cases=`grep -v "$web_server" $failed_case_file|wc -l` +if [ -f "$failed_case_file" ]; then + if [ -n "$web_server" ]; then + failed_cases=$(grep -c -v "$web_server" "$failed_case_file") else - failed_cases=`grep -v "log file:" $failed_case_file|wc -l` + failed_cases=$(grep -c -v "log file:" "$failed_case_file") fi fi -success_cases=$(( total_cases - failed_cases )) -echo "Total Cases: $total_cases" >$stat_file -echo "Successful: $success_cases" >>$stat_file -echo "Failed: $failed_cases" >>$stat_file -cat $stat_file +success_cases=$((total_cases - failed_cases)) +echo "Total Cases: $total_cases" >"$stat_file" +echo "Successful: $success_cases" >>"$stat_file" +echo "Failed: $failed_cases" >>"$stat_file" +cat "$stat_file" RET=0 i=1 if [ -f "${failed_case_file}" ]; then echo "=====================================================" - while read line; do - if [ ! -z "${web_server}" ]; then - echo "$line"|grep -q "${web_server}" - if [ $? -eq 0 ]; then + while read -r line; do + if [ -n "${web_server}" ]; then + + if echo "$line" | grep -q "${web_server}"; then echo " $line" continue fi else - echo "$line"|grep -q "log file:" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "log file:"; then echo " $line" continue fi fi - line=`echo "$line"|cut -d, -f 3,4,5` + line=$(echo "$line" | cut -d, -f 3,4,5) echo -e "$i. $line \e[31m failed\e[0m" >&2 - i=$(( i + 1 )) - done <${failed_case_file} + i=$((i + 1)) + done <"${failed_case_file}" RET=1 fi diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index bae0fdf278..2837c817be 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -60,16 +60,16 @@ echo "ASAN_DIR : $ASAN_DIR" # prevent delete / folder or /usr/bin if [ ${#SIM_DIR} -lt 10 ]; then - echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" - exit 1 + echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" + exit 1 fi -rm -rf $SIM_DIR/* +rm -rf "${SIM_DIR:?}"/* mkdir -p $PRG_DIR mkdir -p $ASAN_DIR -cd $CODE_DIR +cd "$CODE_DIR" || exit ulimit -n 600000 ulimit -c unlimited @@ -78,10 +78,10 @@ ulimit -c unlimited echo "ExcuteCmd:" $* if [[ "$TD_OS" == "Alpine" ]]; then - $* + "$@" else AsanFile=$ASAN_DIR/psim.info - echo "AsanFile:" $AsanFile + echo "AsanFile:" "$AsanFile" unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 From 826c098aee032c9388bd63716c4188b11536775f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 14:46:04 +0800 Subject: [PATCH 216/236] test: test --- tests/system-test/2-query/last_cache_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 9b6dca56fd..8e8ac837e7 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -394,7 +394,7 @@ class TDTestCase: tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) tdSql.checkRows(1) - tdSql.checkCols(3) + tdSql.checkCols(5) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) From 031bcaec8ff80f84d5ae47b9ef6e10f8d38ffac9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 14:47:25 +0800 Subject: [PATCH 217/236] test: test failed case --- tests/system-test/2-query/last_cache_scan.py | 2 +- tests/system-test/8-stream/stream_basic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 8e8ac837e7..9b6dca56fd 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -394,7 +394,7 @@ class TDTestCase: tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) tdSql.checkRows(1) - tdSql.checkCols(5) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index 3ebc255114..04e635ea24 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -101,7 +101,7 @@ class TDTestCase: # check ts interval is correct sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" tdSql.query(sql) - tdSql.checkRows(0) + tdSql.checkRows(1) # stop def stop(self): From 1eee7b44e08ce34693dc1d8783047bf98d2536f7 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 07:31:35 +0000 Subject: [PATCH 218/236] fix crash found on 3.1 --- include/libs/transport/trpc.h | 4 +- include/util/taoserror.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 9 ++- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 62 +++++++++--------- source/dnode/mnode/impl/src/mndProfile.c | 43 +++++++------ source/libs/executor/src/exchangeoperator.c | 70 +++++++++++---------- source/libs/executor/src/sysscanoperator.c | 15 +++-- source/libs/transport/inc/transComm.h | 5 +- source/libs/transport/src/trans.c | 12 +++- source/libs/transport/src/transCli.c | 12 +++- source/util/src/terror.c | 1 + 11 files changed, 140 insertions(+), 94 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index accb7e6f24..e7b0b25653 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -163,7 +163,8 @@ int rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc in // These functions will not be called in the child process int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int32_t timeoutMs); +int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs); int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); void *rpcAllocHandle(); void rpcSetIpWhite(void *thandl, void *arg); @@ -171,6 +172,7 @@ void rpcSetIpWhite(void *thandl, void *arg); int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); +int32_t rpcCvtErrCode(int32_t code); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c4d61157b6..b41078dfbf 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -68,6 +68,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED TAOS_DEF_ERROR_CODE(0, 0x0020) // "Vgroup could not be connected" #define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) // #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) // +#define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index ac94390619..a95ec42bd0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -159,13 +159,18 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); SEpSet epSet = {0}; + int8_t epUpdated = 0; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, 5000); + rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, 5000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); - char tbuf[256]; + char tbuf[512]; dmEpSetToStr(tbuf, sizeof(tbuf), &epSet); dError("failed to send status req since %s, epSet:%s, inUse:%d", tstrerror(rpcRsp.code), tbuf, epSet.inUse); + } else { + if (epUpdated == 1) { + dmSetMnodeEpSet(pMgmt->pData, &epSet); + } } dmProcessStatusRsp(pMgmt, &rpcRsp); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index af43804db4..b79b3f9e11 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -22,10 +22,8 @@ static void *dmStatusThreadFp(void *param) { int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-status"); - const static int16_t TRIM_FREQ = 30; - int32_t trimCount = 0; - int32_t upTimeCount = 0; - int64_t upTime = 0; + int32_t upTimeCount = 0; + int64_t upTime = 0; while (1) { taosMsleep(200); @@ -38,11 +36,6 @@ static void *dmStatusThreadFp(void *param) { dmSendStatusReq(pMgmt); lastTime = curTime; - trimCount = (trimCount + 1) % TRIM_FREQ; - if (trimCount == 0) { - taosMemoryTrim(0); - } - if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { upTime = taosGetOsUptime() - tsDndStartOsUptime; tsDndUpTime = TMAX(tsDndUpTime, upTime); @@ -55,27 +48,27 @@ static void *dmStatusThreadFp(void *param) { SDmNotifyHandle dmNotifyHdl = {.state = 0}; static void *dmNotifyThreadFp(void *param) { - SDnodeMgmt *pMgmt = param; - setThreadName("dnode-notify"); + SDnodeMgmt *pMgmt = param; + setThreadName("dnode-notify"); - if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { - return NULL; + if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { + return NULL; } - bool wait = true; - while (1) { - if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + bool wait = true; + while (1) { + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; if (wait) tsem_wait(&dmNotifyHdl.sem); atomic_store_8(&dmNotifyHdl.state, 1); - dmSendNotifyReq(pMgmt); - if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { - wait = true; - continue; + dmSendNotifyReq(pMgmt); + if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { + wait = true; + continue; } - wait = false; + wait = false; } - return NULL; + return NULL; } static void *dmMonitorThreadFp(void *param) { @@ -83,6 +76,9 @@ static void *dmMonitorThreadFp(void *param) { int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-monitor"); + static int32_t TRIM_FREQ = 20; + int32_t trimCount = 0; + while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; @@ -93,6 +89,11 @@ static void *dmMonitorThreadFp(void *param) { if (interval >= tsMonitorInterval) { (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; + + trimCount = (trimCount + 1) % TRIM_FREQ; + if (trimCount == 0) { + taosMemoryTrim(0); + } } } @@ -126,14 +127,15 @@ static void *dmCrashReportThreadFp(void *param) { setThreadName("dnode-crashReport"); char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s%s.taosdCrashLog", tsLogDir, TD_DIRSEP); - char *pMsg = NULL; - int64_t msgLen = 0; + char *pMsg = NULL; + int64_t msgLen = 0; TdFilePtr pFile = NULL; - bool truncateFile = false; - int32_t sleepTime = 200; - int32_t reportPeriodNum = 3600 * 1000 / sleepTime;; + bool truncateFile = false; + int32_t sleepTime = 200; + int32_t reportPeriodNum = 3600 * 1000 / sleepTime; + ; int32_t loopTimes = reportPeriodNum; - + while (1) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; if (loopTimes++ < reportPeriodNum) { @@ -167,13 +169,13 @@ static void *dmCrashReportThreadFp(void *param) { pMsg = NULL; continue; } - + if (pFile) { taosReleaseCrashLogFile(pFile, truncateFile); pFile = NULL; truncateFile = false; } - + taosMsleep(sleepTime); loopTimes = 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 34ee18a5cc..e0559b4c48 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" +#include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -26,7 +27,6 @@ #include "mndView.h" #include "tglobal.h" #include "tversion.h" -#include "audit.h" typedef struct { uint32_t id; @@ -530,23 +530,24 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } #ifdef TD_ENTERPRISE - bool needCheck = true; - int32_t key = HEARTBEAT_KEY_DYN_VIEW; - SDynViewVersion* pDynViewVer = NULL; - SKv* pKv = taosHashGet(pHbReq->info, &key, sizeof(key)); + bool needCheck = true; + int32_t key = HEARTBEAT_KEY_DYN_VIEW; + SDynViewVersion *pDynViewVer = NULL; + SKv *pKv = taosHashGet(pHbReq->info, &key, sizeof(key)); if (NULL != pKv) { pDynViewVer = pKv->value; mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer); - SDynViewVersion* pRspVer = NULL; + SDynViewVersion *pRspVer = NULL; if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) { return -1; } - + if (needCheck) { SKv kv1 = {.key = HEARTBEAT_KEY_DYN_VIEW, .valueLen = sizeof(*pDynViewVer), .value = pRspVer}; taosArrayPush(hbRsp.info, &kv1); - mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, pRspVer->dynViewVer); + mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, + pRspVer->dynViewVer); } } #endif @@ -594,7 +595,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb if (!needCheck) { break; } - + void *rspMsg = NULL; int32_t rspLen = 0; mndValidateViewInfo(pMnode, kv->value, kv->valueLen / sizeof(SViewVersion), &rspMsg, &rspLen); @@ -814,8 +815,9 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl * @param offset skip [offset] queries in pConn * @param rowsToPack at most rows to pack * @return rows packed -*/ -static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBlock* pBlock, uint32_t offset, uint32_t rowsToPack) { + */ +static int32_t packQueriesIntoBlock(SShowObj *pShow, SConnObj *pConn, SSDataBlock *pBlock, uint32_t offset, + uint32_t rowsToPack) { int32_t cols = 0; taosRLockLatch(&pConn->queryLock); int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); @@ -826,7 +828,7 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc int32_t i = offset; for (; i < numOfQueries && (i - offset) < rowsToPack; ++i) { - int32_t curRowIndex = pBlock->info.rows; + int32_t curRowIndex = pBlock->info.rows; SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i); cols = 0; @@ -877,14 +879,19 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->subPlanNum, false); char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; + int64_t reserve = 64; int32_t strSize = sizeof(subStatus); int32_t offset = VARSTR_HEADER_SIZE; - for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) { + for (int32_t i = 0; i < pQuery->subPlanNum && offset + reserve < strSize; ++i) { if (i) { - offset += snprintf(subStatus + offset, strSize - offset - 1, ","); + offset += sprintf(subStatus + offset, ","); + } + if (offset + reserve < strSize) { + SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); + offset += sprintf(subStatus + offset, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); + } else { + break; } - SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); - offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); } varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -903,8 +910,8 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc } static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode * pMnode = pReq->info.node; - SSdb * pSdb = pMnode->pSdb; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SConnObj *pConn = NULL; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index ef8bf9674a..2797cb2d82 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -16,16 +16,14 @@ #include "executorInt.h" #include "filter.h" #include "function.h" -#include "index.h" #include "operator.h" -#include "os.h" #include "query.h" #include "querytask.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" -#include "tname.h" #include "tref.h" +#include "trpc.h" typedef struct SFetchRspHandleWrapper { uint32_t exchangeId; @@ -131,14 +129,14 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, try next %d/%" PRIzu, + " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%" PRId64 + ", totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } @@ -232,7 +230,7 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { if (blockDataGetNumOfRows(pBlock) == 0) { continue; } - + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); @@ -261,7 +259,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const if (pInfo->dynamicOp) { return TSDB_CODE_SUCCESS; } - + for (int32_t i = 0; i < numOfSources; ++i) { SSourceDataInfo dataInfo = {0}; dataInfo.status = EX_SOURCE_DATA_NOT_READY; @@ -343,8 +341,8 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode goto _error; } - pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); return pOperator; _error: @@ -383,7 +381,7 @@ void doDestroyExchangeOperatorInfo(void* param) { blockDataDestroy(pExInfo->pDummyBlock); tSimpleHashCleanup(pExInfo->pHashSources); - + tsem_destroy(&pExInfo->ready); taosMemoryFreeClear(param); } @@ -411,13 +409,18 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, + pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); - pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), - pExchangeInfo); + pSourceDataInfo->code = rpcCvtErrCode(code); + if (pSourceDataInfo->code != code) { + qError("%s fetch rsp received, index:%d, error:%s, cvted error: %s, %p", pSourceDataInfo->taskId, index, + tstrerror(code), tstrerror(pSourceDataInfo->code), pExchangeInfo); + } else { + qError("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), + pExchangeInfo); + } } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -450,7 +453,7 @@ int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, in return TSDB_CODE_OUT_OF_MEMORY; } pScan->tableSeq = tableSeq; - + (*ppRes)->opType = srcOpType; (*ppRes)->downstreamIdx = 0; (*ppRes)->value = pScan; @@ -459,9 +462,8 @@ int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, in return TSDB_CODE_SUCCESS; } - int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); if (EX_SOURCE_DATA_NOT_READY != pDataInfo->status) { return TSDB_CODE_SUCCESS; } @@ -490,7 +492,8 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas req.queryId = pTaskInfo->id.queryId; req.execId = pSource->execId; if (pDataInfo->pSrcUidList) { - int32_t code = buildTableScanOperatorParam(&req.pOpParam, pDataInfo->pSrcUidList, pDataInfo->srcOpType, pDataInfo->tableSeq); + int32_t code = + buildTableScanOperatorParam(&req.pOpParam, pDataInfo->pSrcUidList, pDataInfo->srcOpType, pDataInfo->tableSeq); taosArrayDestroy(pDataInfo->pSrcUidList); pDataInfo->pSrcUidList = NULL; if (TSDB_CODE_SUCCESS != code) { @@ -499,7 +502,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return pTaskInfo->code; } } - + int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; @@ -741,8 +744,8 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 + ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); @@ -769,7 +772,7 @@ _error: } int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasicParam* pBasicParam) { - SExchangeInfo* pExchangeInfo = pOperator->info; + SExchangeInfo* pExchangeInfo = pOperator->info; SExchangeSrcIndex* pIdx = tSimpleHashGet(pExchangeInfo->pHashSources, &pBasicParam->vgId, sizeof(pBasicParam->vgId)); if (NULL == pIdx) { qError("No exchange source for vgId: %d", pBasicParam->vgId); @@ -784,7 +787,7 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL); dataInfo.srcOpType = pBasicParam->srcOpType; dataInfo.tableSeq = pBasicParam->tableSeq; - + taosArrayPush(pExchangeInfo->pSourceDataInfo, &dataInfo); pIdx->inUseIdx = taosArrayGetSize(pExchangeInfo->pSourceDataInfo) - 1; } else { @@ -800,15 +803,14 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic return TSDB_CODE_SUCCESS; } - int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - int32_t code = TSDB_CODE_SUCCESS; + SExchangeInfo* pExchangeInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; SExchangeOperatorBasicParam* pBasicParam = NULL; - SExchangeOperatorParam* pParam = (SExchangeOperatorParam*)pOperator->pOperatorGetParam->value; + SExchangeOperatorParam* pParam = (SExchangeOperatorParam*)pOperator->pOperatorGetParam->value; if (pParam->multiParams) { SExchangeOperatorBatchParam* pBatch = (SExchangeOperatorBatchParam*)pOperator->pOperatorGetParam->value; - int32_t iter = 0; + int32_t iter = 0; while (NULL != (pBasicParam = tSimpleHashIterate(pBatch->pBatchs, pBasicParam, &iter))) { code = addSingleExchangeSource(pOperator, pBasicParam); if (code) { @@ -826,11 +828,11 @@ int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; - int32_t code = TSDB_CODE_SUCCESS; - if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) { + int32_t code = TSDB_CODE_SUCCESS; + if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || + (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 76de204cab..0c421bf354 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -15,9 +15,7 @@ #include "executorInt.h" #include "filter.h" -#include "function.h" #include "functionMgt.h" -#include "os.h" #include "querynodes.h" #include "systable.h" #include "tname.h" @@ -32,6 +30,7 @@ #include "storageapi.h" #include "tcompare.h" #include "thash.h" +#include "trpc.h" #include "ttypes.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); @@ -1789,8 +1788,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); return pOperator; _error: @@ -1867,7 +1866,13 @@ int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->handle = htobe64(pRsp->handle); pRsp->compLen = htonl(pRsp->compLen); } else { - operator->pTaskInfo->code = code; + operator->pTaskInfo->code = rpcCvtErrCode(code); + if (operator->pTaskInfo->code != code) { + qError("load systable rsp received, error:%s, cvted error:%s", tstrerror(code), + tstrerror(operator->pTaskInfo->code)); + } else { + qError("load systable rsp received, error:%s", tstrerror(code)); + } } tsem_post(&pScanResInfo->ready); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 19b2dfc300..7483588593 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -122,6 +122,8 @@ typedef struct SExHandle { typedef struct { STransMsg* pRsp; + SEpSet epSet; + int8_t hasEpSet; tsem_t* pSem; int8_t inited; SRWLatch latch; @@ -317,7 +319,8 @@ int transReleaseSrvHandle(void* handle); int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); -int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int32_t timeoutMs); +int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs); int transSendResponse(const STransMsg* msg); int transRegisterMsg(const STransMsg* msg); int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index b23d229931..64302a78fc 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -169,8 +169,9 @@ int rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, in int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { return transSendRecv(shandle, pEpSet, pMsg, pRsp); } -int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int32_t timeoutMs) { - return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, timeoutMs); +int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); } int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } @@ -197,6 +198,13 @@ int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { return transUtilSWhiteListToStr(pWhiteList, ppBuf); } +int32_t rpcCvtErrCode(int32_t code) { + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return TSDB_CODE_RPC_NETWORK_ERROR; + } + return code; +} + int32_t rpcInit() { transInit(); return 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 936d11c151..4b3000b47e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2425,6 +2425,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STransSyncMsg* pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); if (pSyncMsg != NULL) { memcpy(pSyncMsg->pRsp, (char*)pResp, sizeof(*pResp)); + if (cliIsEpsetUpdated(pResp->code, pCtx)) { + pSyncMsg->hasEpSet = 1; + epsetAssign(&pSyncMsg->epSet, &pCtx->epSet); + } tsem_post(pSyncMsg->pSem); taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); } else { @@ -2640,10 +2644,12 @@ int64_t transCreateSyncMsg(STransMsg* pTransMsg) { pSyncMsg->inited = 0; pSyncMsg->pRsp = pTransMsg; pSyncMsg->pSem = sem; + pSyncMsg->hasEpSet = 0; return taosAddRef(transGetSyncMsgMgt(), pSyncMsg); } -int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int32_t timeoutMs) { +int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransInst == NULL) { @@ -2695,6 +2701,10 @@ int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pRe ret = TSDB_CODE_TIMEOUT_ERROR; } else { memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); + if (pSyncMsg->hasEpSet == 1) { + epsetAssign(pEpSet, &pSyncMsg->epSet); + *epUpdated = 1; + } ret = 0; } _RETURN: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c294270e4a..360689cef3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -53,6 +53,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_BROKEN_LINK, "Conn is broken") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED, "some vnode/qnode/mnode(s) out of service") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, "rpc open too many session") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_ERROR, "rpc network error") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") From 1c28f535255f91b0430e1560719cf56de0cce02d Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 08:28:00 +0000 Subject: [PATCH 219/236] refactor stream code --- source/libs/stream/inc/streamBackendRocksdb.h | 3 - source/libs/stream/src/streamBackendRocksdb.c | 128 +++++++++--------- 2 files changed, 63 insertions(+), 68 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 9bfec5577c..b89664a6c1 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -19,10 +19,7 @@ #include "rocksdb/c.h" //#include "streamInt.h" #include "streamState.h" -#include "tcoding.h" #include "tcommon.h" -#include "tcompare.h" -#include "ttimer.h" typedef struct SCfComparator { rocksdb_comparator_t** comp; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index a5a4bd4dc1..4b2a9e4a65 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -14,8 +14,6 @@ */ #include "streamBackendRocksdb.h" -#include "executor.h" -#include "query.h" #include "streamInt.h" #include "tcommon.h" #include "tref.h" @@ -1059,14 +1057,14 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { rocksdb_column_family_handle_t** ppCf = NULL; int32_t nCf = chkpGetAllDbCfHandle2(pTaskDb, &ppCf); - qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); + stDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); if ((code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf)) == 0) { if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) { stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir); } else { - qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, - taosGetTimestampMs() - st); + stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, + taosGetTimestampMs() - st); } } else { stError("stream backend:%p failed to flush db at:%s", pTaskDb, pChkpIdDir); @@ -1797,7 +1795,7 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { cfNames = NULL; } - qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); + stDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); return pTaskDb; _EXIT: @@ -1826,7 +1824,7 @@ void taskDbDestroy(void* pDb, bool flush) { streamMetaRemoveDB(wrapper->pMeta, wrapper->idstr); - qDebug("succ to destroy stream backend:%p", wrapper); + stDebug("succ to destroy stream backend:%p", wrapper); int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); @@ -2323,7 +2321,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { stError("failed to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { - qDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); + stDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); wrapper->pCf[idx] = cf; } } @@ -2365,14 +2363,14 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe char* err = NULL; \ int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ + stWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ wrapper->dataWritten += 1; \ char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ @@ -2385,54 +2383,54 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(err); \ code = -1; \ } else { \ - qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ - ttlVLen, wrapper); \ + stTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ + ttlVLen, wrapper); \ } \ taosMemoryFree(ttlV); \ } while (0); -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->db; \ - rocksdb_readoptions_t* opts = wrapper->readOpt; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL || len == 0) { \ - if (err == NULL) { \ - qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ - } else { \ - stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFreeClear(err); \ - } \ - code = -1; \ - } else { \ - char* p = NULL; \ - int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (tlen <= 0) { \ - stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ - funcname); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, \ - wrapper); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = tlen; \ - } \ +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + stWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_readoptions_t* opts = wrapper->readOpt; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + stTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + stTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, \ + tlen, wrapper); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ + } \ } while (0); #define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ @@ -2442,14 +2440,14 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe char* err = NULL; \ int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ - qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ + stWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ code = -1; \ break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ wrapper->dataWritten += 1; \ char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_t* db = wrapper->db; \ @@ -2460,7 +2458,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(err); \ code = -1; \ } else { \ - qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ + stTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ } \ } while (0); @@ -2681,7 +2679,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { } #ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateGetCur_rocksdb"); + stDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); @@ -2775,7 +2773,7 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k } SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { - qDebug("streamStateSessionSeekToLast_rocksdb"); + stDebug("streamStateSessionSeekToLast_rocksdb"); int32_t code = 0; @@ -2812,7 +2810,7 @@ SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { } int32_t streamStateSessionCurPrev_rocksdb(SStreamStateCur* pCur) { - qDebug("streamStateCurPrev_rocksdb"); + stDebug("streamStateCurPrev_rocksdb"); if (!pCur) return -1; rocksdb_iter_prev(pCur->iter); @@ -2862,7 +2860,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta return pCur; } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { - qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); + stDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { @@ -2900,7 +2898,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { - qDebug("streamStateSessionSeekKeyNext_rocksdb"); + stDebug("streamStateSessionSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { @@ -3004,7 +3002,7 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { } SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillGetCur_rocksdb"); + stDebug("streamStateFillGetCur_rocksdb"); SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -3064,7 +3062,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, } SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillSeekKeyNext_rocksdb"); + stDebug("streamStateFillSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (!pCur) { @@ -3102,7 +3100,7 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const return NULL; } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillSeekKeyPrev_rocksdb"); + stDebug("streamStateFillSeekKeyPrev_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { From 4c5441da7ee711ac919a302306cf857c22b27f56 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 08:38:16 +0000 Subject: [PATCH 220/236] refactor stream code --- source/libs/stream/src/streamMeta.c | 6 +- source/libs/stream/src/streamSessionState.c | 126 ++++++++++---------- source/libs/stream/src/streamSnapshot.c | 14 +-- source/libs/stream/src/streamUpdate.c | 6 +- 4 files changed, 77 insertions(+), 75 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9c94799237..7ea80ad8d8 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -201,7 +201,7 @@ int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { void* key = taosHashGetKey(pIter, NULL); code = streamStateCvtDataFormat(pMeta->path, key, *(void**)pIter); if (code != 0) { - qError("failed to cvt data"); + stError("failed to cvt data"); goto _EXIT; } @@ -225,11 +225,11 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { if (compatible == STREAM_STATA_COMPATIBLE) { return 0; } else if (compatible == STREAM_STATA_NEED_CONVERT) { - qInfo("stream state need covert backend format"); + stInfo("stream state need covert backend format"); return streamMetaCvtDbFormat(pMeta); } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { - qError( + stError( "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " "manually", tsDataDir); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 5f19b47e1f..2cb77c60dc 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -17,17 +17,14 @@ #include "query.h" #include "streamBackendRocksdb.h" -#include "taos.h" #include "tcommon.h" -#include "thash.h" #include "tsimplehash.h" - -typedef int (*__session_compare_fn_t) (const SSessionKey* pWin, const void* pDatas, int pos); +typedef int (*__session_compare_fn_t)(const SSessionKey* pWin, const void* pDatas, int pos); int sessionStateKeyCompare(const SSessionKey* pWin1, const void* pDatas, int pos) { SRowBuffPos* pPos2 = taosArrayGetP(pDatas, pos); - SSessionKey* pWin2 = (SSessionKey*) pPos2->pKey; + SSessionKey* pWin2 = (SSessionKey*)pPos2->pKey; return sessionWinKeyCmpr(pWin1, pWin2); } @@ -61,12 +58,12 @@ int32_t binarySearch(void* keyList, int num, const void* key, __session_compare_ int64_t getSessionWindowEndkey(void* data, int32_t index) { SArray* pWinInfos = (SArray*)data; SRowBuffPos** ppos = taosArrayGet(pWinInfos, index); - SSessionKey* pWin = (SSessionKey*)((*ppos)->pKey); + SSessionKey* pWin = (SSessionKey*)((*ppos)->pKey); return pWin->win.ekey; } bool inSessionWindow(SSessionKey* pKey, TSKEY ts, int64_t gap) { - if (ts + gap >= pKey->win.skey && ts - gap <= pKey->win.ekey) { + if (ts + gap >= pKey->win.skey && ts - gap <= pKey->win.ekey) { return true; } return false; @@ -80,7 +77,8 @@ static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pW return pNewPos; } -static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, int32_t index) { +static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, + int32_t index) { SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); @@ -97,11 +95,12 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe return pNewPos; } -int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen) { - int32_t code = TSDB_CODE_SUCCESS; +int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, + int32_t* pVLen) { + int32_t code = TSDB_CODE_SUCCESS; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -146,7 +145,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, if (index + 1 < size) { pPos = taosArrayGetP(pWinStates, index + 1); - if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) ) { + if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap))) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -157,9 +156,9 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, if (index + 1 == 0) { if (!isDeteled(pFileState, endTs) && isFlushedState(pFileState, endTs, gap)) { - void* p = NULL; - void* pFileStore = getStateFileStore(pFileState); - int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); + void* p = NULL; + void* pFileStore = getStateFileStore(pFileState); + int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); code = code_file; @@ -184,10 +183,10 @@ _end: } int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSessionKey* pKey = pPos->pKey; - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -202,7 +201,7 @@ int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) } // find the first position which is smaller than the pKey - int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); + int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); if (index >= 0) { taosArrayInsert(pWinStates, index, &pPos); } else { @@ -218,7 +217,7 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; - void* pBuff = NULL; + void* pBuff = NULL; int32_t code = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); if (code != TSDB_CODE_SUCCESS) { return code; @@ -229,16 +228,16 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v return TSDB_CODE_SUCCESS; } -int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen) { - SSHashObj* pSessionBuff = (SSHashObj*) pBuff; - SSessionKey* pWinKey = (SSessionKey*) key; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); +int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen) { + SSHashObj* pSessionBuff = (SSHashObj*)pBuff; + SSessionKey* pWinKey = (SSessionKey*)key; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (!ppBuff) { return TSDB_CODE_SUCCESS; } SArray* pWinStates = (SArray*)(*ppBuff); int32_t size = taosArrayGetSize(pWinStates); - TSKEY gap = 0; + TSKEY gap = 0; int32_t index = binarySearch(pWinStates, size, pWinKey, sessionStateKeyCompare); if (index >= 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, index); @@ -251,15 +250,15 @@ int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen) } int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SSessionKey* pWinKey = (SSessionKey*) pPos->pKey; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SSessionKey* pWinKey = (SSessionKey*)pPos->pKey; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (!ppBuff) { return TSDB_CODE_SUCCESS; } SArray* pWinStates = (SArray*)(*ppBuff); int32_t size = taosArrayGetSize(pWinStates); - TSKEY gap = 0; + TSKEY gap = 0; int32_t index = binarySearch(pWinStates, size, pWinKey, sessionStateKeyCompare); if (index >= 0) { SRowBuffPos* pItemPos = taosArrayGetP(pWinStates, index); @@ -270,11 +269,12 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP return TSDB_CODE_SUCCESS; } -int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { +int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, + const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { SRowBuffPos* pNewPos = NULL; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); - SArray* pWinStates = NULL; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; if (!ppBuff) { pWinStates = taosArrayInit(16, POINTER_BYTES); tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); @@ -297,10 +297,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream } else { if (size > 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); - if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >=0) { + if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >= 0) { // pCur is invalid SSessionKey pTmpKey = *pWinKey; - int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); + int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); ASSERT(code == TSDB_CODE_FAILED); goto _end; } @@ -320,7 +320,7 @@ void sessionWinStateClear(SStreamFileState* pFileState) { void* pIte = NULL; size_t keyLen = 0; int32_t iter = 0; - void* pBuff = getRowStateBuff(pFileState); + void* pBuff = getRowStateBuff(pFileState); while ((pIte = tSimpleHashIterate(pBuff, pIte, &iter)) != NULL) { SArray* pWinStates = *((void**)pIte); int32_t size = taosArrayGetSize(pWinStates); @@ -336,7 +336,7 @@ void sessionWinStateCleanup(void* pBuff) { size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pBuff, pIte, &iter)) != NULL) { - SArray* pWinStates = (SArray*) (*(void**)pIte); + SArray* pWinStates = (SArray*)(*(void**)pIte); taosArrayDestroy(pWinStates); } tSimpleHashCleanup(pBuff); @@ -395,11 +395,13 @@ static void transformCursor(SStreamFileState* pFileState, SStreamStateCur* pCur) pCur->pStreamFileState = pFileState; } -static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t groupId, SArray* pWinStates, SStreamStateCur** ppCur) { +static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t groupId, SArray* pWinStates, + SStreamStateCur** ppCur) { SSessionKey key = {.groupId = groupId}; - int32_t code = streamStateSessionGetKVByCur_rocksdb(*ppCur, &key, NULL, NULL); - if (taosArrayGetSize(pWinStates) > 0 && (code == TSDB_CODE_FAILED || sessionStateKeyCompare(&key, pWinStates, 0) >= 0)) { - if ( !(*ppCur) ) { + int32_t code = streamStateSessionGetKVByCur_rocksdb(*ppCur, &key, NULL, NULL); + if (taosArrayGetSize(pWinStates) > 0 && + (code == TSDB_CODE_FAILED || sessionStateKeyCompare(&key, pWinStates, 0) >= 0)) { + if (!(*ppCur)) { (*ppCur) = createStreamStateCursor(); } transformCursor(pFileState, *ppCur); @@ -410,8 +412,8 @@ static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t } SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey) { - SArray* pWinStates = NULL; - int32_t index = -1; + SArray* pWinStates = NULL; + int32_t index = -1; SStreamStateCur* pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); if (pCur) { if (sessionStateKeyCompare(pWinKey, pWinStates, index) > 0) { @@ -427,8 +429,8 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, } SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey) { - SArray* pWinStates = NULL; - int32_t index = -1; + SArray* pWinStates = NULL; + int32_t index = -1; SStreamStateCur* pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); if (pCur) { sessionWinStateMoveToNext(pCur); @@ -449,7 +451,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void SSHashObj* pSessionBuff = getRowStateBuff(pCur->pStreamFileState); void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); - SArray* pWinStates = NULL; + SArray* pWinStates = NULL; if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } @@ -467,7 +469,8 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void } else { void* pData = NULL; code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, &pData, pVLen); - if (taosArrayGetSize(pWinStates) > 0 && (code == TSDB_CODE_FAILED || sessionStateKeyCompare(pKey, pWinStates, 0) >= 0)) { + if (taosArrayGetSize(pWinStates) > 0 && + (code == TSDB_CODE_FAILED || sessionStateKeyCompare(pKey, pWinStates, 0) >= 0)) { transformCursor(pCur->pStreamFileState, pCur); SRowBuffPos* pPos = taosArrayGetP(pWinStates, pCur->buffIndex); if (pVal) { @@ -498,9 +501,9 @@ int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey) { SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key); - SSessionKey tmpKey = *key; - int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); - bool hasCurrentPrev = true; + SSessionKey tmpKey = *key; + int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); + bool hasCurrentPrev = true; if (code == TSDB_CODE_FAILED) { streamStateFreeCur(pCur); pCur = sessionWinStateSeekKeyNext(pFileState, key); @@ -535,13 +538,13 @@ _end: } int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { SSessionKey* pWinKey = key; - TSKEY gap = 0; - int32_t code = TSDB_CODE_SUCCESS; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + TSKEY gap = 0; + int32_t code = TSDB_CODE_SUCCESS; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -560,7 +563,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); } else { (*pVal) = addNewSessionWindow(pFileState, pWinStates, key); code = TSDB_CODE_FAILED; @@ -589,7 +593,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (index + 1 < size) { pPos = taosArrayGetP(pWinStates, index + 1); void* stateKey = (char*)(pPos->pRowBuff) + (valSize - keyDataLen); - if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) || fn(pKeyData, stateKey) == true) { + if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) || + fn(pKeyData, stateKey) == true) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -607,7 +612,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; - qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); goto _end; } else { taosMemoryFree(p); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index e29f2ba7de..a69cb75dad 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -15,10 +15,8 @@ #include "streamSnapshot.h" #include "query.h" -#include "rocksdb/c.h" #include "streamBackendRocksdb.h" #include "streamInt.h" -#include "tcommon.h" enum SBackendFileType { ROCKSDB_OPTIONS_TYPE = 1, @@ -158,7 +156,7 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { } sprintf(buf + strlen(buf) - 1, "]"); - qInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, + stInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, pSnapFile->snapInfo.taskId, buf); taosMemoryFree(buf); } @@ -203,7 +201,7 @@ int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) { int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { TdDirPtr pDir = taosOpenDir(pSnapFile->path); if (NULL == pDir) { - qError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); + stError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); return -1; } @@ -397,7 +395,7 @@ _NEXT: } item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); - qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 + stDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d, total set:%d, current set idx: %d", STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx, (int)taosArrayGetSize(pHandle->pDbSnapSet), pHandle->currIdx); @@ -515,7 +513,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); return code; } else { - qInfo("succ to write data %s", pItem->name); + stInfo("succ to write data %s", pItem->name); } pSnapFile->offset += bytes; } else { @@ -538,7 +536,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t } taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); - qInfo("succ to write data %s", pItem->name); + stInfo("succ to write data %s", pItem->name); pSnapFile->offset += pHdr->size; } code = 0; @@ -563,7 +561,7 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa "checkpoint", snapInfo.chkpId); if (!taosIsDir(path)) { code = taosMulMkDir(path); - qInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); + stInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); ASSERT(code == 0); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index f78f6f4df1..d607f26de3 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#include "query.h" -#include "tdatablock.h" #include "tencode.h" #include "tstreamUpdate.h" #include "ttime.h" @@ -387,8 +385,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); - bool res = true; - if ( pMapMaxTs && ts < *pMapMaxTs ) { + bool res = true; + if (pMapMaxTs && ts < *pMapMaxTs) { res = false; } else { taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); From 5e45ecd3fda3e3ea5530d800f672760f871184ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Jan 2024 16:46:07 +0800 Subject: [PATCH 221/236] fix: possible race condition --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbMemTable.c | 3 ++ source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 48 +++++++++++++--------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 75f90af378..9d8d5013fa 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -397,6 +397,7 @@ struct STbData { tb_uid_t uid; TSKEY minKey; TSKEY maxKey; + SRWLatch lock; SDelData *pHead; SDelData *pTail; SMemSkipList sl; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index cc77474e79..4657b60f62 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -181,6 +181,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pDelData->sKey = sKey; pDelData->eKey = eKey; pDelData->pNext = NULL; + taosWLockLatch(&pTbData->lock); if (pTbData->pHead == NULL) { ASSERT(pTbData->pTail == NULL); pTbData->pHead = pTbData->pTail = pDelData; @@ -188,6 +189,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pTbData->pTail->pNext = pDelData; pTbData->pTail = pDelData; } + taosWUnLockLatch(&pTbData->lock); pMemTable->nDel++; pMemTable->minVer = TMIN(pMemTable->minVer, version); @@ -401,6 +403,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SL_NODE_BACKWARD(pTbData->sl.pHead, iLevel) = NULL; SL_NODE_FORWARD(pTbData->sl.pTail, iLevel) = NULL; } + taosInitRWLatch(&pTbData->lock); taosWLockLatch(&pMemTable->latch); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index a223a2dc2d..33b7667168 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -22,9 +22,14 @@ #include "tsdbUtil2.h" #include "tsimplehash.h" -#define INIT_TIMEWINDOW(_w) do { (_w)->skey = INT64_MAX; (_w)->ekey = INT64_MIN;} while(0); +#define INIT_TIMEWINDOW(_w) \ + do { \ + (_w)->skey = INT64_MAX; \ + (_w)->ekey = INT64_MIN; \ + } while (0); -static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order); +static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, + int32_t order); static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { int32_t num = numOfTables / pBuf->numPerBucket; @@ -449,7 +454,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); taosArrayPush(pBlockIter->blockList, &blockInfo); - STableBlockScanInfo *pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; + STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; if (pTableScanInfo->pBlockIdxList == NULL) { size_t szTableDataBlocks = taosArrayGetSize(pTableScanInfo->pBlockList); pTableScanInfo->pBlockIdxList = taosArrayInit(szTableDataBlocks, sizeof(STableDataBlockIdx)); @@ -504,7 +509,7 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ int32_t code = 0; STombRecord record = {0}; - uint64_t uid = pReader->status.uidList.tableUidList[*j]; + uint64_t uid = pReader->status.uidList.tableUidList[*j]; STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); @@ -595,12 +600,12 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs return code; } -// uint64_t uid = pReader->status.uidList.tableUidList[j]; + // uint64_t uid = pReader->status.uidList.tableUidList[j]; -// STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); -// if (pScanInfo->pFileDelData == NULL) { -// pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); -// } + // STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + // if (pScanInfo->pFileDelData == NULL) { + // pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); + // } ETombBlkCheckEnum ret = 0; code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret); @@ -650,6 +655,7 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT SDelData* p = NULL; if (pMemTbData != NULL) { + taosRLockLatch(&pMemTbData->lock); p = pMemTbData->pHead; while (p) { if (p->version <= ver) { @@ -658,6 +664,7 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT p = p->pNext; } + taosRUnLockLatch(&pMemTbData->lock); } if (piMemTbData != NULL) { @@ -681,7 +688,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo } int32_t i = 0; - while((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) { + while ((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) { ++i; } @@ -689,7 +696,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo return 0; } - SStatisBlk *p = &pStatisBlkArray->data[i]; + SStatisBlk* p = &pStatisBlkArray->data[i]; STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); tStatisBlockInit(pStatisBlock); @@ -759,14 +766,14 @@ void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { if (size < numOfFileObj) { int32_t inc = numOfFileObj - size; for (int32_t k = 0; k < inc; ++k) { - SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); taosArrayPush(pLDIterList, &pIter); } } else if (size > numOfFileObj) { // remove unused LDataIter int32_t inc = size - numOfFileObj; for (int i = 0; i < inc; ++i) { - SLDataIter *pIter = taosArrayPop(pLDIterList); + SLDataIter* pIter = taosArrayPop(pLDIterList); destroyLDataIter(pIter); } } @@ -781,9 +788,9 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) taosArrayPush(pSttFileBlockIterArray, &pList); } - for(int32_t j = 0; j < numOfLevels; ++j) { + for (int32_t j = 0; j < numOfLevels; ++j) { SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; - SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); + SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); } @@ -829,8 +836,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file - TStatisBlkArray *pStatisBlkArray = NULL; - int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pStatisBlkArray); + TStatisBlkArray* pStatisBlkArray = NULL; + int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); continue; @@ -899,11 +906,12 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) { if (px1->skey == px2->skey) { return 0; } else { - return px1->skey < px2->skey? -1:1; + return px1->skey < px2->skey ? -1 : 1; } } -bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo *pScanInfo, int32_t order) { +bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, + int32_t order) { // check if it overlap with del skyline taosArraySort(pTimewindowList, sortUidComparFn); @@ -935,7 +943,7 @@ bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableB } static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, - int32_t startIndex) { + int32_t startIndex) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); for (int32_t i = startIndex; i < num; i += 1) { From 767feba65a24ec22b78b9aa48294667966cac19d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 16:49:02 +0800 Subject: [PATCH 222/236] test: revert failed case --- tests/system-test/8-stream/stream_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index 04e635ea24..3ebc255114 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -101,7 +101,7 @@ class TDTestCase: # check ts interval is correct sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" tdSql.query(sql) - tdSql.checkRows(1) + tdSql.checkRows(0) # stop def stop(self): From 75c25a194fd78626e3020afe080aa0dd90da3a5b Mon Sep 17 00:00:00 2001 From: zk66214 Date: Mon, 8 Jan 2024 17:19:48 +0800 Subject: [PATCH 223/236] add test cases for TD-28164 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/backslash_g.py | 72 ++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 tests/system-test/2-query/backslash_g.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3db6eccffc..336f58afa3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -546,6 +546,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/backslash_g.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py diff --git a/tests/system-test/2-query/backslash_g.py b/tests/system-test/2-query/backslash_g.py new file mode 100644 index 0000000000..69b06289c3 --- /dev/null +++ b/tests/system-test/2-query/backslash_g.py @@ -0,0 +1,72 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def checksql(self, sql): + result = os.popen(f"taos -s \"{sql}\" ") + res = result.read() + print(res) + if ("Query OK" in res): + tdLog.info(f"checkEqual success") + else : + tdLog.exit(f"checkEqual error") + + def td_28164(self): + tdSql.execute("drop database if exists td_28164;") + tdSql.execute("create database td_28164;") + tdSql.execute("create table td_28164.test (ts timestamp, name varchar(10));") + tdSql.execute("insert into td_28164.test values (now(), 'ac\\\\G') (now() + 1s, 'ac\\\\G') (now()+2s, 'ac\\G') (now()+3s, 'acG') (now()+4s, 'acK') ;") + + tdSql.query(f"select * from td_28164.test;") + tdSql.checkRows(5) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\\\\\G';") + tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\\\G';") + tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\G';") + tdSql.checkRows(2) + + # tdSql.query(f"select * from td_28164.test where name like 'ac\\\g';") + # tdSql.checkRows(0) + # + # tdSql.query(f"select * from td_28164.test where name like 'ac\\g';") + # tdSql.checkRows(0) + + self.checksql(f'select * from td_28164.test where name like \'ac\\G\'\G;') + # tdSql.checkRows(2) + + self.checksql(f"select * from td_28164.test where name like \'ac\\G\' \G;") + # tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac/\\G';") + tdSql.checkRows(0) + + tdSql.query(f"select * from td_28164.test where name like 'ac/G';") + tdSql.checkRows(0) + + def run(self): + # tdSql.prepare() + self.td_28164() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 5bd29a413b30df61024d4c6dfa5d1efbc669a02b Mon Sep 17 00:00:00 2001 From: fullhonest Date: Mon, 8 Jan 2024 15:04:04 +0800 Subject: [PATCH 224/236] Fix: not use last row scan when select tbname and last --- source/libs/planner/src/planOptimizer.c | 2 +- tests/system-test/2-query/last.py | 81 ++++++++++++++++++++ tests/system-test/2-query/last_cache_scan.py | 8 +- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index aa3181e166..035377d22e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2529,7 +2529,7 @@ static bool lastRowScanOptCheckFuncList(SLogicNode* pNode, bool* hasOtherFunc) { SNode* pParam = nodesListGetNode(pAggFunc->pParameterList, 0); if (QUERY_NODE_COLUMN == nodeType(pParam)) { SColumnNode* pCol = (SColumnNode*)pParam; - if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { + if (COLUMN_TYPE_COLUMN == pCol->colType && PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { if (selectNonPKColId != pCol->colId) { selectNonPKColId = pCol->colId; selectNonPKColNum++; diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index f4a1ab1790..a160aeab11 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -289,6 +289,82 @@ class TDTestCase: tdSql.query(f"select * from {stbname}") tdSql.checkRows(tables * rows) + def check_explain_res_has_row(self, plan_str_expect: str, rows, sql): + plan_found = False + for row in rows: + if str(row).find(plan_str_expect) >= 0: + tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row))) + plan_found = True + break + if not plan_found: + tdLog.exit("plan: %s not found in res: [%s] in sql: %s" % (plan_str_expect, str(rows), sql)) + + def check_explain_res_no_row(self, plan_str_not_expect: str, res, sql): + for row in res: + if str(row).find(plan_str_not_expect) >= 0: + tdLog.exit('plan: [%s] found in: [%s] for sql: %s' % (plan_str_not_expect, str(row), sql)) + + def explain_sql(self, sql: str): + sql = "explain " + sql + tdSql.query(sql, queryTimes=1) + return tdSql.queryResult + + def last_check_scan_type(self, cacheModel): + tdSql.execute("create database test_last_tbname cachemodel '%s';" % cacheModel) + tdSql.execute("use test_last_tbname;") + tdSql.execute("create stable test_last_tbname.st(ts timestamp, id int) tags(tid int);") + tdSql.execute("create table test_last_tbname.test_t1 using test_last_tbname.st tags(1);") + + maxRange = 100 + # 2023-11-13 00:00:00.000 + startTs = 1699804800000 + for i in range(maxRange): + insertSqlString = "insert into test_last_tbname.test_t1 values(%d, %d);" % (startTs + i, i) + tdSql.execute(insertSqlString) + + last_ts = startTs + maxRange + tdSql.execute("insert into test_last_tbname.test_t1 (ts) values(%d)" % (last_ts)) + sql = f'select tbname, last(ts) from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, "test_t1") + tdSql.checkData(0, 1, last_ts) + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + + sql = f'select last(ts), tbname from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, last_ts) + tdSql.checkData(0, 1, "test_t1") + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + sql = f'select tbname, last(ts), tbname from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, "test_t1") + tdSql.checkData(0, 1, last_ts) + tdSql.checkData(0, 2, "test_t1") + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + tdSql.execute("drop table if exists test_last_tbname.test_t1 ;") + tdSql.execute("drop stable if exists test_last_tbname.st;") + tdSql.execute("drop database if exists test_last_tbname;") def run(self): self.last_check_stb_tb_base() @@ -296,6 +372,11 @@ class TDTestCase: self.last_check_stb_distribute() self.last_file_check() + self.last_check_scan_type("none") + self.last_check_scan_type("last_row") + self.last_check_scan_type("last_value") + self.last_check_scan_type("both") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 23433277c2..1269bd4fde 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -181,7 +181,7 @@ class TDTestCase: select_items = [ "last(ts), ts", "last(ts), c1", "last(ts), c2", "last(ts), c3",\ "last(ts), c4", "last(ts), tbname", "last(ts), t1", "last(ts), ts, ts"] - has_last_row_scan_res = [1, 0, 0, 0, 0, 0, 0, 1] + has_last_row_scan_res = [1, 0, 0, 0, 0, 1, 1, 1] res_expect = [ ["2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"], None, None, None, None, None, None, @@ -193,7 +193,7 @@ class TDTestCase: select_items = ["last(c1),ts", "last(c1), c1", "last(c1), c2", "last(c1), c3",\ "last(c1), c4", "last(c1), tbname", "last(c1), t1", "last(c1), ts, ts", "last(c1), c1, c1"] - has_last_row_scan_res = [1, 1, 0, 0, 0, 0, 0, 1, 1] + has_last_row_scan_res = [1, 1, 0, 0, 0, 1, 1, 1, 1] res_expect = [ [999, "2018-11-25 19:30:00.000"], [999, 999], None, None, None, None, None, @@ -207,7 +207,7 @@ class TDTestCase: sql_template = 'select %s from t1' select_items = ["last(c4),ts", "last(c4), c1", "last(c4), c2", "last(c4), c3",\ "last(c4), c4", "last(c4), tbname", "last(c4), t1"] - has_last_row_scan_res = [1, 0, 0, 0, 1, 0, 0] + has_last_row_scan_res = [1, 0, 0, 0, 1, 1, 1] res_expect = [ [4999.000000000000000, "2018-11-25 19:30:00.000"], None,None,None, @@ -220,7 +220,7 @@ class TDTestCase: sql_template = 'select %s from meters' select_items = ["last(c8), ts", "last(c8), c1", "last(c8), c8", "last(c8), tbname", \ "last(c8), t1", "last(c8), c8, c8", "last(c8), ts, ts"] - has_last_row_scan_res = [1, 0, 1, 0, 0, 1, 1] + has_last_row_scan_res = [1, 0, 1, 1, 1, 1, 1] res_expect = [ ["binary9999", "2018-11-25 19:30:00.000"], None, From d93b4f88b38c7aa5616d9c58f0f097741bcc713a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Jan 2024 15:23:15 +0800 Subject: [PATCH 225/236] fix(tsdb/cache): destory brin block with fs loading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index a39c74a679..ab504acea7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1955,6 +1955,7 @@ typedef struct SFSNextRowIter { SArray *pIndexList; int32_t iBrinIndex; SBrinBlock brinBlock; + SBrinBlock *pBrinBlock; int32_t iBrinRecord; SBrinRecord brinRecord; SBlockData blockData; @@ -2143,6 +2144,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie pBrinBlk = taosArrayGet(state->pIndexList, state->iBrinIndex); } + if (!state->pBrinBlock) { + state->pBrinBlock = &state->brinBlock; + } else { + tBrinBlockClear(&state->brinBlock); + } code = tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2420,6 +2426,16 @@ int32_t clearNextRowFromFS(void *iter) { state->pBlockData = NULL; } + if (state->pBrinBlock) { + tBrinBlockDestroy(state->pBrinBlock); + state->pBrinBlock = NULL; + } + + if (state->pIndexList) { + taosArrayDestroy(state->pIndexList); + state->pIndexList = NULL; + } + if (state->pTSRow) { taosMemoryFree(state->pTSRow); state->pTSRow = NULL; From 404e17e67797f99f1c491171348ac463e0f99261 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 9 Jan 2024 16:18:55 +0800 Subject: [PATCH 226/236] fix: get expired user for each cluster --- source/client/src/clientHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1de5941ac3..b3d9a9ced5 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -902,6 +902,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } + taosHashPut(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(uint64_t), NULL, 0); } // invoke after hbGetExpiredUserInfo @@ -1121,7 +1122,6 @@ static void *hbThreadFunc(void *param) { asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); - taosHashPut(clientHbMgr.appHbHash, &pAppHbMgr->pAppInstInfo->clusterId, sizeof(uint64_t), NULL, 0); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } From 6444af84e47f5ed90f1af99e7e549a2be968f97b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 9 Jan 2024 16:28:58 +0800 Subject: [PATCH 227/236] opti:[TD-28142]memory size if block num is too large --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 109 ++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 25 ++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 19 +++- source/util/src/tarray.c | 11 ++- 4 files changed, 110 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index b57d2dfb45..255d5d4105 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -882,6 +882,21 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo } } +static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo){ + record->uid = pBlockInfo->uid; + record->firstKey = pBlockInfo->firstKey; + record->lastKey = pBlockInfo->lastKey; + record->minVer = pBlockInfo->minVer; + record->maxVer = pBlockInfo->maxVer; + record->blockOffset = pBlockInfo->blockOffset; + record->smaOffset = pBlockInfo->smaOffset; + record->blockSize = pBlockInfo->blockSize; + record->blockKeySize = pBlockInfo->blockKeySize; + record->smaSize = pBlockInfo->smaSize; + record->numRow = pBlockInfo->numRow; + record->count = pBlockInfo->count; +} + static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -899,7 +914,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; - SBrinRecord* pRecord = &pBlockInfo->record; + SBrinRecord tmp; + blockInfoToRecord(&tmp, pBlockInfo); + SBrinRecord* pRecord = &tmp; // no data exists, return directly. if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) { @@ -1092,14 +1109,16 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBrinRecord* pRecord = &pBlockInfo->record; + SBrinRecord tmp; + blockInfoToRecord(&tmp, pBlockInfo); + SBrinRecord* pRecord = &tmp; code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1], pSup->numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, code:%s %s", - pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.firstKey, - pBlockInfo->record.lastKey, pBlockInfo->record.numRow, tstrerror(code), pReader->idStr); + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->firstKey, + pBlockInfo->lastKey, pBlockInfo->numRow, tstrerror(code), pReader->idStr); return code; } @@ -1120,10 +1139,10 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI * This is an two rectangles overlap cases. */ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) { - return (pWindow->ekey < pBlock->record.lastKey && pWindow->ekey >= pBlock->record.firstKey) || - (pWindow->skey > pBlock->record.firstKey && pWindow->skey <= pBlock->record.lastKey) || - (pVerRange->minVer > pBlock->record.minVer && pVerRange->minVer <= pBlock->record.maxVer) || - (pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer); + return (pWindow->ekey < pBlock->lastKey && pWindow->ekey >= pBlock->firstKey) || + (pWindow->skey > pBlock->firstKey && pWindow->skey <= pBlock->lastKey) || + (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || + (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, @@ -1142,7 +1161,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); - memcpy(pRecord, &p->record, sizeof(SBrinRecord)); + blockInfoToRecord(pRecord, p); *nextIndex = pBlockInfo->tbBlockIdx + step; return true; @@ -1211,9 +1230,9 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { - return pBlock->record.lastKey == pRec->firstKey; + return pBlock->lastKey == pRec->firstKey; } else { - return pBlock->record.firstKey == pRec->lastKey; + return pBlock->firstKey == pRec->lastKey; } } @@ -1223,9 +1242,9 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanI int64_t key = 0; if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; - key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt); + key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt); } else { - key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey; + key = ascScan ? pBlock->firstKey : pBlock->lastKey; } return key; @@ -1241,8 +1260,8 @@ static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBloc } static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) { - return (key.ts >= pBlock->record.firstKey && key.ts <= pBlock->record.lastKey) && - (pBlock->record.maxVer >= pVerRange->minVer) && (pBlock->record.minVer <= pVerRange->maxVer); + return (key.ts >= pBlock->firstKey && key.ts <= pBlock->lastKey) && + (pBlock->maxVer >= pVerRange->minVer) && (pBlock->minVer <= pVerRange->maxVer); } static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, @@ -1258,18 +1277,20 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order); } + SBrinRecord pRecord; + blockInfoToRecord(&pRecord, pBlockInfo); // has duplicated ts of different version in this block - pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count) || (pBlockInfo->record.count <= 0); - pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order); + pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0); + pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, pReader->info.order); ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey; pInfo->overlapWithSttBlock = - !(pBlockInfo->record.lastKey < nextProcKeyInStt || pBlockInfo->record.firstKey > nextProcKeyInStt); + !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt); } - pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity; + pInfo->moreThanCapcity = pBlockInfo->numRow > pReader->resBlockInfo.capacity; pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo); pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange); } @@ -2361,17 +2382,14 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBrinRecord* pRecord = NULL; STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo == NULL) { return 0; } - pRecord = &pBlockInfo->record; - if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; } @@ -2380,13 +2398,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { goto _end; } - pRecord = &pBlockInfo->record; TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly int64_t cap = pReader->resBlockInfo.capacity; - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) { - if (((asc && (pRecord->firstKey < keyInBuf.ts)) || (!asc && (pRecord->lastKey > keyInBuf.ts))) && + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pBlockInfo->numRow <= cap)) { + if (((asc && (pBlockInfo->firstKey < keyInBuf.ts)) || (!asc && (pBlockInfo->lastKey > keyInBuf.ts))) && (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) { code = copyBlockDataToSDataBlock(pReader); if (code) { @@ -2394,7 +2411,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } // record the last key value - pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey; + pBlockScanInfo->lastProcKey = asc ? pBlockInfo->lastKey : pBlockInfo->firstKey; goto _end; } } @@ -2422,7 +2439,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { bool loadNeighbor = false; code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor); if ((!loadNeighbor) || (code != 0)) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); break; } } @@ -2441,7 +2458,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); break; } @@ -2801,7 +2818,7 @@ static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockSc return true; } else { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; - return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt); + return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt); } } @@ -2815,7 +2832,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; } @@ -2850,20 +2867,20 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; - pInfo->rows = pBlockInfo->record.numRow; + pInfo->rows = pBlockInfo->numRow; pInfo->id.uid = pScanInfo->uid; pInfo->dataLoad = 0; - pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey}; + pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey}; setComposedBlockFlag(pReader, false); - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); // update the last key for the corresponding table pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey; tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, " "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", - pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow, - pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, + pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); } else { SBlockData* pBData = &pReader->status.fileBlockData; tBlockDataReset(pBData); @@ -2892,10 +2909,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // data in stt now overlaps with current active file data block, need to composed with file data block. int64_t lastKeyInStt = getCurrentKeyInSttBlock(pSttBlockReader); - if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) || - (lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) { + if ((lastKeyInStt >= pBlockInfo->firstKey && asc) || + (lastKeyInStt <= pBlockInfo->lastKey && (!asc))) { tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader, - lastKeyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + lastKeyInStt, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); break; } } @@ -3012,8 +3029,8 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) lastKey = pScanInfo->lastProcKey; } - pDumpInfo->totalRows = pBlockInfo->record.numRow; - pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->record.numRow - 1; + pDumpInfo->totalRows = pBlockInfo->numRow; + pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { pDumpInfo->totalRows = 0; pDumpInfo->rowIndex = 0; @@ -4632,7 +4649,9 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, // int64_t st = taosGetTimestampUs(); TARRAY2_CLEAR(&pSup->colAggArray, 0); - code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pFBlock->record, &pSup->colAggArray); + SBrinRecord pRecord; + blockInfoToRecord(&pRecord, pFBlock); + code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pRecord, &pSup->colAggArray); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), pReader->idStr); @@ -4663,7 +4682,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pFBlock->record.numRow, numOfCols, pTsAgg); + doFillNullColSMA(pSup, pFBlock->numRow, numOfCols, pTsAgg); size_t size = pSup->colAggArray.size; @@ -4884,7 +4903,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT while (true) { if (hasNext) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); - int32_t numOfRows = pBlockInfo->record.numRow; + int32_t numOfRows = pBlockInfo->numRow; pTableBlockInfo->totalRows += numOfRows; @@ -4896,7 +4915,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT pTableBlockInfo->minRows = numOfRows; } - pTableBlockInfo->totalSize += pBlockInfo->record.blockSize; + pTableBlockInfo->totalSize += pBlockInfo->blockSize; int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets); pTableBlockInfo->blockRowsHisto[bucketIndex]++; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 33b7667168..b4b92c0008 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -359,6 +359,21 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; } +static void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record){ + pBlockInfo->uid = record->uid; + pBlockInfo->firstKey = record->firstKey; + pBlockInfo->lastKey = record->lastKey; + pBlockInfo->minVer = record->minVer; + pBlockInfo->maxVer = record->maxVer; + pBlockInfo->blockOffset = record->blockOffset; + pBlockInfo->smaOffset = record->smaOffset; + pBlockInfo->blockSize = record->blockSize; + pBlockInfo->blockKeySize = record->blockKeySize; + pBlockInfo->smaSize = record->smaSize; + pBlockInfo->numRow = record->numRow; + pBlockInfo->count = record->count; +} + int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -416,8 +431,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 pTableScanInfo->pBlockIdxList = taosArrayInit(numOfBlocks, sizeof(STableDataBlockIdx)); } for (int32_t i = 0; i < numOfBlocks; ++i) { - SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; - blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + SFileDataBlockInfo blockInfo = {.tbBlockIdx = i}; + SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + recordToBlockInfo(&blockInfo, record); taosArrayPush(pBlockIter->blockList, &blockInfo); STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; @@ -450,8 +466,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 int32_t pos = tMergeTreeGetChosenIndex(pTree); int32_t index = sup.indexPerTable[pos]++; - SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[pos][index].uid, .tbBlockIdx = index}; - blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + SFileDataBlockInfo blockInfo = {.tbBlockIdx = index}; + SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + recordToBlockInfo(&blockInfo, record); taosArrayPush(pBlockIter->blockList, &blockInfo); STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 39e65f22b1..cd9942a0a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -175,9 +175,22 @@ typedef struct SFilesetIter { typedef struct SFileDataBlockInfo { // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it - uint64_t uid; - int32_t tbBlockIdx; - SBrinRecord record; + // int64_t suid; + int64_t uid; + int64_t firstKey; +// int64_t firstKeyVer; + int64_t lastKey; +// int64_t lastKeyVer; + int64_t minVer; + int64_t maxVer; + int64_t blockOffset; + int64_t smaOffset; + int32_t blockSize; + int32_t blockKeySize; + int32_t smaSize; + int32_t numRow; + int32_t count; + int32_t tbBlockIdx; } SFileDataBlockInfo; typedef struct SDataBlockIter { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index bc98461592..88eb51d500 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -18,6 +18,9 @@ #include "tcoding.h" // todo refactor API +#define BOUNDARY_SIZE 1024*1024*1024 // 1G +#define BOUNDARY_SMALL_FACTOR 1.2 +#define BOUNDARY_BIG_FACTOR 2 SArray* taosArrayInit(size_t size, size_t elemSize) { if (elemSize == 0) { @@ -85,9 +88,13 @@ static int32_t taosArrayResize(SArray* pArray) { int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { if (newCap > pArray->capacity) { - size_t tsize = (pArray->capacity << 1u); + float factor = BOUNDARY_BIG_FACTOR; + if(newCap * pArray->elemSize > BOUNDARY_SIZE){ + factor = BOUNDARY_SMALL_FACTOR; + } + size_t tsize = (pArray->capacity * factor); while (newCap > tsize) { - tsize = (tsize << 1u); + tsize = (tsize * factor); } pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); From 220cef4a61c1c2a8d857c812549b2ab378e6dd47 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 9 Jan 2024 11:02:10 +0000 Subject: [PATCH 228/236] opt msg on mnode --- source/dnode/mnode/impl/src/mndMain.c | 164 +++++++++++++++++--------- 1 file changed, 109 insertions(+), 55 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 38a92b43e7..c0fd93c65d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" #include "mndCluster.h" +#include "mndCompact.h" +#include "mndCompactDetail.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -42,8 +44,6 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndView.h" -#include "mndCompact.h" -#include "mndCompactDetail.h" static inline int32_t mndAcquireRpc(SMnode *pMnode) { int32_t code = 0; @@ -158,16 +158,16 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { } } -static void mndStreamCheckpointRemain(SMnode* pMnode) { +static void mndStreamCheckpointRemain(SMnode *pMnode) { int32_t contLen = 0; - void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); + void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } -static void mndStreamCheckNode(SMnode* pMnode) { +static void mndStreamCheckNode(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -275,6 +275,100 @@ static void mndCheckDnodeOffline(SMnode *pMnode) { mndReleaseRpc(pMnode); } +static bool mnodeIsNotLeader(SMnode *pMnode) { + terrno = 0; + taosThreadRwlockRdlock(&pMnode->lock); + SSyncState state = syncGetState(pMnode->syncMgmt.sync); + if (terrno != 0) { + taosThreadRwlockUnlock(&pMnode->lock); + return true; + } + + if (state.state != TAOS_SYNC_STATE_LEADER) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_NOT_LEADER; + return true; + } + if (!state.restored || !pMnode->restored) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_RESTORING; + return true; + } + taosThreadRwlockUnlock(&pMnode->lock); + return false; +} + +static int32_t minCronTime() { + int64_t min = INT64_MAX; + min = TMIN(min, tsTtlPushIntervalSec); + min = TMIN(min, tsTrimVDbIntervalSec); + min = TMIN(min, tsTransPullupInterval); + min = TMIN(min, tsCompactPullupInterval); + min = TMIN(min, tsMqRebalanceInterval); + min = TMIN(min, tsStreamCheckpointInterval); + min = TMIN(min, 5); // checkpointRemain + min = TMIN(min, tsStreamNodeCheckInterval); + + int64_t telemInt = TMIN(60, (tsTelemInterval - 1)); + min = TMIN(min, telemInt); + min = TMIN(min, tsGrantHBInterval); + min = TMIN(min, tsUptimeInterval); + + return min <= 1 ? 2 : min; +} +void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { + if (sec % tsTtlPushIntervalSec == 0) { + mndPullupTtl(pMnode); + } + + if (sec % tsTrimVDbIntervalSec == 0) { + mndPullupTrimDb(pMnode); + } + + if (sec % tsTransPullupInterval == 0) { + mndPullupTrans(pMnode); + } + + if (sec % tsCompactPullupInterval == 0) { + mndPullupCompacts(pMnode); + } + + if (sec % tsMqRebalanceInterval == 0) { + mndCalMqRebalance(pMnode); + } + + if (sec % tsStreamCheckpointInterval == 0) { + mndStreamCheckpointTick(pMnode, sec); + } + + if (sec % 5 == 0) { + mndStreamCheckpointRemain(pMnode); + } + + if (sec % tsStreamNodeCheckInterval == 0) { + mndStreamCheckNode(pMnode); + } + + if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { + mndPullupTelem(pMnode); + } + + if (sec % tsGrantHBInterval == 0) { + mndPullupGrant(pMnode); + } + + if (sec % tsUptimeInterval == 0) { + mndIncreaseUpTime(pMnode); + } +} +void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) { + if (sec % (tsStatusInterval * 5) == 0) { + mndCheckDnodeOffline(pMnode); + } + if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) { + mndSyncCheckTimeout(pMnode); + } +} static void *mndThreadFp(void *param) { SMnode *pMnode = param; int64_t lastTime = 0; @@ -287,57 +381,15 @@ static void *mndThreadFp(void *param) { if (lastTime % 10 != 0) continue; int64_t sec = lastTime / 10; - if (sec % tsTtlPushIntervalSec == 0) { - mndPullupTtl(pMnode); - } + mndDoTimerCheckTask(pMnode, sec); - if (sec % tsTrimVDbIntervalSec == 0) { - mndPullupTrimDb(pMnode); - } - - if (sec % tsTransPullupInterval == 0) { - mndPullupTrans(pMnode); - } - - if (sec % tsCompactPullupInterval == 0) { - mndPullupCompacts(pMnode); - } - - if (sec % tsMqRebalanceInterval == 0) { - mndCalMqRebalance(pMnode); - } - - if (sec % tsStreamCheckpointInterval == 0) { - mndStreamCheckpointTick(pMnode, sec); - } - - if (sec % 5 == 0) { - mndStreamCheckpointRemain(pMnode); - } - - if (sec % tsStreamNodeCheckInterval == 0) { - mndStreamCheckNode(pMnode); - } - - if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { - mndPullupTelem(pMnode); - } - - if (sec % tsGrantHBInterval == 0) { - mndPullupGrant(pMnode); - } - - if (sec % tsUptimeInterval == 0) { - mndIncreaseUpTime(pMnode); - } - - if (sec % (tsStatusInterval * 5) == 0) { - mndCheckDnodeOffline(pMnode); - } - - if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) { - mndSyncCheckTimeout(pMnode); + int64_t minCron = minCronTime(); + if (sec % minCron == 0 && mnodeIsNotLeader(pMnode)) { + // not leader, do nothing + mTrace("timer not process since mnode is not leader, reason: %s", tstrerror(terrno)) terrno = 0; + continue; } + mndDoTimerPullupTask(pMnode, sec); } return NULL; @@ -712,7 +764,9 @@ _OVER: if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER || - pMsg->msgType == TDMT_MND_COMPACT_TIMER) { + pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || + pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE || + pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, pMnode->stopped, state.restored, syncStr(state.state)); return -1; From eac0d148685fd67faa91cdc1b92d2a4813d7a74c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Jan 2024 11:02:44 +0800 Subject: [PATCH 229/236] fix(stream): add check when merging rows from files and buffer. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 104 +++------------------ source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 9 +- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 6 ++ 3 files changed, 21 insertions(+), 98 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 255d5d4105..68ba00eefc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1490,7 +1490,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLast = INT64_MIN; - if (hasDataInSttBlock(pBlockScanInfo)) { + if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { tsLast = getCurrentKeyInSttBlock(pSttBlockReader); } @@ -1509,7 +1509,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* int64_t minKey = 0; if (pReader->info.order == TSDB_ORDER_ASC) { minKey = INT64_MAX; // chosen the minimum value - if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } @@ -1522,7 +1522,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } else { minKey = INT64_MIN; - if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } @@ -1626,91 +1626,6 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } -static int32_t doMergeFileBlockAndLastBlock(SSttBlockReader* pSttBlockReader, STsdbReader* pReader, - STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, - bool mergeBlockData) { - SRowMerger* pMerger = &pReader->status.merger; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - - int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); - bool copied = false; - int32_t code = TSDB_CODE_SUCCESS; - SRow* pTSRow = NULL; - TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - - // create local variable to hold the row value - TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; - - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid, - fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr); - - // only stt block exists - if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); - if (code) { - return code; - } - - if (copied) { - pBlockScanInfo->lastProcKey = tsLastBlock; - return TSDB_CODE_SUCCESS; - } else { - code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); - - code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - - taosMemoryFree(pTSRow); - tsdbRowMergerClear(pMerger); - - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } else { // not merge block data - code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); - - // merge with block data if ts == key - if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); - } - - code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - - taosMemoryFree(pTSRow); - tsdbRowMergerClear(pMerger); - - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - return TSDB_CODE_SUCCESS; -} - static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* pSttBlockReader, int64_t key, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1816,7 +1731,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); int64_t tsLast = INT64_MIN; - if (hasDataInSttBlock(pBlockScanInfo)) { + if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { tsLast = getCurrentKeyInSttBlock(pSttBlockReader); } @@ -1865,7 +1780,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = key; } - if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } } else { @@ -1882,7 +1797,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = key; } - if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } } @@ -2169,10 +2084,15 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pScanInfo->sttKeyInfo.nextProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; hasData = true; - } else { + } else { // not clean stt blocks + INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } } else { + pScanInfo->cleanSttBlocks = false; + INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index b4b92c0008..33604d21de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -22,12 +22,6 @@ #include "tsdbUtil2.h" #include "tsimplehash.h" -#define INIT_TIMEWINDOW(_w) \ - do { \ - (_w)->skey = INT64_MAX; \ - (_w)->ekey = INT64_MIN; \ - } while (0); - static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order); @@ -165,6 +159,9 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf INIT_TIMEWINDOW(&pScanInfo->sttWindow); INIT_TIMEWINDOW(&pScanInfo->filesetWindow); + pScanInfo->cleanSttBlocks = false; + pScanInfo->sttBlockReturned = false; + pUidList->tableUidList[j] = idList[j].uid; if (ASCENDING_TRAVERSE(pTsdbReader->info.order)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index cd9942a0a6..7a93e73a07 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -26,6 +26,12 @@ extern "C" { #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define INIT_TIMEWINDOW(_w) \ + do { \ + (_w)->skey = INT64_MAX; \ + (_w)->ekey = INT64_MIN; \ + } while (0); + typedef enum { READER_STATUS_SUSPEND = 0x1, READER_STATUS_NORMAL = 0x2, From 1b6d55616fd4c5e036b525d13464449db115768f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Jan 2024 11:28:06 +0800 Subject: [PATCH 230/236] fix(stream): fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 68ba00eefc..7360f59301 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2085,13 +2085,13 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; hasData = true; } else { // not clean stt blocks - INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + INIT_TIMEWINDOW(&pScanInfo->sttWindow); //reset the time window pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } } else { pScanInfo->cleanSttBlocks = false; - INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + INIT_TIMEWINDOW(&pScanInfo->sttWindow); //reset the time window pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } From 7744a31deb3dce64623334a67aaa0949b6f4cd8b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 10 Jan 2024 11:29:24 +0800 Subject: [PATCH 231/236] case: fix memory leak with return cause --- tests/army/enterprise/s3/s3_basic.py | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 112cb65b15..8e7c1f2d35 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -86,30 +86,29 @@ class TDTestCase(TBase): tdLog.debug(f"start to excute {__file__}") if eos.isArm64Cpu(): tdLog.success(f"{__file__} arm64 ignore executed") - return + else: + # insert data + self.insertData() - # insert data - self.insertData() + # check insert data correct + self.checkInsertCorrect() - # check insert data correct - self.checkInsertCorrect() + # save + self.snapshotAgg() - # save - self.snapshotAgg() + # do action + self.doAction() - # do action - self.doAction() + # check save agg result correct + self.checkAggCorrect() - # check save agg result correct - self.checkAggCorrect() + # check insert correct again + self.checkInsertCorrect() - # check insert correct again - self.checkInsertCorrect() + # drop database and free s3 file + self.dropDb() - # drop database and free s3 file - self.dropDb() - - tdLog.success(f"{__file__} successfully executed") + tdLog.success(f"{__file__} successfully executed") From b0202bb8536718d117e89aa527b77603ddaf9c57 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Jan 2024 21:40:27 +0800 Subject: [PATCH 232/236] fix(tsdb): open reader during resume task for main reader after suspending reader. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 100 ++++++++++++++---------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 7360f59301..62be0968e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -676,6 +676,10 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { + if (pBlockIter->blockList == NULL) { + return NULL; + } + size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); @@ -1969,30 +1973,34 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea return TSDB_CODE_SUCCESS; } -static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, - STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { +static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STableBlockScanInfo* pBlockScanInfo, bool asc, + STsdbReaderInfo* pInfo) { // it is an multi-table data block if (pBlockData->aUid != NULL) { - uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex]; + uint64_t uid = pBlockData->aUid[rowIndex]; if (uid != pBlockScanInfo->uid) { // move to next row return false; } } // check for version and time range - int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex]; - if (ver > pReader->info.verRange.maxVer || ver < pReader->info.verRange.minVer) { + int64_t ver = pBlockData->aVersion[rowIndex]; + if (ver > pInfo->verRange.maxVer || ver < pInfo->verRange.minVer) { return false; } - int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - if (ts > pReader->info.window.ekey || ts < pReader->info.window.skey) { + int64_t ts = pBlockData->aTSKEY[rowIndex]; + if (ts > pInfo->window.ekey || ts < pInfo->window.skey) { + return false; + } + + if ((asc && (ts <= pBlockScanInfo->lastProcKey)) || ((!asc) && (ts >= pBlockScanInfo->lastProcKey))) { return false; } if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { bool dropped = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, - pReader->info.order, &pReader->info.verRange); + pInfo->order, &pInfo->verRange); if (dropped) { return false; } @@ -2344,7 +2352,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { { while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block - if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + if (isValidFileBlockRow(pBlockData, pDumpInfo->rowIndex, pBlockScanInfo, asc, &pReader->info)) { hasBlockData = true; break; } @@ -4130,37 +4138,21 @@ void tsdbReaderClose2(STsdbReader* pReader) { taosMemoryFreeClear(pReader); } -int32_t tsdbReaderSuspend2(STsdbReader* pReader) { - // save reader's base state & reset top state to be reconstructed from base state - int32_t code = 0; - SReaderStatus* pStatus = &pReader->status; - STableBlockScanInfo* pBlockScanInfo = NULL; - - pReader->status.suspendInvoked = true; // record the suspend status +static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { + SReaderStatus* pStatus = &pCurrentReader->status; if (pStatus->loadFromFile) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - if (pBlockInfo != NULL) { - pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { - goto _err; - } - } else { - pBlockScanInfo = *pStatus->pTableIter; - } + tsdbDataFileReaderClose(&pCurrentReader->pFileReader); - tsdbDataFileReaderClose(&pReader->pFileReader); - - SReadCostSummary* pCost = &pReader->cost; - pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); - pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + SReadCostSummary* pCost = &pCurrentReader->cost; + pStatus->pLDataIterArray = destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); + pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); } // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; - + int32_t step = ASCENDING_TRAVERSE(pCurrentReader->info.order) ? 1 : -1; int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -4171,6 +4163,26 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { pStatus->uidList.currentIndex = 0; initReaderStatus(pStatus); + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbReaderSuspend2(STsdbReader* pReader) { + // save reader's base state & reset top state to be reconstructed from base state + int32_t code = 0; + pReader->status.suspendInvoked = true; // record the suspend status + + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + if (pReader->step == EXTERNAL_ROWS_PREV) { + doSuspendCurrentReader(pReader->innerReader[0]); + } else if (pReader->step == EXTERNAL_ROWS_MAIN) { + doSuspendCurrentReader(pReader); + } else { + doSuspendCurrentReader(pReader->innerReader[1]); + } + } else { + doSuspendCurrentReader(pReader); + } + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; if (pReader->bFilesetDelimited) { @@ -4179,16 +4191,16 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { } pReader->flag = READER_STATUS_SUSPEND; + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + clearSharedPtr(pReader->innerReader[0]); + clearSharedPtr(pReader->innerReader[1]); + } + #if SUSPEND_RESUME_TEST tsem_post(&pReader->resumeAfterSuspend); #endif - tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, - pReader->idStr); - return code; - -_err: - tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbDebug("reader: %p suspended in this query %s, step:%d", pReader, pReader->idStr, pReader->step); return code; } @@ -4219,8 +4231,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { int32_t code = 0; STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; - // restore reader's state - // task snapshot + // restore reader's state, task snapshot int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { qTrace("tsdb/reader: %p, take snapshot", pReader); @@ -4245,7 +4256,14 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { pNextReader->resBlockInfo.capacity = 1; setSharedPtr(pNextReader, pReader); - code = doOpenReaderImpl(pPrevReader); + if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) { + code = doOpenReaderImpl(pPrevReader); + } else if (pReader->step == EXTERNAL_ROWS_MAIN) { + code = doOpenReaderImpl(pReader); + } else { + code = doOpenReaderImpl(pNextReader); + } + if (code != TSDB_CODE_SUCCESS) { return code; } From 27b63b721fb40e96a74f920a50edbf9040d5d5bf Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Jan 2024 17:36:51 +0800 Subject: [PATCH 233/236] fix:decode mqDataMsg error for the message with old version --- source/common/src/tmsg.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3ff814dda9..c9e2908e8a 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -8334,7 +8334,7 @@ int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { return 0; } -int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { +int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; @@ -8356,11 +8356,16 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { } } } + return 0; +} + +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { + if (tEncodeMqDataRspCommon(pEncoder, pRsp) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1; return 0; } -int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { +int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; @@ -8402,7 +8407,15 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } } } - if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + + return 0; +} + +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { + if (tDecodeMqDataRspCommon(pDecoder, pRsp) < 0) return -1; + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + } return 0; } @@ -8418,7 +8431,7 @@ void tDeleteMqDataRsp(SMqDataRsp *pRsp) { } int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { - if (tEncodeMqDataRsp(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; + if (tEncodeMqDataRspCommon(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { @@ -8432,7 +8445,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { } int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { - if (tDecodeMqDataRsp(pDecoder, (SMqDataRsp *)pRsp) < 0) return -1; + if (tDecodeMqDataRspCommon(pDecoder, (SMqDataRsp*)pRsp) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { @@ -8447,6 +8460,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { taosArrayPush(pRsp->createTableReq, &pCreate); } } + return 0; } From df69a89da202a60df6915c2653e9af00f74aa6a7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Jan 2024 18:46:10 +0800 Subject: [PATCH 234/236] fix:[TD-28185]add pause & resume logic for snode --- include/dnode/vnode/tqCommon.h | 2 + source/dnode/snode/src/snode.c | 6 +- source/dnode/vnode/src/tq/tq.c | 101 +------------------- source/dnode/vnode/src/tqCommon/tqCommon.c | 105 +++++++++++++++++++++ source/libs/stream/src/streamMeta.c | 8 +- 5 files changed, 118 insertions(+), 104 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index da2325f006..82c4102369 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -33,5 +33,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg); +int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); #endif // TDENGINE_TQ_COMMON_H diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 33eb9cd3ed..9177559f61 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -195,7 +195,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_TASK_DEPLOY: { void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); - return tqStreamTaskProcessDeployReq(pSnode->pMeta, -1, pReq, len, true, true); + return tqStreamTaskProcessDeployReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pReq, len, true, true); } case TDMT_STREAM_TASK_DROP: @@ -204,6 +204,10 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return tqStreamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); case TDMT_VND_STREAM_TASK_RESET: return tqStreamTaskProcessTaskResetReq(pSnode->pMeta, pMsg); + case TDMT_STREAM_TASK_PAUSE: + return tqStreamTaskProcessTaskPauseReq(pSnode->pMeta, pMsg->pCont); + case TDMT_STREAM_TASK_RESUME: + return tqStreamTaskProcessTaskResumeReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pMsg->pCont, false); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 65cc56c0b4..275bbffe25 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1094,108 +1094,11 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { } int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)msg; - - SStreamMeta* pMeta = pTq->pStreamMeta; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask == NULL) { - tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, - pReq->taskId); - // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active - return TSDB_CODE_SUCCESS; - } - - tqDebug("s-task:%s receive pause msg from mnode", pTask->id.idStr); - streamTaskPause(pTask, pMeta); - - SStreamTask* pHistoryTask = NULL; - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); - if (pHistoryTask == NULL) { - tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 - ", it may have been dropped already", - pMeta->vgId, pTask->hTaskInfo.id.taskId); - streamMetaReleaseTask(pMeta, pTask); - - // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active - return TSDB_CODE_SUCCESS; - } - - tqDebug("s-task:%s fill-history task handle paused along with related stream task", pHistoryTask->id.idStr); - - streamTaskPause(pHistoryTask, pMeta); - streamMetaReleaseTask(pMeta, pHistoryTask); - } - - streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; -} - -int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, int8_t igUntreated) { - int32_t vgId = pTq->pStreamMeta->vgId; - if (pTask == NULL) { - return -1; - } - - streamTaskResume(pTask); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); - - int32_t level = pTask->info.taskLevel; - if (level == TASK_LEVEL__SINK) { - if (status == TASK_STATUS__UNINIT) { - } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; - } - - if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) { - // no lock needs to secure the access of the version - if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { - // discard all the data when the stream task is suspended. - walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); - tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 - ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); - } else { // from the previous paused version and go on - tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); - } - - if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { - streamStartScanHistoryAsync(pTask, igUntreated); - } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { - tqScanWalAsync(pTq, false); - } else { - streamSchedExec(pTask); - } - } else if (status == TASK_STATUS__UNINIT) { - // todo: fill-history task init ? - if (pTask->info.fillHistory == 0) { - EStreamTaskEvent event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; - streamTaskHandleEvent(pTask->status.pSM, event); - } - } - - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; + return tqStreamTaskProcessTaskPauseReq(pTq->pStreamMeta, msg); } int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, pReq->taskId); - int32_t code = tqProcessTaskResumeImpl(pTq, pTask, sversion, pReq->igUntreated); - if (code != 0) { - return code; - } - - STaskId* pHTaskId = &pTask->hTaskInfo.id; - SStreamTask* pHistoryTask = streamMetaAcquireTask(pTq->pStreamMeta, pHTaskId->streamId, pHTaskId->taskId); - if (pHistoryTask) { - code = tqProcessTaskResumeImpl(pTq, pHistoryTask, sversion, pReq->igUntreated); - } - - return code; + return tqStreamTaskProcessTaskResumeReq(pTq, sversion, msg, true); } int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a3d860fd78..70bfa00609 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -752,4 +752,109 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; +} + +int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg){ + SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, + pReq->taskId); + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s receive pause msg from mnode", pTask->id.idStr); + streamTaskPause(pTask, pMeta); + + SStreamTask* pHistoryTask = NULL; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); + if (pHistoryTask == NULL) { + tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 + ", it may have been dropped already", + pMeta->vgId, pTask->hTaskInfo.id.taskId); + streamMetaReleaseTask(pMeta, pTask); + + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s fill-history task handle paused along with related stream task", pHistoryTask->id.idStr); + + streamTaskPause(pHistoryTask, pMeta); + streamMetaReleaseTask(pMeta, pHistoryTask); + } + + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; +} + +static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t sversion, int8_t igUntreated, bool fromVnode) { + SStreamMeta *pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; + int32_t vgId = pMeta->vgId; + if (pTask == NULL) { + return -1; + } + + streamTaskResume(pTask); + ETaskStatus status = streamTaskGetStatus(pTask, NULL); + + int32_t level = pTask->info.taskLevel; + if (level == TASK_LEVEL__SINK) { + if (status == TASK_STATUS__UNINIT) { + } + streamMetaReleaseTask(pMeta, pTask); + return 0; + } + + if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) { + // no lock needs to secure the access of the version + if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { + // discard all the data when the stream task is suspended. + walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); + tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 + ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); + } else { // from the previous paused version and go on + tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); + } + + if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { + streamStartScanHistoryAsync(pTask, igUntreated); + } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { + tqScanWalAsync((STQ*)handle, false); + } else { + streamSchedExec(pTask); + } + } else if (status == TASK_STATUS__UNINIT) { + // todo: fill-history task init ? + if (pTask->info.fillHistory == 0) { + EStreamTaskEvent event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; + streamTaskHandleEvent(pTask->status.pSM, event); + } + } + + streamMetaReleaseTask(pMeta, pTask); + return 0; +} + +int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode){ + SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; + SStreamMeta *pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + int32_t code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); + if (code != 0) { + return code; + } + + STaskId* pHTaskId = &pTask->hTaskInfo.id; + SStreamTask* pHistoryTask = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); + if (pHistoryTask) { + code = tqProcessTaskResumeImpl(pMeta, pHistoryTask, sversion, pReq->igUntreated, fromVnode); + } + + return code; } \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7ea80ad8d8..4e30dc2555 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -377,16 +377,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->rid = taosAddRef(streamMetaId, pMeta); // set the attribute when running on Linux OS -#if defined LINUX TdThreadRwlockAttr attr; taosThreadRwlockAttrInit(&attr); +#ifdef LINUX pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - taosThreadRwlockInit(&pMeta->lock, &attr); - - taosThreadRwlockAttrDestroy(&attr); #endif + taosThreadRwlockInit(&pMeta->lock, &attr); + taosThreadRwlockAttrDestroy(&attr); + int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); metaRefMgtAdd(pMeta->vgId, pRid); From fb42c6afd5a06d7a748afa69df6e9436a31528f6 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 10 Jan 2024 19:59:44 +0800 Subject: [PATCH 235/236] coverage: update coverage script file and BlockSMA trigger is 1 --- tests/script/coverage_test.sh | 363 +++++++++++--------------- tests/system-test/2-query/blockSMA.py | 2 +- 2 files changed, 160 insertions(+), 205 deletions(-) mode change 100755 => 100644 tests/script/coverage_test.sh diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh old mode 100755 new mode 100644 index 535b6802f2..1b9e9753d3 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -12,7 +12,6 @@ fi today=`date +"%Y%m%d"` TDENGINE_DIR=/root/TDengine JDBC_DIR=/root/taos-connector-jdbc -TAOSKEEPER_DIR=/root/taoskeeper TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log # Color setting @@ -23,52 +22,52 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' function buildTDengine() { - echo "check if TDengine need build" - cd $TDENGINE_DIR - git remote prune origin > /dev/null - git remote update > /dev/null - REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` - LOCAL_COMMIT=`git rev-parse --short @` - echo " LOCAL: $LOCAL_COMMIT" - echo "REMOTE: $REMOTE_COMMIT" + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" - # reset counter - lcov -d . --zerocounters + # reset counter + lcov -d . --zerocounters - if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then - echo "repo up-to-date" - else - echo "repo need to pull" - fi + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + fi - git reset --hard - git checkout -- . - git checkout $branch - git checkout -- . - git clean -dfx - git pull - - [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug - cd $TDENGINE_DIR/debug + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -dfx + git pull + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug echo "rebuild.." - LOCAL_COMMIT=`git rev-parse --short @` + LOCAL_COMMIT=`git rev-parse --short @` - rm -rf * - if [ "$branch" == "3.0" ]; then - echo "3.0 =============" - cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_HTTP=false -DBUILD_TOOLS=true .. > /dev/null - else - cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null - fi - make -j - make install + rm -rf * + if [ "$branch" == "3.0" ]; then + echo "3.0 =============" + cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_CONTRIB=true .. + else + cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null + fi + make -j + make install } function runCasesOneByOne () { - while read -r line; do - if [[ "$line" != "#"* ]]; then - cmd=`echo $line | cut -d',' -f 5` + while read -r line; do + if [[ "$line" != "#"* ]]; then + cmd=`echo $line | cut -d',' -f 5` if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then case=`echo $cmd | cut -d' ' -f 3` start_time=`date +%s` @@ -77,18 +76,7 @@ function runCasesOneByOne () { || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_COVERAGE_REPORT - elif [[ "$2" == "system-test" ]] && [[ $line == *"system-test"* ]]; then - if [[ "$cmd" == *"pytest.sh"* ]]; then - cmd=`echo $cmd | cut -d' ' -f 2-20` - fi - case=`echo $cmd | cut -d' ' -f 4-20` - start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && $cmd > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \ - echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT - end_time=`date +%s` - echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_COVERAGE_REPORT - elif [[ "$2" == "develop-test" ]] && [[ $line == *"develop-test"* ]]; then + elif [$line == *"$2"* ]; then if [[ "$cmd" == *"pytest.sh"* ]]; then cmd=`echo $cmd | cut -d' ' -f 2-20` fi @@ -101,209 +89,176 @@ function runCasesOneByOne () { echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_COVERAGE_REPORT fi fi - done < $1 + done < $1 } function runUnitTest() { - echo "=== Run unit test case ===" - echo " $TDENGINE_DIR/debug" - cd $TDENGINE_DIR/debug - ctest -j12 - - echo " $TDENGINE_DIR/tests/script/api" - cd $TDENGINE_DIR/tests/script/api - make clean && make - - stopTaosd - stopTaosadapter - - nohup taosd -c /etc/taos >> /dev/null 2>&1 & - ./batchprepare 127.0.0.1 - ./dbTableRoute 127.0.0.1 - ./stopquery 127.0.0.1 demo t1 - - echo "3.0 unit test done" + echo "=== Run unit test case ===" + echo " $TDENGINE_DIR/debug" + cd $TDENGINE_DIR/debug + ctest -j12 + echo "3.0 unit test done" } function runSimCases() { - echo "=== Run sim cases ===" - - cd $TDENGINE_DIR/tests/script - runCasesOneByOne ../parallel_test/cases.task sim - - totalSuccess=`grep 'sim success' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalSuccess" -gt "0" ]; then - echo "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + echo "=== Run sim cases ===" - totalFailed=`grep 'sim failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalFailed" -ne "0" ]; then - echo "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + cd $TDENGINE_DIR/tests/script + runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/cases.task sim + + totalSuccess=`grep 'sim success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + echo "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'sim failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi } function runPythonCases() { - echo "=== Run python cases ===" + echo "=== Run python cases ===" - cd $TDENGINE_DIR/tests/parallel_test - sed -i '/compatibility.py/d' cases.task - - cd $TDENGINE_DIR/tests/system-test - runCasesOneByOne ../parallel_test/cases.task system-test + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' cases.task - cd $TDENGINE_DIR/tests/develop-test - runCasesOneByOne ../parallel_test/cases.task develop-test + # army + cd $TDENGINE_DIR/tests/army + runCasesOneByOne ../parallel_test/cases.task army - totalSuccess=`grep 'py success' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalSuccess" -gt "0" ]; then - echo "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + # system-test + cd $TDENGINE_DIR/tests/system-test + runCasesOneByOne ../parallel_test/cases.task system-test - totalFailed=`grep 'py failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalFailed" -ne "0" ]; then - echo "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + # develop-test + cd $TDENGINE_DIR/tests/develop-test + runCasesOneByOne ../parallel_test/cases.task develop-test + + totalSuccess=`grep 'py success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + echo "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'py failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi } function runJDBCCases() { - echo "=== Run JDBC cases ===" + echo "=== Run JDBC cases ===" - cd $JDBC_DIR - git checkout -- . - git reset --hard HEAD - git checkout main - git pull + cd $JDBC_DIR + git checkout -- . + git reset --hard HEAD + git checkout main + git pull - stopTaosd - stopTaosadapter + stopTaosd + stopTaosadapter - nohup taosd -c /etc/taos >> /dev/null 2>&1 & - nohup taosadapter >> /dev/null 2>&1 & + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos >> /dev/null 2>&1 & + nohup taosadapter >> /dev/null 2>&1 & - mvn clean test > result.txt 2>&1 - summary=`grep "Tests run:" result.txt | tail -n 1` - echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT -} - -function runTaosKeeperCases() { - echo "=== Run taoskeeper cases ===" - - cd $TAOSKEEPER_DIR - git checkout -- . - git reset --hard HEAD - git checkout master - git pull - - stopTaosd - stopTaosadapter - - taosd -c /etc/taos >> /dev/null 2>&1 & - taosadapter >> /dev/null 2>&1 & - - go mod tidy && go test -v ./... + mvn clean test > result.txt 2>&1 + summary=`grep "Tests run:" result.txt | tail -n 1` + echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT } function runTest() { - echo "run Test" - - cd $TDENGINE_DIR - [ -d sim ] && rm -rf sim + echo "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT - runUnitTest - runSimCases - runPythonCases - runJDBCCases - runTaosKeeperCases - - stopTaosd - cd $TDENGINE_DIR/tests/script - find . -name '*.sql' | xargs rm -f + runUnitTest + runSimCases + runPythonCases + runJDBCCases - cd $TDENGINE_DIR/tests/pytest - find . -name '*.sql' | xargs rm -f + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f } function lcovFunc { - echo "collect data by lcov" - cd $TDENGINE_DIR + echo "collect data by lcov" + cd $TDENGINE_DIR - # collect data - lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + # collect data + lcov --ignore-errors negative --ignore-errors mismatch -d . --capture --rc lcov_branch_coverage=1 --rc branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info - # remove exclude paths - if [ "$branch" == "main" ] ; then - lcov --remove coverage.info \ - '*/contrib/*' '*/tests/*' '*/test/*' '*/tools/*' '*/libs/sync/*'\ - '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ - '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ - '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\ - '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c' '*/shellAuto.c' '*/shellTire.c'\ - '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' '*/catalog.c'\ - '*/tqSnapshot.c' '*/tsdbSnapshot.c''*/metaSnapshot.c' '*/smaSnapshot.c' '*/tqOffsetSnapshot.c'\ - '*/vnodeSnapshot.c' '*/metaSnapshot.c' '*/tsdbSnapshot.c' '*/mndGrant.c' '*/mndSnode.c' '*/streamRecover.c'\ - '*/osAtomic.c' '*/osDir.c' '*/osFile.c' '*/osMath.c' '*/osSignal.c' '*/osSleep.c' '*/osString.c' '*/osSystem.c'\ - '*/osThread.c' '*/osTime.c' '*/osTimezone.c' \ - --rc lcov_branch_coverage=1 -o coverage.info - else - lcov --remove coverage.info \ - '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c' '*/clientJniConnector.c'\ - --rc lcov_branch_coverage=1 -o coverage.info - fi + # remove exclude paths + lcov --remove coverage.info \ + '*/contrib/*' '*/tests/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/examples/*' '*/tidpool.c' '*/tmempool.c'\ + '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c'\ + '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ + '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ + --rc branch_coverage=1 -o coverage.info + # generate result + echo "generate result" + lcov -l --rc branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT - # generate result - echo "generate result" - lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + sed -i 's/\/root\/TDengine\/sql.c/\/root\/TDengine\/source\/libs\/parser\/inc\/sql.c/g' coverage.info + sed -i 's/\/root\/TDengine\/sql.y/\/root\/TDengine\/source\/libs\/parser\/inc\/sql.y/g' coverage.info - # push result to coveralls.io - echo "push result to coveralls.io" - /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + # push result to coveralls.io + echo "push result to coveralls.io" + /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + + #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT + #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT + cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT - #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT - #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT - cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT - } function sendReport { - echo "send report" - receiver="develop@taosdata.com" - mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" - cd $TDENGINE_DIR + cd $TDENGINE_DIR - sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT - BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` - echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ - (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ - /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" } function stopTaosd { - echo "Stop taosd start" + echo "Stop taosd start" systemctl stop taosd - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do pkill -TERM -x taosd sleep 1 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - done - echo "Stop tasod end" + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasod end" } function stopTaosadapter { - echo "Stop taosadapter" - systemctl stop taosadapter.service - PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do - pkill -TERM -x taosadapter - sleep 1 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - done - echo "Stop tasoadapter end" + echo "Stop taosadapter" + systemctl stop taosadapter.service + PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosadapter + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasoadapter end" } @@ -318,7 +273,7 @@ buildTDengine runTest lcovFunc -sendReport +#sendReport stopTaosd date >> $WORK_DIR/cron.log diff --git a/tests/system-test/2-query/blockSMA.py b/tests/system-test/2-query/blockSMA.py index 85c0189e27..605542cb3c 100644 --- a/tests/system-test/2-query/blockSMA.py +++ b/tests/system-test/2-query/blockSMA.py @@ -16,7 +16,7 @@ class TDTestCase: def run(self): dbname = "db" - tdSql.prepare() + tdSql.prepare(dbname=dbname, drop=True, stt_trigger=1) tdSql.execute(f'''create table {dbname}.ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') From 216830f8b4e35254e707fbf23189e77d4ee05f83 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Jan 2024 20:33:13 +0800 Subject: [PATCH 236/236] fix:[TD-28185]add pause & resume logic for snode --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 70bfa00609..399788e731 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -853,7 +853,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m STaskId* pHTaskId = &pTask->hTaskInfo.id; SStreamTask* pHistoryTask = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); if (pHistoryTask) { - code = tqProcessTaskResumeImpl(pMeta, pHistoryTask, sversion, pReq->igUntreated, fromVnode); + code = tqProcessTaskResumeImpl(handle, pHistoryTask, sversion, pReq->igUntreated, fromVnode); } return code;