From 6bbfc1aead963f031c6792401f1c69aaa885dcc4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Dec 2024 19:19:57 +0800 Subject: [PATCH 01/36] fix(stream): do rsp for checkpoint reqs. --- source/dnode/mnode/impl/src/mndStream.c | 3 +-- source/dnode/vnode/src/tq/tq.c | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 6336cd6e49..afaf33bafa 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -21,7 +21,6 @@ #include "mndShow.h" #include "mndStb.h" #include "mndTrans.h" -#include "mndVgroup.h" #include "osMemory.h" #include "parser.h" #include "taoserror.h" @@ -2336,7 +2335,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { } int32_t total = taosArrayGetSize(*pReqTaskList); - if (total == numOfTasks) { // all tasks has send the reqs + if (total == numOfTasks) { // all tasks have sent the reqs int64_t checkpointId = mndStreamGenChkptId(pMnode, false); mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a234777441..43446683c5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1250,6 +1250,14 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); + SRpcMsg rsp = {0}; // make the mnode retry until this task status completed + int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SYN_PROPOSE_NOT_READY); + if (ret) { // suppress the error in build checkpoint-source rsp + tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); + } + + tmsgSendRsp(&rsp); // error occurs + return TSDB_CODE_SUCCESS; } else { // checkpoint already finished, and not in checkpoint status if (req.checkpointId <= pTask->chkInfo.checkpointId) { From f2e9e914879a9347ebd2512920f199b4065d3107 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 11 Dec 2024 16:22:29 +0800 Subject: [PATCH 02/36] Add sdbCancelFetch and sdbRelease while exit iter. --- source/dnode/mnode/impl/src/mndMnode.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6b1c97b399..413fd3aec5 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -917,7 +917,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } @@ -927,7 +929,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, b1, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } @@ -948,7 +952,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } const char *status = "ready"; @@ -960,14 +966,18 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)b3, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } @@ -975,7 +985,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&roleTimeMs, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } From cfcb85ef94ad30b2fdd5d548ab3bbe21d7902e1e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Dec 2024 17:25:01 +0800 Subject: [PATCH 03/36] refactor(stream): update mnode epset in hb. --- include/common/streamMsg.h | 1 + source/common/src/msg/streamMsg.c | 2 + source/dnode/mnode/impl/src/mndStreamHb.c | 22 ++++-- source/libs/stream/src/streamHb.c | 94 ++++++++++++++++++----- 4 files changed, 92 insertions(+), 27 deletions(-) diff --git a/include/common/streamMsg.h b/include/common/streamMsg.h index 3db92ba58d..d410bd17e0 100644 --- a/include/common/streamMsg.h +++ b/include/common/streamMsg.h @@ -188,6 +188,7 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg); typedef struct { SMsgHead head; int32_t msgId; + SEpSet mndEpset; } SMStreamHbRspMsg; int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp); diff --git a/source/common/src/msg/streamMsg.c b/source/common/src/msg/streamMsg.c index c92ab52ac1..b5f6900321 100644 --- a/source/common/src/msg/streamMsg.c +++ b/source/common/src/msg/streamMsg.c @@ -651,6 +651,7 @@ int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp) { TAOS_CHECK_EXIT(tStartEncode(pEncoder)); TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->msgId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pRsp->mndEpset)); tEndEncode(pEncoder); _exit: @@ -663,6 +664,7 @@ int32_t tDecodeStreamHbRsp(SDecoder* pDecoder, SMStreamHbRspMsg* pRsp) { TAOS_CHECK_EXIT(tStartDecode(pDecoder)); TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->msgId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pRsp->mndEpset)); tEndDecode(pDecoder); _exit: diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4b3db28aa1..b7b2764442 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -15,6 +15,8 @@ #include "mndStream.h" #include "mndTrans.h" +#include "mndMnode.h" +#include "tmisce.h" typedef struct SFailedCheckpointInfo { int64_t streamUid; @@ -31,7 +33,7 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); static int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info); static bool validateHbMsg(const SArray *pNodeList, int32_t vgId); static void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArray *pOrphanTasks); -static void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId); +static void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, SEpSet* pEpset, int32_t vgId, int32_t msgId); static void checkforOrphanTask(SMnode* pMnode, STaskStatusEntry* p, SArray* pOrphanTasks); void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { @@ -329,6 +331,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pFailedChkpt = NULL; SArray *pOrphanTasks = NULL; int32_t code = 0; + SDecoder decoder = {0}; + SEpSet mnodeEpset = {0}; if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) { @@ -336,7 +340,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { } } - SDecoder decoder = {0}; tDecoderInit(&decoder, pReq->pCont, pReq->contLen); if (tDecodeStreamHbMsg(&decoder, &req) < 0) { @@ -357,13 +360,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { TAOS_RETURN(terrno); } + mndGetMnodeEpSet(pMnode, &mnodeEpset); + streamMutexLock(&execInfo.lock); mndInitStreamExecInfo(pMnode, &execInfo); if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { mError("vgId:%d not exists in nodeList buf, discarded", req.vgId); - doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(terrno, &pReq->info, &mnodeEpset, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); @@ -383,9 +388,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if ((pEntry->lastHbMsgId == req.msgId) && (pEntry->lastHbMsgTs == req.ts)) { mError("vgId:%d HbMsgId:%d already handled, bh msg discard, and send HbRsp", pEntry->nodeId, req.msgId); - // return directly and after the vnode to continue to send the next HbMsg. + // return directly and allow the vnode to continue to send the next HbMsg. terrno = TSDB_CODE_SUCCESS; - doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(terrno, &pReq->info, &mnodeEpset, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); @@ -529,7 +534,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { streamMutexUnlock(&execInfo.lock); - doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, &mnodeEpset, req.vgId, req.msgId); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); return code; @@ -552,12 +557,13 @@ void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArr taosArrayDestroy(pOrphanTasks); } -void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId) { +void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, SEpSet* pMndEpset, int32_t vgId, int32_t msgId) { int32_t ret = 0; int32_t tlen = 0; void *buf = NULL; - const SMStreamHbRspMsg msg = {.msgId = msgId}; + SMStreamHbRspMsg msg = {.msgId = msgId};//, .mndEpset = *pMndEpset}; + epsetAssign(&msg.mndEpset, pMndEpset); tEncodeSize(tEncodeStreamHbRsp, &msg, tlen, ret); if (ret < 0) { diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 25cb28f77c..aadf10db6a 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -108,6 +108,67 @@ static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* p return tmsgSendReq(pEpset, &msg); } +static int32_t streamTaskGetMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); + if (code != 0) { + continue; + } + + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + epsetAssign(pEpSet, &pTask->info.mnodeEpset); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_FAILED; +} + +static int32_t streamTaskUpdateMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); + if (code != 0) { + stError("vgId:%d s-task:0x%x failed to acquire it for updating mnode epset, code:%s", pMeta->vgId, pId->taskId, + tstrerror(code)); + continue; + } + + // ignore this error since it is only for log file + char buf[256] = {0}; + int32_t ret = epsetToStr(&pTask->info.mnodeEpset, buf, tListLen(buf)); + if (ret != 0) { // print error and continue + stError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + + char newBuf[256] = {0}; + ret = epsetToStr(pEpSet, newBuf, tListLen(newBuf)); + if (ret != 0) { + stError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + + epsetAssign(&pTask->info.mnodeEpset, pEpSet); + stInfo("s-task:0x%x update mnd epset, from %s to %s", pId->taskId, buf, newBuf); + streamMetaReleaseTask(pMeta, pTask); + } + + return TSDB_CODE_SUCCESS; +} + // NOTE: this task should be executed within the SStreamMeta lock region. int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { SEpSet epset = {0}; @@ -121,24 +182,11 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { stDebug("vgId:%d hbMsg rsp not recv, send current hbMsg, msgId:%d, total:%d again", pMeta->vgId, pInfo->hbMsg.msgId, pInfo->hbCount); - for(int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); - STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; - SStreamTask* pTask = NULL; - - code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); - if (code != 0) { - continue; - } - - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - epsetAssign(&epset, &pTask->info.mnodeEpset); - streamMetaReleaseTask(pMeta, pTask); - break; + code = streamTaskGetMndEpset(pMeta, &epset); + if (code != 0) { + stError("vgId:%d failed to get the mnode epset, not retrying sending hbMsg, msgId:%d", pMeta->vgId, + pInfo->hbMsg.msgId); + return code; } pInfo->msgSendTs = taosGetTimestampMs(); @@ -372,9 +420,10 @@ void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSe } int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { - stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId); SMetaHbInfo* pInfo = pMeta->pHbInfo; + SEpSet epset = {0}; + stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId); streamMetaWLock(pMeta); // current waiting rsp recved @@ -384,6 +433,13 @@ int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { pInfo->hbCount += 1; pInfo->msgSendTs = -1; + + streamTaskGetMndEpset(pMeta, &epset); + if (!isEpsetEqual(&pRsp->mndEpset, &epset)) { + // we need to update the mnode epset for each tasks + stInfo("vgId:%d mnode epset updated, update mnode epset for all tasks", pMeta->vgId); + streamTaskUpdateMndEpset(pMeta, &pRsp->mndEpset); + } } else { stWarn("vgId:%d recv expired hb rsp, msgId:%d, discarded", pMeta->vgId, pRsp->msgId); } From a9d288f2d20e0b82756d7bcb8783b09b858ce5b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Dec 2024 09:23:34 +0800 Subject: [PATCH 04/36] fix(stream): update the function signature. --- source/libs/stream/src/streamHb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 16466bee7d..35d520cae8 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -152,7 +152,7 @@ static int32_t streamTaskGetMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { return TSDB_CODE_FAILED; } -static int32_t streamTaskUpdateMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { +static void streamTaskUpdateMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); for (int32_t i = 0; i < numOfTasks; ++i) { @@ -185,7 +185,7 @@ static int32_t streamTaskUpdateMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { streamMetaReleaseTask(pMeta, pTask); } - return TSDB_CODE_SUCCESS; + stDebug("vgId:%d update mnd epset for %d tasks completed", pMeta->vgId, numOfTasks); } // NOTE: this task should be executed within the SStreamMeta lock region. From 2d86c5df91449e0058e69840937bf08b324003cf Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Dec 2024 11:44:31 +0800 Subject: [PATCH 05/36] docs: add faq of why database disappear and clusterId change --- docs/zh/27-train-faq/01-faq.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index b7d20cac5a..487fb29acb 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -289,3 +289,12 @@ https://docs.taosdata.com/reference/components/taosd/#%E7%9B%91%E6%8E%A7%E7%9B%B 因此用户首先要检查服务端，集群的所有端口（原生连接默认6030，http连接默认6041）有无打开；其次是客户端的hosts文件中是否配置了集群所有节点的fqdn与IP信息。 如仍无法解决，则需要联系涛思技术人员支持。 + +### 32 同一台服务器，数据库的数据目录 dataDir 不变，为什么原有数据库丢失且集群 ID 发生了变化？ +背景知识：TDengine 服务端进程（taosd）在启动时，若数据目录（dataDir，该目录在配置文件 taos.cfg 中指定）下不存在有效的数据文件子目录（如 mnode、dnode 和 vnode 等），则会自动创建这些目录。在创建新的 mnode 目录的同时，会分配一个新的集群 ID，从而创建一个新的集群。 + +原因分析：taosd 的数据目录 dataDir 可以指向多个不同的挂载点。如果这些挂载点未在 fstab 文件中配置自动挂载，服务器重启后，dataDir 将仅作为一个本地磁盘的普通目录存在，而未能按预期指向挂载的磁盘。此时，若 taosd 服务启动，它将在 dataDir 下新建目录，从而产生一个新的集群。 + +问题影响：服务器重启后，原有数据库丢失且集群 ID 发生变化，导致无法访问原有数据库。对于企业版用户，如果已针对集群 ID 进行授权，则会发现集群服务器的机器码未变，但原有的授权已失效。如果未进行监控或者未及时发现并进行处理，则不会注意到数据库已经丢失，从而造成损失，增加运维成本。 + +问题解决：应在 fstab 文件中配置 dataDir 目录的自动挂载，确保 dataDir 始终指向预期的挂载点和目录，此时，再重启服务器，会找回原有的数据库和集群。在后续的版本中，我们将开发一个功能，使 taosd 在检测到启动前后 dataDir 发生变化时，在启动阶段退出，同时提供相应的错误提示。 \ No newline at end of file From 9b178855b80ceb2932d02fd004633dec9ba33e07 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Dec 2024 11:49:38 +0800 Subject: [PATCH 06/36] docs: add faq of why database disappear and clusterId change --- docs/zh/27-train-faq/01-faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 487fb29acb..58e3c452c8 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -295,6 +295,6 @@ https://docs.taosdata.com/reference/components/taosd/#%E7%9B%91%E6%8E%A7%E7%9B%B 原因分析：taosd 的数据目录 dataDir 可以指向多个不同的挂载点。如果这些挂载点未在 fstab 文件中配置自动挂载，服务器重启后，dataDir 将仅作为一个本地磁盘的普通目录存在，而未能按预期指向挂载的磁盘。此时，若 taosd 服务启动，它将在 dataDir 下新建目录，从而产生一个新的集群。 -问题影响：服务器重启后，原有数据库丢失且集群 ID 发生变化，导致无法访问原有数据库。对于企业版用户，如果已针对集群 ID 进行授权，则会发现集群服务器的机器码未变，但原有的授权已失效。如果未进行监控或者未及时发现并进行处理，则不会注意到数据库已经丢失，从而造成损失，增加运维成本。 +问题影响：服务器重启后，原有数据库丢失（注：并非真正丢失，只是原有的数据磁盘未挂载，暂时看不到）且集群 ID 发生变化，导致无法访问原有数据库。对于企业版用户，如果已针对集群 ID 进行授权，还会发现集群服务器的机器码未变，但原有的授权已失效。如果未针对该问题进行监控或者未及时发现并进行处理，则不会注意到原有数据库已经丢失，从而造成损失，增加运维成本。 问题解决：应在 fstab 文件中配置 dataDir 目录的自动挂载，确保 dataDir 始终指向预期的挂载点和目录，此时，再重启服务器，会找回原有的数据库和集群。在后续的版本中，我们将开发一个功能，使 taosd 在检测到启动前后 dataDir 发生变化时，在启动阶段退出，同时提供相应的错误提示。 \ No newline at end of file From f41e96baca0fb7ae8ab300d2b79b92424832a296 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Dec 2024 11:51:01 +0800 Subject: [PATCH 07/36] docs: add faq of why database disappear and clusterId change --- docs/zh/27-train-faq/01-faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 58e3c452c8..0a519770f8 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -291,7 +291,7 @@ https://docs.taosdata.com/reference/components/taosd/#%E7%9B%91%E6%8E%A7%E7%9B%B 如仍无法解决，则需要联系涛思技术人员支持。 ### 32 同一台服务器，数据库的数据目录 dataDir 不变，为什么原有数据库丢失且集群 ID 发生了变化？ -背景知识：TDengine 服务端进程（taosd）在启动时，若数据目录（dataDir，该目录在配置文件 taos.cfg 中指定）下不存在有效的数据文件子目录（如 mnode、dnode 和 vnode 等），则会自动创建这些目录。在创建新的 mnode 目录的同时，会分配一个新的集群 ID，从而创建一个新的集群。 +背景知识：TDengine 服务端进程（taosd）在启动时，若数据目录（dataDir，该目录在配置文件 taos.cfg 中指定）下不存在有效的数据文件子目录（如 mnode、dnode 和 vnode 等），则会自动创建这些目录。在创建新的 mnode 目录的同时，会分配一个新的集群 ID，从而产生一个新的集群。 原因分析：taosd 的数据目录 dataDir 可以指向多个不同的挂载点。如果这些挂载点未在 fstab 文件中配置自动挂载，服务器重启后，dataDir 将仅作为一个本地磁盘的普通目录存在，而未能按预期指向挂载的磁盘。此时，若 taosd 服务启动，它将在 dataDir 下新建目录，从而产生一个新的集群。 From b28930f3352ef7da5cf46b422b57837c98836e7e Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 12 Dec 2024 11:52:47 +0800 Subject: [PATCH 08/36] docs: add faq of why database disappear and clusterId change --- docs/zh/27-train-faq/01-faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 0a519770f8..ece7c9f309 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -295,6 +295,6 @@ https://docs.taosdata.com/reference/components/taosd/#%E7%9B%91%E6%8E%A7%E7%9B%B 原因分析：taosd 的数据目录 dataDir 可以指向多个不同的挂载点。如果这些挂载点未在 fstab 文件中配置自动挂载，服务器重启后，dataDir 将仅作为一个本地磁盘的普通目录存在，而未能按预期指向挂载的磁盘。此时，若 taosd 服务启动，它将在 dataDir 下新建目录，从而产生一个新的集群。 -问题影响：服务器重启后，原有数据库丢失（注：并非真正丢失，只是原有的数据磁盘未挂载，暂时看不到）且集群 ID 发生变化，导致无法访问原有数据库。对于企业版用户，如果已针对集群 ID 进行授权，还会发现集群服务器的机器码未变，但原有的授权已失效。如果未针对该问题进行监控或者未及时发现并进行处理，则不会注意到原有数据库已经丢失，从而造成损失，增加运维成本。 +问题影响：服务器重启后，原有数据库丢失（注：并非真正丢失，只是原有的数据磁盘未挂载，暂时看不到）且集群 ID 发生变化，导致无法访问原有数据库。对于企业版用户，如果已针对集群 ID 进行授权，还会发现集群服务器的机器码未变，但原有的授权已失效。如果未针对该问题进行监控或者未及时发现并进行处理，则用户不会注意到原有数据库已经丢失，从而造成损失，增加运维成本。 问题解决：应在 fstab 文件中配置 dataDir 目录的自动挂载，确保 dataDir 始终指向预期的挂载点和目录，此时，再重启服务器，会找回原有的数据库和集群。在后续的版本中，我们将开发一个功能，使 taosd 在检测到启动前后 dataDir 发生变化时，在启动阶段退出，同时提供相应的错误提示。 \ No newline at end of file From b8fde1baf4d35bc47b13af3ee589ee1448320802 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Dec 2024 13:53:53 +0800 Subject: [PATCH 09/36] fix(stream): check return value. --- source/libs/stream/src/streamHb.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 35d520cae8..53b6a38b35 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -434,6 +434,7 @@ void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSe int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { SMetaHbInfo* pInfo = pMeta->pHbInfo; SEpSet epset = {0}; + int32_t code = 0; stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId); streamMetaWLock(pMeta); @@ -446,8 +447,8 @@ int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { pInfo->hbCount += 1; pInfo->msgSendTs = -1; - streamTaskGetMndEpset(pMeta, &epset); - if (!isEpsetEqual(&pRsp->mndEpset, &epset)) { + code = streamTaskGetMndEpset(pMeta, &epset); + if (!isEpsetEqual(&pRsp->mndEpset, &epset) && (code == 0)) { // we need to update the mnode epset for each tasks stInfo("vgId:%d mnode epset updated, update mnode epset for all tasks", pMeta->vgId); streamTaskUpdateMndEpset(pMeta, &pRsp->mndEpset); From 217c4c55067c761f07028b47223f96e9621be2a7 Mon Sep 17 00:00:00 2001 From: Yibo Liu Date: Thu, 12 Dec 2024 15:29:39 +0800 Subject: [PATCH 10/36] Update 12-tdinsight.md --- .../01-components/12-tdinsight.md | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/en/14-reference/01-components/12-tdinsight.md b/docs/en/14-reference/01-components/12-tdinsight.md index 12423c512d..c9464760e7 100644 --- a/docs/en/14-reference/01-components/12-tdinsight.md +++ b/docs/en/14-reference/01-components/12-tdinsight.md @@ -171,7 +171,37 @@ Metric details: 5. **Writes**: Total number of writes 6. **Other**: Total number of other requests -There are also line charts for the above categories. +There are also line charts for the above categories. + +### Automatic import of preconfigured alert rules + +After summarizing user experience, 14 commonly used alert rules are sorted out. These alert rules can monitor key indicators of the TDengine cluster and report alerts, such as abnormal and exceeded indicators. +Starting from TDengine-Server 3.3.4.3 (TDengine-datasource 3.6.3), TDengine Datasource supports automatic import of preconfigured alert rules. You can import 14 alert rules to Grafana (version 11 or later) with one click. +In the TDengine-datasource setting interface, turn on the "Load Tengine Alert" switch, click the "Save & test" button, the plugin will automatically load the mentioned 14 alert rules. The rules will be placed in the Grafana alerts directory. If not required, turn off the "Load TDengine Alert" switch, and click the button next to "Clear TDengine Alert" to clear all the alert rules imported into this data source. + +After importing, click on "Alert rules" on the left side of the Grafana interface to view all current alert rules. By configuring contact points, users can receive alert notifications. + +The specific configuration of the 14 alert rules is as follows: + +| alert rule| Rule threshold| Behavior when no data | Data scanning interval |Duration | SQL | +| ------ | --------- | ---------------- | ----------- |------- |----------------------| +|CPU load of dnode node|average > 80%|Trigger alert|5 minutes|5 minutes |`select now(), dnode_id, last(cpu_system) as cup_use from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts < now partition by dnode_id having first(_ts) > 0 `| +|Memory of dnode node |average > 60%|Trigger alert|5 minutes|5 minutes|`select now(), dnode_id, last(mem_engine) / last(mem_total) * 100 as taosd from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts 80%|Trigger alert|5 minutes|5 minutes|`select now(), dnode_id, data_dir_level, data_dir_name, last(used) / last(total) * 100 as used from log.taosd_dnodes_data_dirs where _ts >= (now - 5m) and _ts < now partition by dnode_id, data_dir_level, data_dir_name`| +|Authorization expires |< 60天|Trigger alert|1 day|0 0 seconds|`select now(), cluster_id, last(grants_expire_time) / 86400 as expire_time from log.taosd_cluster_info where _ts >= (now - 24h) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|The used measurement points has reached the authorized number|>= 90%|Trigger alert|1 day|0 seconds|`select now(), cluster_id, CASE WHEN max(grants_timeseries_total) > 0.0 THEN max(grants_timeseries_used) /max(grants_timeseries_total) * 100.0 ELSE 0.0 END AS result from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1m) > 0`| +|Number of concurrent query requests | > 100|Do not trigger alert|1 minute|0 seconds|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries`| +|Maximum time for slow query execution (no time window) |> 300秒|Do not trigger alert|1 minute|0 seconds|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries where exec_usec>300000000`| +|dnode offline |total != alive|Trigger alert|30 seconds|0 seconds|`select now(), cluster_id, last(dnodes_total) - last(dnodes_alive) as dnode_offline from log.taosd_cluster_info where _ts >= (now -30s) and _ts < now partition by cluster_id having first(_ts) > 0`| +|vnode offline |total != alive|Trigger alert|30 seconds|0 seconds|`select now(), cluster_id, last(vnodes_total) - last(vnodes_alive) as vnode_offline from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|Number of data deletion requests |> 0|Do not trigger alert|30 seconds|0 seconds|``select now(), count(`count`) as `delete_count` from log.taos_sql_req where sql_type = 'delete' and _ts >= (now -30s) and _ts < now``| +|Adapter RESTful request fail |> 5|Do not trigger alert|30 seconds|0 seconds|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=0 and ts >= (now -30s) and ts < now``| +|Adapter WebSocket request fail |> 5|Do not trigger alert|30 seconds|0 seconds|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=1 and ts >= (now -30s) and ts < now``| +|Dnode data reporting is missing |< 3|Trigger alert|180 seconds|0 seconds|`select now(), cluster_id, count(*) as dnode_report from log.taosd_cluster_info where _ts >= (now -180s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1h) > 0`| +|Restart dnode |max(update_time) > last(update_time)|Trigger alert|90 seconds|0 seconds|`select now(), dnode_id, max(uptime) - last(uptime) as dnode_restart from log.taosd_dnodes_info where _ts >= (now - 90s) and _ts < now partition by dnode_id`| + +TDengine users can modify and improve these alert rules according to their own business needs. In Grafana 7.5 and below versions, the Dashboard and Alert rules functions are combined, while in subsequent new versions, the two functions are separated. To be compatible with Grafana7.5 and below versions, an Alert Used Only panel has been added to the TDinsight panel, which is only required for Grafana7.5 and below versions. + ## Upgrade From 69a40355ed7977222702c94d4151db1b1ff6a0fc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Dec 2024 15:35:32 +0800 Subject: [PATCH 11/36] fix compile error --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 11d2385d1c..30a5bb76db 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -4631,7 +4631,7 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { if (fname == NULL) { return terrno; } - tstrncpy(fname, name, strlen(name)); + tstrncpy(fname, name, strlen(name) + 1); if (taosArrayPush(diff, &fname) == NULL) { taosMemoryFree(fname); return terrno; From c7f22d23df08f81de9f41edacb273bb128602aad Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Dec 2024 15:36:44 +0800 Subject: [PATCH 12/36] fix compile error --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 30a5bb76db..55b8365dbb 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -4825,7 +4825,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { return terrno; } - tstrncpy(fname, name, strlen(name)); + tstrncpy(fname, name, strlen(name) + 1); if (taosArrayPush(p->pAdd, &fname) == NULL) { taosMemoryFree(fname); TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); From 137430a3ed3a16fe0dc6119545eb057859f793ee Mon Sep 17 00:00:00 2001 From: Yibo Liu Date: Thu, 12 Dec 2024 16:13:02 +0800 Subject: [PATCH 13/36] Update index.md --- docs/en/27-train-faq/index.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/en/27-train-faq/index.md b/docs/en/27-train-faq/index.md index a5ec977e29..ca6cd91714 100644 --- a/docs/en/27-train-faq/index.md +++ b/docs/en/27-train-faq/index.md @@ -286,4 +286,14 @@ This connection only reports the most basic information that does not involve an This feature is an optional configuration item, which is enabled by default in the open-source version. The specific parameter is telemetryReporting, as explained in the [official documentation](../tdengine-reference/components/taosd/). You can disable this parameter at any time by modifying telemetryReporting to 0 in taos.cfg, then restarting the database service. Code located at: [https://github.com/taosdata/TDengine/blob/62e609c558deb764a37d1a01ba84bc35115a85a4/source/dnode/mnode/impl/src/mndTelem.c](https://github.com/taosdata/TDengine/blob/62e609c558deb764a37d1a01ba84bc35115a85a4/source/dnode/mnode/impl/src/mndTelem.c). -Additionally, for the highly secure enterprise version, TDengine Enterprise, this parameter will not be operational. +Additionally, for the highly secure enterprise version, TDengine Enterprise, this parameter will not be operational. + +### 31 What should I do if I encounter 'Sync leader is unreachable' when connecting to the cluster for the first time? + +Reporting this error indicates that the first connection to the cluster was successful, but the IP address accessed for the first time was not the leader of mnode. An error occurred when the client attempted to establish a connection with the leader. The client searches for the leader node through EP, which specifies the fqdn and port number. There are two common reasons for this error: + +- The ports of other dnodes in the cluster are not open +- The client's hosts file is not configured correctly + +Therefore, first, check whether all ports on the server and cluster (default 6030 for native connections and 6041 for HTTP connections) are open; Next, check if the client's hosts file has configured the fqdn and IP information for all dnodes in the cluster. +If the issue still cannot be resolved, it is necessary to contact Taos technical personnel for support. From 8605f1c32a1618dcd1dc8127b3803c0d7229206f Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Wed, 11 Dec 2024 19:59:12 +0800 Subject: [PATCH 14/36] fix(query)[TD-33181]. reset error code when retry in vnodeGetBufPoolToUse --- source/dnode/vnode/src/vnd/vnodeCommit.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3ebcf50858..28d27b8893 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -103,9 +103,11 @@ static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) { } code = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); - if (code && code != TSDB_CODE_TIMEOUT_ERROR) { - TSDB_CHECK_CODE(code, lino, _exit); + // ignore timeout error and retry + if (code == TSDB_CODE_TIMEOUT_ERROR) { + code = TSDB_CODE_SUCCESS; } + TSDB_CHECK_CODE(code, lino, _exit); } } } From 63705d9b832ff893d3dd1a7ef15f34bea4b45b7f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Dec 2024 17:25:46 +0800 Subject: [PATCH 15/36] fix compile error --- source/libs/stream/src/streamBackendRocksdb.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 55b8365dbb..f7cac3b562 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -4627,11 +4627,12 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { while (pIter) { char* name = taosHashGetKey(pIter, &len); if (!isBkdDataMeta(name, len) && !taosHashGet(p1, name, len)) { - char* fname = taosMemoryCalloc(1, len + 1); + int32_t cap = len + 1; + char* fname = taosMemoryCalloc(1, cap); if (fname == NULL) { return terrno; } - tstrncpy(fname, name, strlen(name) + 1); + tstrncpy(fname, name, cap); if (taosArrayPush(diff, &fname) == NULL) { taosMemoryFree(fname); return terrno; @@ -4819,13 +4820,14 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { size_t len = 0; char* name = taosHashGetKey(pIter, &len); if (name != NULL && !isBkdDataMeta(name, len)) { - char* fname = taosMemoryCalloc(1, len + 1); + int32_t cap = len + 1; + char* fname = taosMemoryCalloc(1, cap); if (fname == NULL) { TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return terrno; } - tstrncpy(fname, name, strlen(name) + 1); + tstrncpy(fname, name, cap); if (taosArrayPush(p->pAdd, &fname) == NULL) { taosMemoryFree(fname); TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); From e9f398b3f8099e75276635bb26b5adf6fa6a5e6e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 00:15:10 +0800 Subject: [PATCH 16/36] fix(test): check null ptr. --- source/dnode/mnode/impl/src/mndMnode.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6b1c97b399..e49a974748 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -248,6 +248,10 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { } void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { + if (pMnode == NULL || pEpSet == NULL) { + return; + } + SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); if (totalMnodes == 0) { From 8136363fbe33cb5a78bbd9d4511c2755b4200344 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 13 Dec 2024 11:14:21 +0800 Subject: [PATCH 17/36] refact: merge unnecessary code. --- source/dnode/mnode/impl/src/mndMnode.c | 36 +++++++++----------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 413fd3aec5..5ea0c342d5 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -951,12 +951,8 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB STR_WITH_MAXSIZE_TO_VARSTR(b2, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); - sdbCancelFetch(pSdb, pShow->pIter); - sdbRelease(pSdb, pObj); - goto _out; - } + if (code != 0) goto _err; + const char *status = "ready"; if (objStatus == SDB_STATUS_CREATING) status = "creating"; if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; @@ -965,31 +961,16 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)b3, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); - sdbCancelFetch(pSdb, pShow->pIter); - sdbRelease(pSdb, pObj); - goto _out; - } + if (code != 0) goto _err; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); - sdbCancelFetch(pSdb, pShow->pIter); - sdbRelease(pSdb, pObj); - goto _out; - } + if (code != 0) goto _err; int64_t roleTimeMs = (isDnodeOnline) ? pObj->roleTimeMs : 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&roleTimeMs, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); - sdbCancelFetch(pSdb, pShow->pIter); - sdbRelease(pSdb, pObj); - goto _out; - } + if (code != 0) goto _err; numOfRows++; sdbRelease(pSdb, pObj); @@ -1000,6 +981,13 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB _out: sdbRelease(pSdb, pSelfObj); return numOfRows; + +_err: + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); + sdbRelease(pSdb, pSelfObj); + return numOfRows; } static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { From 0170cdb5cbfa829f885987942474007b3db1ceb9 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Fri, 13 Dec 2024 11:51:54 +0800 Subject: [PATCH 18/36] docs/add the units of configuration cacheload and cachesize in Update 02-database.md --- docs/zh/14-reference/03-taos-sql/02-database.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index fabebc44da..867e1d7aa1 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -134,11 +134,11 @@ alter_database_option: { 1. 如何查看 cachesize? -通过 select * from information_schema.ins_databases; 可以查看这些 cachesize 的具体值。 +通过 select * from information_schema.ins_databases; 可以查看这些 cachesize 的具体值（单位为 MB）。。 2. 如何查看 cacheload? -通过 show \.vgroups; 可以查看 cacheload +通过 show \.vgroups; 可以查看 cacheload（单位为字节）。 3. 判断 cachesize 是否够用 From 7658144359f114ac7a8691ff19008cf64404ad6d Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Fri, 13 Dec 2024 09:27:42 +0800 Subject: [PATCH 19/36] fix:[TS-5763] Fix error when using selection function with JSON param. --- source/libs/function/src/builtinsimpl.c | 63 +++++++++-- .../test_selection_function_with_json.py | 106 ++++++++++++++++++ tests/parallel_test/cases.task | 1 + 3 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 tests/army/query/function/test_selection_function_with_json.py diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 83227dea9e..aa0711f421 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2449,13 +2449,21 @@ static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (IS_VAR_DATA_TYPE(type)) { - pInfo->bytes = varDataTLen(pData); + if (type == TSDB_DATA_TYPE_JSON) { + pInfo->bytes = getJsonValueLen(pData); + } else { + pInfo->bytes = varDataTLen(pData); + } } (void)memcpy(pInfo->buf, pData, pInfo->bytes); if (pkData != NULL) { if (IS_VAR_DATA_TYPE(pInfo->pkType)) { - pInfo->pkBytes = varDataTLen(pkData); + if (pInfo->pkType == TSDB_DATA_TYPE_JSON) { + pInfo->pkBytes = getJsonValueLen(pkData); + } else { + pInfo->pkBytes = varDataTLen(pkData); + } } (void)memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); pInfo->pkData = pInfo->buf + pInfo->bytes; @@ -2985,7 +2993,11 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex pInfo->isNull = false; if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { - pInfo->bytes = varDataTLen(pData); + if (pInputCol->info.type == TSDB_DATA_TYPE_JSON) { + pInfo->bytes = getJsonValueLen(pData); + } else { + pInfo->bytes = varDataTLen(pData); + } } (void)memcpy(pInfo->buf, pData, pInfo->bytes); @@ -2994,7 +3006,11 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex if (pCtx->hasPrimaryKey && !colDataIsNull_s(pkCol, rowIndex)) { char* pkData = colDataGetData(pkCol, rowIndex); if (IS_VAR_DATA_TYPE(pInfo->pkType)) { - pInfo->pkBytes = varDataTLen(pkData); + if (pInfo->pkType == TSDB_DATA_TYPE_JSON) { + pInfo->pkBytes = getJsonValueLen(pkData); + } else { + pInfo->pkBytes = varDataTLen(pkData); + } } (void)memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); pInfo->pkData = pInfo->buf + pInfo->bytes; @@ -5872,7 +5888,11 @@ void modeFunctionCleanupExt(SqlFunctionCtx* pCtx) { static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pInfo, STuplePos* pPos) { if (IS_VAR_DATA_TYPE(pInfo->colType)) { - (void)memcpy(pInfo->buf, data, varDataTLen(data)); + if (pInfo->colType == TSDB_DATA_TYPE_JSON) { + (void)memcpy(pInfo->buf, data, getJsonValueLen(data)); + } else { + (void)memcpy(pInfo->buf, data, varDataTLen(data)); + } } else { (void)memcpy(pInfo->buf, data, pInfo->colBytes); } @@ -5882,7 +5902,16 @@ static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pI static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { int32_t code = TSDB_CODE_SUCCESS; - int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; + int32_t hashKeyBytes; + if (IS_VAR_DATA_TYPE(pInfo->colType)) { + if (pInfo->colType == TSDB_DATA_TYPE_JSON) { + hashKeyBytes = getJsonValueLen(data); + } else { + hashKeyBytes = varDataTLen(data); + } + } else { + hashKeyBytes = pInfo->colBytes; + } SModeItem* pHashItem = (SModeItem *)taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { @@ -6654,14 +6683,32 @@ static void doSaveRateInfo(SRateInfo* pRateInfo, bool isFirst, int64_t ts, char* pRateInfo->firstValue = v; pRateInfo->firstKey = ts; if (pRateInfo->firstPk) { - int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; + int32_t pkBytes; + if (IS_VAR_DATA_TYPE(pRateInfo->pkType)) { + if (pRateInfo->pkType == TSDB_DATA_TYPE_JSON) { + pkBytes = getJsonValueLen(pk); + } else { + pkBytes = varDataTLen(pk); + } + } else { + pkBytes = pRateInfo->pkBytes; + } (void)memcpy(pRateInfo->firstPk, pk, pkBytes); } } else { pRateInfo->lastValue = v; pRateInfo->lastKey = ts; if (pRateInfo->lastPk) { - int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; + int32_t pkBytes; + if (IS_VAR_DATA_TYPE(pRateInfo->pkType)) { + if (pRateInfo->pkType == TSDB_DATA_TYPE_JSON) { + pkBytes = getJsonValueLen(pk); + } else { + pkBytes = varDataTLen(pk); + } + } else { + pkBytes = pRateInfo->pkBytes; + } (void)memcpy(pRateInfo->lastPk, pk, pkBytes); } } diff --git a/tests/army/query/function/test_selection_function_with_json.py b/tests/army/query/function/test_selection_function_with_json.py new file mode 100644 index 0000000000..e1f8090ae3 --- /dev/null +++ b/tests/army/query/function/test_selection_function_with_json.py @@ -0,0 +1,106 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from frame import etool +from frame.etool import * +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.common import * + +class TDTestCase(TBase): + updatecfgDict = { + "keepColumnName": "1", + "ttlChangeOnWrite": "1", + "querySmaOptimize": "1", + "slowLogScope": "none", + "queryBufferSize": 10240 + } + + def insert_data(self): + tdLog.info(f"insert data.") + tdSql.execute("drop database if exists ts_5763;") + tdSql.execute("create database ts_5763;") + tdSql.execute("use ts_5763;") + tdSql.execute("select database();") + tdSql.execute("CREATE STABLE metrics (ts TIMESTAMP, v DOUBLE) TAGS (labels JSON)") + tdSql.execute("""CREATE TABLE `metrics_0` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.167"}');""") + tdSql.execute("""CREATE TABLE `metrics_1` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.168"}');""") + tdSql.execute("""CREATE TABLE `metrics_2` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.169"}');""") + tdSql.execute("""CREATE TABLE `metrics_3` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.170"}');""") + tdSql.execute("""CREATE TABLE `metrics_5` USING `metrics` (`labels`) TAGS ('{"asset_name":"中国政务网"}');""") + tdSql.execute("""CREATE TABLE `metrics_6` USING `metrics` (`labels`) TAGS ('{"asset_name":"地大物博阿拉丁快解放啦上课交电费"}');""") + tdSql.execute("""CREATE TABLE `metrics_7` USING `metrics` (`labels`) TAGS ('{"asset_name":"no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方"}');""") + tdSql.execute("""CREATE TABLE `metrics_8` USING `metrics` (`labels`) TAGS ('{"asset_name":"no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方","ident":"192.168.0.1"}');""") + tdSql.execute("""CREATE TABLE `metrics_9` USING `metrics` (`labels`) TAGS ('{"asset_name":"no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方","ident":"192.168.0.1"}');""") + tdSql.execute("""CREATE TABLE `metrics_10` USING `metrics` (`labels`) TAGS ('{"asset_name":"上的咖啡机no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方","ident":"192.168.0.1"}');""") + + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:39.326',1)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:40.891',2)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:41.986',3)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:42.992',4)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:46.927',5)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:48.473',6)") + tdSql.execute("insert into metrics_1 select * from metrics_0") + tdSql.execute("insert into metrics_2 select * from metrics_0") + tdSql.execute("insert into metrics_3 select * from metrics_0") + tdSql.execute("insert into metrics_5 select * from metrics_0") + tdSql.execute("insert into metrics_6 select * from metrics_0") + tdSql.execute("insert into metrics_7 select * from metrics_0") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:36.459',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:37.388',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:37.622',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:37.852',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.081',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.307',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.535',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.792',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:39.035',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:39.240',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:29.270',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:30.508',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:31.035',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:31.523',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:31.760',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.001',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.228',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.453',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.690',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.906',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:14.538',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:15.114',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:15.613',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:15.853',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.054',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.295',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.514',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.731',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.958',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:17.176',1)") + + for i in range(1, 10): + tdSql.query("select _wstart,first(v)-last(v), first(labels->'asset_name'),first(labels->'ident'),mode(labels->'asset_name'),mode(labels->'ident'),last(labels->'asset_name'),last(labels->'ident') from ts_5763.metrics interval(1s)") + tdSql.checkRows(18) + + def run(self): + tdLog.debug(f"start to excute {__file__}") + + self.insert_data() + + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 879d93ab3a..e82fd5a85d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -15,6 +15,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_function.py +,,y,army,./pytest.sh python3 ./test.py -f query/function/test_selection_function_with_json.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_percentile.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_resinfo.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_interp.py From dea8d6fd62f6612a598f8c5b840a997761b71da6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 15:03:01 +0800 Subject: [PATCH 20/36] fix some code --- source/common/src/tglobal.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 10 +++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +-- source/libs/wal/src/walRead.c | 2 +- source/util/src/tconfig.c | 4 +-- source/util/src/tenv.c | 4 ++- source/util/src/tunit.c | 36 ++----------------- 9 files changed, 18 insertions(+), 48 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e43bdc4687..c06f9600c2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -568,7 +568,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; if (taosGetFqdn(defaultFqdn) != 0) { - (void)strcpy(defaultFqdn, "localhost"); + tstrncpy(defaultFqdn, "localhost", TSDB_FQDN_LEN); } TAOS_CHECK_RETURN(cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 6dba1825ad..77e89763c2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -104,7 +104,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF } pFD->path = (char *)&pFD[1]; - strcpy(pFD->path, path); + tstrncpy(pFD->path, path, strlen(path) + 1); pFD->szPage = szPage; pFD->flag = flag; pFD->szPage = szPage; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b91abe93af..aa9dbd6cf7 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -199,7 +199,7 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr char *tsdbFilePrefixPos = strstr(oldRname, tsdbFilePrefix); if (tsdbFilePrefixPos == NULL) continue; - int32_t tsdbFileVgId = 0; // atoi(tsdbFilePrefixPos + prefixLen); + int32_t tsdbFileVgId = 0; ret = taosStr2int32(tsdbFilePrefixPos + prefixLen, &tsdbFileVgId); if (ret != 0) { vError("vgId:%d, failed to get tsdb file vgid since %s", dstVgId, tstrerror(ret)); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6ca5803c19..0e0fddfc1f 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -71,7 +71,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (infoReq.option == REQ_OPT_TBUID) reqTbUid = true; metaRsp.dbId = pVnode->config.dbId; - (void)strcpy(metaRsp.tbName, infoReq.tbName); + tstrncpy(metaRsp.tbName, infoReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); if (!reqTbUid) { @@ -105,7 +105,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.tuid = mer1.me.uid; if (mer1.me.type == TSDB_SUPER_TABLE) { - (void)strcpy(metaRsp.stbName, mer1.me.name); + tstrncpy(metaRsp.stbName, mer1.me.name, TSDB_TABLE_NAME_LEN); schema = mer1.me.stbEntry.schemaRow; schemaTag = mer1.me.stbEntry.schemaTag; metaRsp.suid = mer1.me.uid; @@ -113,7 +113,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; - (void)strcpy(metaRsp.stbName, mer2.me.name); + tstrncpy(metaRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN); metaRsp.suid = mer2.me.uid; schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; @@ -220,7 +220,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { goto _exit; } - (void)strcpy(cfgRsp.tbName, cfgReq.tbName); + tstrncpy(cfgRsp.tbName, cfgReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); (void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); @@ -246,7 +246,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; - (void)strcpy(cfgRsp.stbName, mer2.me.name); + tstrncpy(cfgRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN); schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b1c206bdae..fa3a31eb16 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -968,7 +968,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { return; } - strcpy(pMetaRsp->dbFName, pVnode->config.dbname); + tstrncpy(pMetaRsp->dbFName, pVnode->config.dbname, TSDB_DB_FNAME_LEN); pMetaRsp->dbId = pVnode->config.dbId; pMetaRsp->vgId = TD_VID(pVnode); pMetaRsp->precision = pVnode->config.tsdbCfg.precision; @@ -1216,7 +1216,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, rcode = -1; goto _exit; } - strcpy(str, pCreateReq->name); + tstrncpy(str, pCreateReq->name, TSDB_TABLE_FNAME_LEN); if (taosArrayPush(tbNames, &str) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; rcode = -1; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index da5e1f47e9..43a2ff6a23 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -539,7 +539,7 @@ int32_t decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const opts.source = pHead->head.body; opts.result = newBody; opts.unitLen = 16; - TAOS_UNUSED(strncpy((char *)opts.key, cfg->encryptKey, 16)); + tstrncpy((char *)opts.key, cfg->encryptKey, sizeof(opts.key)); int32_t count = CBC_Decrypt(&opts); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 7ff1f18387..9a69d1df54 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1214,14 +1214,14 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { int32_t len = 0; char newValue[1024] = {0}; - strcpy(newValue, value); + tstrncpy(newValue, value, sizeof(newValue)); int32_t count = 1; while (vlen < 1024) { (void)paGetToken(value + vlen + 1 * count, &tmp, &len); if (len == 0) break; tmp[len] = 0; - strcpy(newValue + vlen, tmp); + tstrncpy(newValue + vlen, tmp, sizeof(newValue) - vlen); vlen += len; count++; } diff --git a/source/util/src/tenv.c b/source/util/src/tenv.c index 539687878b..a03d8fa51d 100644 --- a/source/util/src/tenv.c +++ b/source/util/src/tenv.c @@ -57,7 +57,9 @@ int32_t taosEnvToCfg(const char *envStr, char *cfgStr) { if (envStr == NULL || cfgStr == NULL) { return TSDB_CODE_INVALID_PARA; } - if (cfgStr != envStr) strcpy(cfgStr, envStr); + if (cfgStr != envStr) { + tstrncpy(cfgStr, envStr, strlen(envStr) + 1); + } char *p = strchr(cfgStr, '='); if (p != NULL) { diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c index e73045cc89..357e942faa 100644 --- a/source/util/src/tunit.c +++ b/source/util/src/tunit.c @@ -95,33 +95,14 @@ static int32_t parseCfgIntWithUnit(const char* str, int64_t* res) { } int32_t taosStrHumanToInt64(const char* str, int64_t* out) { - int64_t res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) *out = (int64_t)res; return code; } -#ifdef BUILD_NO_CALL -void taosInt64ToHumanStr(int64_t val, char* outStr) { - if (((val >= UNIT_ONE_EXBIBYTE) || (-val >= UNIT_ONE_EXBIBYTE)) && ((val % UNIT_ONE_EXBIBYTE) == 0)) { - sprintf(outStr, "%qdE", (long long)val / UNIT_ONE_EXBIBYTE); - } else if (((val >= UNIT_ONE_PEBIBYTE) || (-val >= UNIT_ONE_PEBIBYTE)) && ((val % UNIT_ONE_PEBIBYTE) == 0)) { - sprintf(outStr, "%qdP", (long long)val / UNIT_ONE_PEBIBYTE); - } else if (((val >= UNIT_ONE_TEBIBYTE) || (-val >= UNIT_ONE_TEBIBYTE)) && ((val % UNIT_ONE_TEBIBYTE) == 0)) { - sprintf(outStr, "%qdT", (long long)val / UNIT_ONE_TEBIBYTE); - } else if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { - sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); - } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { - sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); - } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { - sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); - } else - sprintf(outStr, "%qd", (long long)val); -} -#endif - int32_t taosStrHumanToInt32(const char* str, int32_t* out) { - int64_t res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) { if (res < INT32_MIN || res > INT32_MAX) { @@ -131,16 +112,3 @@ int32_t taosStrHumanToInt32(const char* str, int32_t* out) { } return code; } - -#ifdef BUILD_NO_CALL -void taosInt32ToHumanStr(int32_t val, char* outStr) { - if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { - sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); - } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { - sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); - } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { - sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); - } else - sprintf(outStr, "%qd", (long long)val); -} -#endif From 921704e0437c4e96c0a5e2015a0db5bb6b583821 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 15:28:15 +0800 Subject: [PATCH 21/36] more code --- source/common/src/msg/tmsg.c | 8 ++-- source/common/src/tdataformat.c | 2 +- source/common/src/tname.c | 2 +- source/common/src/ttime.c | 4 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/mgmt/node_util/src/dmFile.c | 2 +- source/dnode/vnode/src/meta/metaOpen.c | 4 +- source/dnode/vnode/src/meta/metaQuery.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 - source/dnode/vnode/src/vnd/vnodeAsync.c | 2 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 8 ++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 4 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- source/libs/geometry/src/geosWrapper.c | 4 +- source/libs/sync/src/syncMain.c | 2 +- source/libs/tdb/src/db/tdbPager.c | 8 ++-- source/util/src/tenv.c | 3 -- source/util/src/tlog.c | 42 ++++--------------- 19 files changed, 37 insertions(+), 70 deletions(-) diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index d18876d655..05d5d1c5a8 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -76,7 +76,7 @@ static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas *p TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); //Obsolete + TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); // Obsolete TAOS_CHECK_RETURN(tEncodeCStr(encoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -87,7 +87,7 @@ static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas *pMoni TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); //Obsolete + TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); // Obsolete TAOS_CHECK_RETURN(tDecodeCStrTo(decoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -10182,7 +10182,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { char name[TSDB_COL_NAME_LEN] = {0}; char *tmp = NULL; TAOS_CHECK_EXIT(tDecodeCStr(pCoder, &tmp)); - strncpy(name, tmp, TSDB_COL_NAME_LEN - 1); + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); if (taosArrayPush(pReq->ctb.tagName, name) == NULL) { TAOS_CHECK_EXIT(terrno); } @@ -12218,7 +12218,7 @@ void setFieldWithOptions(SFieldWithOptions *fieldWithOptions, SField *field) { fieldWithOptions->bytes = field->bytes; fieldWithOptions->flags = field->flags; fieldWithOptions->type = field->type; - strncpy(fieldWithOptions->name, field->name, TSDB_COL_NAME_LEN); + tstrncpy(fieldWithOptions->name, field->name, TSDB_COL_NAME_LEN); } int32_t tSerializeTableTSMAInfoReq(void *buf, int32_t bufLen, const STableTSMAInfoReq *pReq) { SEncoder encoder = {0}; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a38842735c..c1ab7ccff0 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1434,7 +1434,7 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { char tmpVal[32] = {0}; - strncpy(tmpVal, val, vlen > 31 ? 31 : vlen); + tstrncpy(tmpVal, val, vlen > 31 ? 31 : vlen); printf("%s:%d type:%d vlen:%d, val:\"%s\"\n", tag, ln, (int32_t)type, vlen, tmpVal); } break; case TSDB_DATA_TYPE_FLOAT: diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 9ced37eb38..6bd64fb5e6 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -81,7 +81,7 @@ SName* tNameDup(const SName* name) { } int32_t tNameGetDbName(const SName* name, char* dst) { - strncpy(dst, name->dbname, tListLen(name->dbname)); + tstrncpy(dst, name->dbname, tListLen(name->dbname)); return 0; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 9746fea034..265af4cf13 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1316,7 +1316,7 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int TSFormatNode* format = taosArrayGet(formats, i); if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) { if (s - start + format->len + 1 > outLen) break; - (void)strncpy(s, format->c, format->len); + tstrncpy(s, format->c, format->len); s += format->len; continue; } @@ -1536,7 +1536,7 @@ static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len s = last; } else { char buf[16] = {0}; - (void)strncpy(buf, s, len); + tstrncpy(buf, s, len); int32_t copiedLen = strlen(buf); if (copiedLen < len) { if (!needMoreDigit) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 8e8d70118b..83043b4393 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -211,7 +211,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { - strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); } #else pCfg->tdbEncryptAlgorithm = 0; diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 26f45d2fb8..d5006e1e4d 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -551,7 +551,7 @@ int32_t dmGetEncryptKey() { goto _OVER; } - strncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); + tstrncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); taosMemoryFreeClear(encryptKey); tsEncryptionKeyChksum = taosCalcChecksum(0, tsEncryptKey, strlen(tsEncryptKey)); tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED; diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 5351554631..bc18aa675d 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -197,7 +197,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); TSDB_CHECK_CODE(code, lino, _exit); - sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); + tsnprintf(indexFullPath, sizeof(indexFullPath), "%s/%s", pMeta->path, "invert"); ret = taosMkDir(indexFullPath); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; @@ -209,7 +209,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, // open pTtlMgr ("ttlv1.idx") char logPrefix[128] = {0}; - sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); + tsnprintf(logPrefix, sizeof(logPrefix), "vgId:%d", TD_VID(pVnode)); code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 87beb8842b..c19a2e3ce2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -164,7 +164,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { metaReaderClear(&mr); return code; } - strncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); + tstrncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); metaReaderClear(&mr); return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 77e89763c2..bf79b2482d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -174,7 +174,6 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e opts.source = pFD->pBuf + count; opts.result = PacketData; opts.unitLen = 128; - // strncpy(opts.key, tsEncryptKey, 16); tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Encrypt(&opts); @@ -248,7 +247,6 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor opts.source = pFD->pBuf + count; opts.result = PacketData; opts.unitLen = 128; - // strncpy(opts.key, tsEncryptKey, 16); tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Decrypt(&opts); diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 9e4fbd84a9..424ed0f325 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -330,7 +330,7 @@ static int32_t vnodeAsyncInit(SVAsync **async, const char *label) { return terrno; } - strcpy((char *)((*async) + 1), label); + tstrncpy((char *)((*async) + 1), label, strlen(label) + 1); (*async)->label = (const char *)((*async) + 1); (void)taosThreadMutexInit(&(*async)->mutex, NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 2ceeeca160..9c153bc8a1 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -303,7 +303,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); } } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index aa9dbd6cf7..f80637859d 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -168,7 +168,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t static int32_t vnodeVgroupIdLen(int32_t vgId) { char tmp[TSDB_FILENAME_LEN]; - sprintf(tmp, "%d", vgId); + tsnprintf(tmp, TSDB_FILENAME_LEN, "%d", vgId); return strlen(tmp); } @@ -417,7 +417,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } pVnode->path = (char *)&pVnode[1]; - strcpy(pVnode->path, path); + tstrncpy(pVnode->path, path, strlen(path) + 1); pVnode->config = info.config; pVnode->state.committed = info.state.committed; pVnode->state.commitTerm = info.state.commitTerm; @@ -472,7 +472,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open wal - sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); + tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); @@ -484,7 +484,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open tq - sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); + tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 0e0fddfc1f..de3c482828 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -75,7 +75,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); if (!reqTbUid) { - TAOS_UNUSED(sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName)); + tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit4; @@ -223,7 +223,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { tstrncpy(cfgRsp.tbName, cfgReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); - (void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); + tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", cfgReq.dbFName, cfgReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 091a36f1d7..e394cdb8ff 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1225,7 +1225,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } // validate hash - sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name); + tsnprintf(tbName, TSDB_TABLE_FNAME_LEN, "%s.%s", pVnode->config.dbname, pCreateReq->name); if (vnodeValidateTableHash(pVnode, tbName) < 0) { cRsp.code = TSDB_CODE_VND_HASH_MISMATCH; if (taosArrayPush(rsp.pArray, &cRsp) == NULL) { @@ -1518,7 +1518,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in pRsp->code = terrno; goto _exit; } - strcpy(str, pDropTbReq->name); + tstrncpy(str, pDropTbReq->name, TSDB_TABLE_FNAME_LEN); if (taosArrayPush(tbNames, &str) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pRsp->code = terrno; diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 8789762a85..0eb7c90317 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -99,8 +99,8 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData return terrno; } - (void)sprintf( - wktPatternWithSpace, + tsnprintf( + wktPatternWithSpace, 1024, "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( " "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b9e2d3435f..fbde104f4e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -692,7 +692,7 @@ int32_t syncGetArbToken(int64_t rid, char* outToken) { memset(outToken, 0, TSDB_ARB_TOKEN_SIZE); (void)taosThreadMutexLock(&pSyncNode->arbTokenMutex); - strncpy(outToken, pSyncNode->arbToken, TSDB_ARB_TOKEN_SIZE); + tstrncpy(outToken, pSyncNode->arbToken, TSDB_ARB_TOKEN_SIZE); (void)taosThreadMutexUnlock(&pSyncNode->arbTokenMutex); syncNodeRelease(pSyncNode); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index c2f97982f5..2336425f27 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -262,7 +262,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ // Open the journal char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); @@ -365,7 +365,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); // remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { @@ -595,7 +595,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { } char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); @@ -1179,7 +1179,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { int dirLen = strlen(pPager->pEnv->dbName); memcpy(jname, pPager->pEnv->dbName, dirLen); jname[dirLen] = '/'; - sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); + tsnprintf(jname + dirLen + 1, TD_PATH_MAX - dirLen - 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); code = tdbPagerRestore(pPager, jname); if (code) { taosArrayDestroy(pTxnList); diff --git a/source/util/src/tenv.c b/source/util/src/tenv.c index a03d8fa51d..2108f05c5b 100644 --- a/source/util/src/tenv.c +++ b/source/util/src/tenv.c @@ -24,9 +24,6 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c char *p = cfgNameStr; if (envNameStr[0] != 'T' || envNameStr[1] != 'A' || envNameStr[2] != 'O' || envNameStr[3] != 'S' || envNameStr[4] != '_') { - // if(p != envNameStr) strncpy(p, envNameStr, cfgNameMaxLen - 1); - // p[cfgNameMaxLen - 1] = '\0'; - // return strlen(cfgNameStr); cfgNameStr[0] = '\0'; return TSDB_CODE_INVALID_PARA; } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c3d78b7c17..a9eef1bfc9 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,8 +19,8 @@ #include "tconfig.h" #include "tglobal.h" #include "tjson.h" -#include "tutil.h" #include "ttime.h" +#include "tutil.h" #define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) @@ -166,9 +166,9 @@ static int32_t taosStartLog() { } static int32_t getDay(char *buf, int32_t bufSize) { - time_t t; + time_t t; int32_t code = taosTime(&t); - if(code != 0) { + if (code != 0) { return code; } struct tm tmInfo; @@ -204,8 +204,8 @@ int32_t taosInitSlowLog() { getFullPathName(tsLogObj.slowLogName, logFileName); - char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; - char day[TD_TIME_STR_LEN] = {0}; + char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; + char day[TD_TIME_STR_LEN] = {0}; int32_t code = getDay(day, sizeof(day)); if (code != 0) { (void)printf("failed to get day, reason:%s\n", tstrerror(code)); @@ -299,9 +299,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { return 0; } -void taosSetNoNewFile() { - tsLogObj.openInProgress = 1; -} +void taosSetNoNewFile() { tsLogObj.openInProgress = 1; } static void taosStopLog() { if (tsLogObj.logHandle) { @@ -503,7 +501,7 @@ static void taosOpenNewSlowLogFile() { taosWriteLog(tsLogObj.slowHandle); atomic_store_32(&tsLogObj.slowHandle->lock, 0); - char day[TD_TIME_STR_LEN] = {0}; + char day[TD_TIME_STR_LEN] = {0}; int32_t code = getDay(day, sizeof(day)); if (code != 0) { uError("failed to get day, reason:%s", tstrerror(code)); @@ -882,32 +880,6 @@ void taosPrintSlowLog(const char *format, ...) { taosMemoryFree(buffer); } -#if 0 -void taosDumpData(unsigned char *msg, int32_t len) { - if (!osLogSpaceAvailable()) return; - taosUpdateLogNums(DEBUG_DUMP); - - char temp[256] = {0}; - int32_t i, pos = 0, c = 0; - - for (i = 0; i < len; ++i) { - sprintf(temp + pos, "%02x ", msg[i]); - c++; - pos += 3; - if (c >= 16) { - temp[pos++] = '\n'; - TAOS_UNUSED((taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos) <= 0)); - c = 0; - pos = 0; - } - } - - temp[pos++] = '\n'; - - TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos)); -} -#endif - static void taosCloseLogByFd(TdFilePtr pFile) { if (pFile != NULL) { taosUnLockLogFile(pFile); From 4ea6082a7825d81cca867ddfd0876704e2d1c0ec Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 15:43:19 +0800 Subject: [PATCH 22/36] more code change --- include/libs/crypt/crypt.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/include/libs/crypt/crypt.h b/include/libs/crypt/crypt.h index c294877a57..5f981b7ac8 100644 --- a/include/libs/crypt/crypt.h +++ b/include/libs/crypt/crypt.h @@ -21,19 +21,19 @@ extern "C" { #endif -typedef struct SCryptOpts{ - int32_t len; - char* source; - char* result; - int32_t unitLen; - unsigned char key[17]; -}SCryptOpts; +typedef struct SCryptOpts { + int32_t len; + char* source; + char* result; + int32_t unitLen; + char key[17]; +} SCryptOpts; -int32_t CBC_Decrypt(SCryptOpts *opts); -int32_t CBC_Encrypt(SCryptOpts *opts); +int32_t CBC_Decrypt(SCryptOpts* opts); +int32_t CBC_Encrypt(SCryptOpts* opts); #ifdef __cplusplus } #endif -#endif // _CRYPT_H_ \ No newline at end of file +#endif // _CRYPT_H_ \ No newline at end of file From bf878a557ed0318e082ebc5b6325340b073fb32e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 16:20:26 +0800 Subject: [PATCH 23/36] Update 02-management.md --- docs/zh/06-advanced/06-TDgpt/02-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index ef1206fc04..07203b977d 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -37,7 +37,7 @@ systemctl status taosanoded |/usr/local/taos/taosanode/bin|可执行文件目录| |/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/| |/usr/local/taos/taosanode/lib|库文件目录| -|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model| +|/usr/local/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model| |/var/log/taos/taosanode/|日志文件目录| |/etc/taos/taosanode.ini|配置文件| From 3c90b9ac8af7d610df3ee3276c2d2d7f5adbba19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 16:20:57 +0800 Subject: [PATCH 24/36] Update 03-preprocess.md --- docs/zh/06-advanced/06-TDgpt/03-preprocess.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md index 9efd2bdf11..b63cae0740 100644 --- a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md +++ b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md @@ -12,7 +12,7 @@ import wndata from './pic/white-noise-data.png' 预处理流程 TDgpt 首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还要进行输入（历史）数据的重采样和时间戳对齐处理（异常检测跳过数据重采样和时间戳对齐步骤）。 -预处理完成以后，再进行预测或异常检测操作。预处理过程部署于预测或异常检测处理逻辑的一部分。 +预处理完成以后，再进行预测或异常检测操作。预处理过程不属于预测或异常检测处理逻辑的一部分。 ### 白噪声检查 From 1c63cd75b10abef88ae7eccb692f576a805d2411 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 16:21:38 +0800 Subject: [PATCH 25/36] Update index.md --- docs/zh/06-advanced/06-TDgpt/04-forecast/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index c7388ab9c0..a1149772d0 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -14,7 +14,7 @@ description: 预测算法 ```bash taos> select * from foo; - ts | k | + ts | i32 | ======================================== 2020-01-01 00:00:12.681 | 13 | 2020-01-01 00:00:13.727 | 14 | From 4f878078bdf4770fd224e9566f2410258a2fd0ba Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 16:35:52 +0800 Subject: [PATCH 26/36] Update index.md --- docs/zh/06-advanced/06-TDgpt/04-forecast/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index a1149772d0..3981fff8c6 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -42,7 +42,7 @@ algo=expr1 ``` 1. `column_expr`：预测的时序数据列。与异常检测相同，只支持数值类型列输入。 -2. `options`：异常检测函数的参数，使用规则与 anomaly_window 相同。预测支持 `conf`, `every`, `rows`, `start`, `rows` 几个控制参数，其含义如下： +2. `options`：预测函数的参数。字符串类型，其中使用 K=V 方式调用算法及相关参数。采用逗号分隔的 K=V 字符串表示，其中的字符串不需要使用单引号、双引号、或转义号等符号，不能使用中文及其他宽字符。预测支持 `conf`, `every`, `rows`, `start`, `rows` 几个控制参数，其含义如下： ### 参数说明 From 697b8749ddd1215b5f895daea9a9978387eaf0b5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 16:43:52 +0800 Subject: [PATCH 27/36] Update index.md --- docs/zh/06-advanced/06-TDgpt/06-dev/index.md | 43 ++++++++------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md index 072a66c7d3..bcd972df8e 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md @@ -19,24 +19,25 @@ Anode的主要目录结构如下图所示 ```bash . +├── bin ├── cfg -├── model -│   └── ad_autoencoder -├── release -├── script -└── taosanalytics - ├── algo - │   ├── ad - │   └── fc - ├── misc - └── test +├── lib +│   └── taosanalytics +│   ├── algo +│   │   ├── ad +│   │   └── fc +│   ├── misc +│   └── test +├── log -> /var/log/taos/taosanode +├── model -> /var/lib/taos/taosanode/model +└── venv -> /var/lib/taos/taosanode/venv ``` |目录|说明| |---|---| |taosanalytics| 源代码目录，其下包含了算法具体保存目录 algo，放置杂项目录 misc，单元测试和集成测试目录 test。 algo 目录下 ad 保存异常检测算法代码，fc 目录保存预测算法代码| -|script|是安装脚本和发布脚本放置目录| +|venv| Python 虚拟环境| |model|放置针对数据集完成的训练模型| |cfg|配置文件目录| @@ -63,7 +64,8 @@ Anode采用算法自动加载模式，因此只识别符合命名约定的 Pytho ```SQL --- algo 后面的参数 name 即为类属性 `name` -SELECT COUNT(*) FROM foo ANOMALY_WINDOW(col_name, 'algo=name') +SELECT COUNT(*) +FROM foo ANOMALY_WINDOW(col_name, 'algo=name') ``` ## 添加具有模型的分析算法 @@ -76,19 +78,10 @@ SELECT COUNT(*) FROM foo ANOMALY_WINDOW(col_name, 'algo=name') ```bash . -├── cfg -├── model -│   └── ad_autoencoder -│   ├── ad_autoencoder_foo.dat -│   └── ad_autoencoder_foo.info -├── release -├── script -└── taosanalytics - ├── algo - │   ├── ad - │   └── fc - ├── misc - └── test +└── model + └── ad_autoencoder + ├── ad_autoencoder_foo.dat + └── ad_autoencoder_foo.info ``` From afd11de8fe36f505beaf5f3a62c2eb5b45201a49 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 16:49:53 +0800 Subject: [PATCH 28/36] Update 02-management.md --- docs/zh/06-advanced/06-TDgpt/02-management.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index 07203b977d..c41dade3d3 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -4,11 +4,10 @@ sidebar_label: "安装部署" --- ### 环境准备 -使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/MacOS 等平台上，同时需要 3.10 或以上版本的 Python 环境支持。 +使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 运行在 Linux 平台上，并需要 3.10 或以上版本的 Python 环境支持。 > 部署 Anode 需要 TDengine Enterprise 3.3.4.3 及以后版本，请首先确认搭配 Anode 使用的 TDengine 能够支持 Anode。 ### 安装及卸载 -不同操作系统上安装及部署 Anode 有一些差异，主要是卸载操作、安装路径、服务启停等方面。本文以 Linux 系统为例，说明安装部署的流程。 使用 Linux 环境下的安装包 TDengine-enterprise-anode-1.x.x.tar.gz 可进行 Anode 的安装部署工作，命令如下： ```bash From b55108105904626e28857769f4273179f79ab4b4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 17:18:16 +0800 Subject: [PATCH 29/36] Update 03-ad.md --- docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md index 5b49db330e..c48ce42836 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md @@ -16,7 +16,7 @@ sidebar_label: "异常检测" ```python import numpy as np -from service import AbstractAnomalyDetectionService +from taosanalytics.service import AbstractAnomalyDetectionService # 算法实现类名称 需要以下划线 "_" 开始，并以 Service 结束 class _MyAnomalyDetectionService(AbstractAnomalyDetectionService): From 5b829b76ecfed09f1fc10d699cd039c87e67fdf9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 17:24:13 +0800 Subject: [PATCH 30/36] Update 02-forecast.md --- docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index 954076c8fd..7f4f81034e 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -34,7 +34,7 @@ return { ```python import numpy as np -from service import AbstractForecastService +from taosanalytics.service import AbstractForecastService # 算法实现类名称 需要以下划线 "_" 开始，并以 Service 结束 class _MyForecastService(AbstractForecastService): From 0b2705d68022467c9ca01ab0016ac5876b9c83ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 17:36:20 +0800 Subject: [PATCH 31/36] Update 02-management.md --- docs/zh/06-advanced/06-TDgpt/02-management.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index c41dade3d3..b37c39944f 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -63,7 +63,7 @@ pidfile = /usr/local/taos/taosanode/taosanode.pid # conflict with systemctl, so do NOT uncomment this # daemonize = /var/log/taos/taosanode/taosanode.log -# log directory +# uWSGI log files logto = /var/log/taos/taosanode/taosanode.log # wWSGI monitor port @@ -73,7 +73,7 @@ stats = 127.0.0.1:8387 virtualenv = /usr/local/taos/taosanode/venv/ [taosanode] -# default app log file +# default taosanode log file app-log = /var/log/taos/taosanode/taosanode.app.log # model storage directory From 5ff10141f558fd25564a28814f485e7af9b076dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Dec 2024 18:15:41 +0800 Subject: [PATCH 32/36] Update 02-forecast.md --- .../06-TDgpt/06-dev/02-forecast.md | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index 7f4f81034e..0584c87311 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -36,6 +36,7 @@ return { import numpy as np from taosanalytics.service import AbstractForecastService + # 算法实现类名称 需要以下划线 "_" 开始，并以 Service 结束 class _MyForecastService(AbstractForecastService): """ 定义类，从 AbstractForecastService 继承并实现其定义的抽象方法 execute """ @@ -51,12 +52,12 @@ class _MyForecastService(AbstractForecastService): super().__init__() def execute(self): - """ 算法逻辑的核心实现""" + """ 算法逻辑的核心实现""" res = [] """这个预测算法固定返回 1 作为预测值，预测值的数量是用户通过 self.fc_rows 指定""" ts_list = [self.start_ts + i * self.time_step for i in range(self.fc_rows)] - res.app(ts_list) # 设置预测结果时间戳列 + res.append(ts_list) # 设置预测结果时间戳列 """生成全部为 1 的预测结果 """ res_list = [1] * self.fc_rows @@ -64,18 +65,18 @@ class _MyForecastService(AbstractForecastService): """检查用户输入，是否要求返回预测置信区间上下界""" if self.return_conf: - """对于没有计算预测置信区间上下界的算法，直接返回预测值作为上下界即可""" - bound_list = [1] * self.fc_rows - res.append(bound_list) # 预测结果置信区间下界 - res.append(bound_list) # 预测结果执行区间上界 + """对于没有计算预测置信区间上下界的算法，直接返回预测值作为上下界即可""" + bound_list = [1] * self.fc_rows + res.append(bound_list) # 预测结果置信区间下界 + res.append(bound_list) # 预测结果执行区间上界 """返回结果""" - return { "res": res, "mse": 0} + return {"res": res, "mse": 0} - def set_params(self, params): - """该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑""" - pass + """该算法无需任何输入参数，直接调用父类函数，不处理算法参数设置逻辑""" + return super().set_params(params) + ``` 将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口中执行 `SHOW ANODES FULL` 能够看到新加入的算法。应用就可以通过 SQL 语句调用该预测算法。 From b6b9c65b997fcd82ef7f24915a31d1249674e99f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 20:59:39 +0800 Subject: [PATCH 33/36] more code --- source/dnode/vnode/src/meta/metaOpen.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeOpen.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeQuery.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/geometry/src/geosWrapper.c | 2 +- source/libs/tdb/src/db/tdbPager.c | 8 ++++---- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index bc18aa675d..cc440fd73b 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -197,7 +197,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); TSDB_CHECK_CODE(code, lino, _exit); - tsnprintf(indexFullPath, sizeof(indexFullPath), "%s/%s", pMeta->path, "invert"); + (void)tsnprintf(indexFullPath, sizeof(indexFullPath), "%s/%s", pMeta->path, "invert"); ret = taosMkDir(indexFullPath); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; @@ -209,7 +209,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, // open pTtlMgr ("ttlv1.idx") char logPrefix[128] = {0}; - tsnprintf(logPrefix, sizeof(logPrefix), "vgId:%d", TD_VID(pVnode)); + (void)tsnprintf(logPrefix, sizeof(logPrefix), "vgId:%d", TD_VID(pVnode)); code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index f80637859d..b9e686932e 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -168,7 +168,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t static int32_t vnodeVgroupIdLen(int32_t vgId) { char tmp[TSDB_FILENAME_LEN]; - tsnprintf(tmp, TSDB_FILENAME_LEN, "%d", vgId); + (void)tsnprintf(tmp, TSDB_FILENAME_LEN, "%d", vgId); return strlen(tmp); } @@ -472,7 +472,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open wal - tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); + (void)tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); @@ -484,7 +484,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open tq - tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); + (void)tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index de3c482828..f52a0c3aba 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -75,7 +75,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); if (!reqTbUid) { - tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", infoReq.dbFName, infoReq.tbName); + (void)tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit4; @@ -223,7 +223,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { tstrncpy(cfgRsp.tbName, cfgReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); - tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", cfgReq.dbFName, cfgReq.tbName); + (void)tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", cfgReq.dbFName, cfgReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e394cdb8ff..d9b41869c7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1225,7 +1225,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } // validate hash - tsnprintf(tbName, TSDB_TABLE_FNAME_LEN, "%s.%s", pVnode->config.dbname, pCreateReq->name); + (void)tsnprintf(tbName, TSDB_TABLE_FNAME_LEN, "%s.%s", pVnode->config.dbname, pCreateReq->name); if (vnodeValidateTableHash(pVnode, tbName) < 0) { cRsp.code = TSDB_CODE_VND_HASH_MISMATCH; if (taosArrayPush(rsp.pArray, &cRsp) == NULL) { diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 0eb7c90317..69d609b4c4 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -99,7 +99,7 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData return terrno; } - tsnprintf( + (void)tsnprintf( wktPatternWithSpace, 1024, "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( " diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 2336425f27..ca39bd1a23 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -262,7 +262,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ // Open the journal char jTxnFileName[TDB_FILENAME_LEN]; - tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); @@ -365,7 +365,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { char jTxnFileName[TDB_FILENAME_LEN]; - tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); // remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { @@ -595,7 +595,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { } char jTxnFileName[TDB_FILENAME_LEN]; - tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); @@ -1179,7 +1179,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { int dirLen = strlen(pPager->pEnv->dbName); memcpy(jname, pPager->pEnv->dbName, dirLen); jname[dirLen] = '/'; - tsnprintf(jname + dirLen + 1, TD_PATH_MAX - dirLen - 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); + (void)tsnprintf(jname + dirLen + 1, TD_PATH_MAX - dirLen - 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); code = tdbPagerRestore(pPager, jname); if (code) { taosArrayDestroy(pTxnList); From e30ab6a48bf81611f5ac480e9a6bae5003ed89dd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 21:40:12 +0800 Subject: [PATCH 34/36] make CI pass --- source/libs/geometry/src/geosWrapper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 69d609b4c4..9fed5c2a6e 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -100,7 +100,7 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData } (void)tsnprintf( - wktPatternWithSpace, 1024, + wktPatternWithSpace, 4 * 1024, "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( " "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " From 5dca96281406191b5e38e2304e19ca559f3c81e4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 Dec 2024 15:19:19 +0800 Subject: [PATCH 35/36] more fix --- source/common/src/ttime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 265af4cf13..6d85a989dd 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1316,7 +1316,7 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int TSFormatNode* format = taosArrayGet(formats, i); if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) { if (s - start + format->len + 1 > outLen) break; - tstrncpy(s, format->c, format->len); + tstrncpy(s, format->c, format->len + 1); s += format->len; continue; } From ca4ae6739f671b673069ecb29434e3b78ea43019 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 Dec 2024 15:29:40 +0800 Subject: [PATCH 36/36] alright, i give up --- source/common/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 6d85a989dd..9746fea034 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1316,7 +1316,7 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int TSFormatNode* format = taosArrayGet(formats, i); if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) { if (s - start + format->len + 1 > outLen) break; - tstrncpy(s, format->c, format->len + 1); + (void)strncpy(s, format->c, format->len); s += format->len; continue; } @@ -1536,7 +1536,7 @@ static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len s = last; } else { char buf[16] = {0}; - tstrncpy(buf, s, len); + (void)strncpy(buf, s, len); int32_t copiedLen = strlen(buf); if (copiedLen < len) { if (!needMoreDigit) {