From a4962b58a6fb06c594ef3949cb657c3ecabe2af7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 5 Sep 2024 17:52:21 +0800 Subject: [PATCH 01/61] add trace --- source/util/src/tref.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 0eac7b4427..d3fafa4167 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -19,10 +19,11 @@ #include "tlog.h" #include "tutil.h" -#define TSDB_REF_OBJECTS 50 -#define TSDB_REF_STATE_EMPTY 0 -#define TSDB_REF_STATE_ACTIVE 1 -#define TSDB_REF_STATE_DELETED 2 +#define TSDB_REF_OBJECTS 50 +#define TSDB_REF_STATE_EMPTY 0 +#define TSDB_REF_STATE_ACTIVE 1 +#define TSDB_REF_STATE_DELETED 2 +#define TSDB_REF_ITER_THRESHOLD 100 typedef struct SRefNode { struct SRefNode *prev; // previous node @@ -188,6 +189,7 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid) { int32_t hash; SRefNode *pNode; SRefSet *pSet; + int32_t iter = 0; void *p = NULL; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { @@ -220,10 +222,14 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid) { if (pNode->rid == rid) { break; } - + iter++; pNode = pNode->next; } + if (iter >= TSDB_REF_ITER_THRESHOLD) { + uWarn("rsetId:%d rid:%" PRId64 " iter:%d", rsetId, rid, iter); + } + if (pNode) { if (pNode->removed == 0) { pNode->count++; @@ -277,6 +283,7 @@ void *taosIterateRef(int32_t rsetId, int64_t rid) { do { newP = NULL; int32_t hash = 0; + int32_t iter = 0; if (rid > 0) { hash = rid % pSet->max; taosLockList(pSet->lockedBy + hash); @@ -285,6 +292,11 @@ void *taosIterateRef(int32_t rsetId, int64_t rid) { while (pNode) { if (pNode->rid == rid) break; pNode = pNode->next; + iter++; + } + + if (iter >= TSDB_REF_ITER_THRESHOLD) { + uWarn("rsetId:%d rid:%" PRId64 " iter:%d", rsetId, rid, iter); } if (pNode == NULL) { @@ -376,6 +388,7 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { int32_t hash; SRefSet *pSet; SRefNode *pNode; + int32_t iter = 0; int32_t released = 0; int32_t code = 0; @@ -403,6 +416,11 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { if (pNode->rid == rid) break; pNode = pNode->next; + iter++; + } + + if (iter >= TSDB_REF_ITER_THRESHOLD) { + uWarn("rsetId:%d rid:%" PRId64 " iter:%d", rsetId, rid, iter); } if (pNode) { From 0b21b7dff175950abba00dcb4d551afac2dd09af Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 14:25:50 +0800 Subject: [PATCH 02/61] fix indirect leak --- source/libs/transport/src/trans.c | 12 ++- source/libs/transport/src/transCli.c | 109 ++++++++------------ source/libs/transport/src/transSvr.c | 143 ++++++++++----------------- 3 files changed, 98 insertions(+), 166 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1c783f44fe..f017909341 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -103,7 +103,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->timeToGetConn = 10 * 1000; } pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; - + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -124,14 +124,20 @@ _end: } void rpcClose(void* arg) { tInfo("start to close rpc"); + if (arg == NULL) { + return; + } (void)transRemoveExHandle(transGetInstMgt(), (int64_t)arg); (void)transReleaseExHandle(transGetInstMgt(), (int64_t)arg); tInfo("end to close rpc"); return; } void rpcCloseImpl(void* arg) { + if (arg == NULL) return; SRpcInfo* pRpc = (SRpcInfo*)arg; - (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + if (pRpc->tcphandle != NULL) { + (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + } taosMemoryFree(pRpc); } @@ -168,7 +174,7 @@ int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64 return transSendRequest(shandle, pEpSet, pMsg, NULL); } int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { - if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0|| pRid == NULL) { + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { return transSendRequest(shandle, pEpSet, pMsg, pCtx); } else { return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 78320c450c..5052d79ef3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -99,12 +99,11 @@ typedef struct SCliMsg { } SCliMsg; typedef struct SCliThrd { - TdThread thread; // tid - int64_t pid; // pid - uv_loop_t* loop; - SAsyncPool* asyncPool; - uv_prepare_t* prepare; - void* pool; // conn pool + TdThread thread; // tid + int64_t pid; // pid + uv_loop_t* loop; + SAsyncPool* asyncPool; + void* pool; // conn pool // timer handles SArray* timerList; // msg queue @@ -167,7 +166,6 @@ static void cliSendCb(uv_write_t* req, int status); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); static void cliIdleCb(uv_idle_t* handle); -static void cliPrepareCb(uv_prepare_t* handle); static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd); static void cliSendBatchCb(uv_write_t* req, int status); @@ -231,7 +229,9 @@ static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static int32_t createThrdObj(void* trans, SCliThrd** pThrd); static void destroyThrdObj(SCliThrd* pThrd); -static void cliWalkCb(uv_handle_t* handle, void* arg); + +int32_t cliSendQuit(SCliThrd* thrd); +static void cliWalkCb(uv_handle_t* handle, void* arg); #define CLI_RELEASE_UV(loop) \ do { \ @@ -2119,33 +2119,6 @@ static void cliAsyncCb(uv_async_t* handle) { if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); } -static void cliPrepareCb(uv_prepare_t* handle) { - SCliThrd* thrd = handle->data; - tTrace("prepare work start"); - - SAsyncPool* pool = thrd->asyncPool; - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - (void)taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); - - int count = 0; - while (!QUEUE_IS_EMPTY(&wq)) { - queue* h = QUEUE_HEAD(&wq); - QUEUE_REMOVE(h); - - SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - (*cliAsyncHandle[pMsg->type])(pMsg, thrd); - count++; - } - } - tTrace("prepare work end"); - if (thrd->stopMsg != NULL) cliHandleQuit(thrd->stopMsg, thrd); -} void cliDestroyConnMsgs(SCliConn* conn, bool destroy) { transCtxCleanup(&conn->ctx); @@ -2260,6 +2233,12 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, _err: if (cli) { + for (int i = 0; i < cli->numOfThreads; i++) { + if (cli->pThreadObj[i]) { + (void)cliSendQuit(cli->pThreadObj[i]); + destroyThrdObj(cli->pThreadObj[i]); + } + } taosMemoryFree(cli->pThreadObj); taosMemoryFree(cli); } @@ -2339,37 +2318,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(code, NULL, _end); } - pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); - if (pThrd->prepare == NULL) { - tError("failed to create prepre since:%s", tstrerror(code)); - TAOS_CHECK_GOTO(code, NULL, _end); - } - - code = uv_prepare_init(pThrd->loop, pThrd->prepare); - if (code != 0) { - tError("failed to create prepre since:%s", uv_err_name(code)); - TAOS_CHECK_GOTO(TSDB_CODE_THIRDPARTY_ERROR, NULL, _end); - } - pThrd->prepare->data = pThrd; - - int32_t timerSize = 64; - pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); - if (pThrd->timerList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - - for (int i = 0; i < timerSize; i++) { - uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); - if (timer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - (void)uv_timer_init(pThrd->loop, timer); - if (taosArrayPush(pThrd->timerList, &timer) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - } - pThrd->pool = createConnPool(4); if (pThrd->pool == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -2402,6 +2350,23 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); } + int32_t timerSize = 64; + pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); + if (pThrd->timerList == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + for (int i = 0; i < timerSize; i++) { + uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + if (timer == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + (void)uv_timer_init(pThrd->loop, timer); + if (taosArrayPush(pThrd->timerList, &timer) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + } pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = trans; pThrd->quit = false; @@ -2411,17 +2376,21 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { _end: if (pThrd) { + (void)taosThreadMutexDestroy(&pThrd->msgMtx); + (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); - taosMemoryFree(pThrd->prepare); - (void)taosThreadMutexDestroy(&pThrd->msgMtx); transAsyncPoolDestroy(pThrd->asyncPool); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); - taosMemoryFree(pThrd->prepare); + + destroyConnPool(pThrd); + transDQDestroy(pThrd->delayQueue, NULL); + transDQDestroy(pThrd->timeoutQueue, NULL); + transDQDestroy(pThrd->waitConnQueue, NULL); taosHashCleanup(pThrd->fqdn2ipCache); taosHashCleanup(pThrd->failFastCache); taosHashCleanup(pThrd->batchCache); @@ -2450,8 +2419,8 @@ static void destroyThrdObj(SCliThrd* pThrd) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } + uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); - taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); taosHashCleanup(pThrd->failFastCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 11aa468b19..0727e49d8b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -82,14 +82,13 @@ typedef struct { int64_t ver; } SIpWhiteListTab; typedef struct SWorkThrd { - TdThread thread; - uv_connect_t connect_req; - uv_pipe_t* pipe; - uv_os_fd_t fd; - uv_loop_t* loop; - SAsyncPool* asyncPool; - uv_prepare_t* prepare; - queue msg; + TdThread thread; + uv_connect_t connect_req; + uv_pipe_t* pipe; + uv_os_fd_t fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + queue msg; queue conn; void* pTransInst; @@ -98,6 +97,7 @@ typedef struct SWorkThrd { SIpWhiteListTab* pWhiteList; int64_t whiteListVer; int8_t enableIpWhiteList; + int8_t inited; } SWorkThrd; typedef struct SServerObj { @@ -139,7 +139,6 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); static void uvShutDownCb(uv_shutdown_t* req, int status); -static void uvPrepareCb(uv_prepare_t* handle); static bool uvRecvReleaseReq(SSvrConn* conn, STransMsgHead* pHead); @@ -180,6 +179,11 @@ static void uvDestroyConn(uv_handle_t* handle); static void* transWorkerThread(void* arg); static void* transAcceptThread(void* arg); +static void destroyWorkThrd(SWorkThrd* pThrd); +static void destroyWorkThrdObj(SWorkThrd* pThrd); + +static void sendQuitToWorkThrd(SWorkThrd* pThrd); + // add handle loop static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); @@ -849,52 +853,6 @@ static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { } return false; } -static void uvPrepareCb(uv_prepare_t* handle) { - // prepare callback - SWorkThrd* pThrd = handle->data; - SAsyncPool* pool = pThrd->asyncPool; - - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - (void)taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); - - while (!QUEUE_IS_EMPTY(&wq)) { - queue* head = QUEUE_HEAD(&wq); - QUEUE_REMOVE(head); - - SSvrMsg* msg = QUEUE_DATA(head, SSvrMsg, q); - if (msg == NULL) { - tError("unexcept occurred, continue"); - continue; - } - // release handle to rpc init - if (msg->type == Quit || msg->type == Update) { - (*transAsyncHandle[msg->type])(msg, pThrd); - continue; - } else { - STransMsg transMsg = msg->msg; - - SExHandle* exh1 = transMsg.info.handle; - int64_t refId = transMsg.info.refId; - SExHandle* exh2 = transAcquireExHandle(transGetSvrRefMgt(), refId); - if (exh2 == NULL || exh1 != exh2) { - tTrace("handle except msg %p, ignore it", exh1); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); - destroySmsg(msg); - continue; - } - msg->pConn = exh1->handle; - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); - (*transAsyncHandle[msg->type])(msg, pThrd); - } - } - } -} static void uvWorkDoTask(uv_work_t* req) { // doing time-consumeing task @@ -1101,25 +1059,6 @@ static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { QUEUE_INIT(&pThrd->msg); - pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); - if (pThrd->prepare == NULL) { - tError("failed to init prepare"); - return TSDB_CODE_OUT_OF_MEMORY; - } - - code = uv_prepare_init(pThrd->loop, pThrd->prepare); - if (code != 0) { - tError("failed to init prepare since %s", uv_err_name(code)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - - code = uv_prepare_start(pThrd->prepare, uvPrepareCb); - if (code != 0) { - tError("failed to start prepare since %s", uv_err_name(code)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - pThrd->prepare->data = pThrd; - // conn set QUEUE_INIT(&pThrd->conn); @@ -1421,11 +1360,11 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - if (false == taosValidIpAndPort(srv->ip, srv->port)) { - code = TAOS_SYSTEM_ERROR(errno); - tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - goto End; - } + // if (false == taosValidIpAndPort(srv->ip, srv->port)) { + // code = TAOS_SYSTEM_ERROR(errno); + // tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + // goto End; + // } char pipeName[PATH_MAX]; #if defined(WINDOWS) || defined(DARWIN) @@ -1485,12 +1424,14 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, code = TSDB_CODE_OUT_OF_MEMORY; goto End; } + srv->pThreadObj[i] = thrd; thrd->pTransInst = shandle; thrd->quit = false; thrd->pTransInst = shandle; thrd->pWhiteList = uvWhiteListCreate(); if (thrd->pWhiteList == NULL) { + destroyWorkThrdObj(thrd); code = TSDB_CODE_OUT_OF_MEMORY; goto End; } @@ -1501,8 +1442,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - srv->pThreadObj[i] = thrd; - uv_os_sock_t fds[2]; if ((code = uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE)) != 0) { tError("failed to create pipe, errmsg: %s", uv_err_name(code)); @@ -1539,6 +1478,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, tError("failed to create worker-thread:%d", i); goto End; } + thrd->inited = 1; } #endif @@ -1560,6 +1500,12 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->inited = true; return srv; End: + for (int i = 0; i < srv->numOfThreads; i++) { + if (srv->pThreadObj[i] != NULL) { + SWorkThrd* thrd = srv->pThreadObj[i]; + destroyWorkThrd(thrd); + } + } transCloseServer(srv); terrno = code; return NULL; @@ -1663,20 +1609,27 @@ void uvHandleUpdate(SSvrMsg* msg, SWorkThrd* thrd) { taosMemoryFree(msg); } +void destroyWorkThrdObj(SWorkThrd* pThrd) { + if (pThrd == NULL) { + return; + } + transAsyncPoolDestroy(pThrd->asyncPool); + uvWhiteListDestroy(pThrd->pWhiteList); + uv_loop_close(pThrd->loop); + taosMemoryFree(pThrd->loop); + taosMemoryFree(pThrd); +} void destroyWorkThrd(SWorkThrd* pThrd) { if (pThrd == NULL) { return; } - (void)taosThreadJoin(pThrd->thread, NULL); - SRV_RELEASE_UV(pThrd->loop); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); - transAsyncPoolDestroy(pThrd->asyncPool); - - uvWhiteListDestroy(pThrd->pWhiteList); - - taosMemoryFree(pThrd->prepare); - taosMemoryFree(pThrd->loop); - taosMemoryFree(pThrd); + if (pThrd->inited) { + sendQuitToWorkThrd(pThrd); + (void)taosThreadJoin(pThrd->thread, NULL); + SRV_RELEASE_UV(pThrd->loop); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); + } + destroyWorkThrdObj(pThrd); } void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); @@ -1693,13 +1646,15 @@ void transCloseServer(void* arg) { tDebug("send quit msg to accept thread"); (void)uv_async_send(srv->pAcceptAsync); (void)taosThreadJoin(srv->thread, NULL); + SRV_RELEASE_UV(srv->loop); + (void)uv_loop_close(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); } } else { + SRV_RELEASE_UV(srv->loop); (void)uv_loop_close(srv->loop); } @@ -1708,7 +1663,9 @@ void transCloseServer(void* arg) { taosMemoryFree(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - taosMemoryFree(srv->pipe[i]); + if (srv->pipe[i] != NULL) { + taosMemoryFree(srv->pipe[i]); + } } taosMemoryFree(srv->pipe); From b6b0e266eb3033821a1655cadaafa26581124229 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 14:28:34 +0800 Subject: [PATCH 03/61] fix indirect leak --- source/libs/transport/src/transSvr.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 0727e49d8b..7e06767cb5 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1360,11 +1360,11 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - // if (false == taosValidIpAndPort(srv->ip, srv->port)) { - // code = TAOS_SYSTEM_ERROR(errno); - // tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - // goto End; - // } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + code = TAOS_SYSTEM_ERROR(errno); + tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + goto End; + } char pipeName[PATH_MAX]; #if defined(WINDOWS) || defined(DARWIN) From 0973fb6eed8a96bc0ec30dfa6e5b59fcfc35b6b3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 16:14:19 +0800 Subject: [PATCH 04/61] fix indirect leak --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5052d79ef3..85c8f7c21a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2387,7 +2387,7 @@ _end: } taosArrayDestroy(pThrd->timerList); - destroyConnPool(pThrd); + (void)destroyConnPool(pThrd); transDQDestroy(pThrd->delayQueue, NULL); transDQDestroy(pThrd->timeoutQueue, NULL); transDQDestroy(pThrd->waitConnQueue, NULL); @@ -2419,7 +2419,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } - uv_loop_close(pThrd->loop); + (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7e06767cb5..38c7edeeaf 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1615,7 +1615,7 @@ void destroyWorkThrdObj(SWorkThrd* pThrd) { } transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); - uv_loop_close(pThrd->loop); + (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } From 3b283b63507a64fddd30fa7a219e15e559f733ee Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 16:36:19 +0800 Subject: [PATCH 05/61] fix indirect leak --- source/libs/transport/src/transCli.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 85c8f7c21a..c2ce00c1cf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2383,6 +2383,7 @@ _end: transAsyncPoolDestroy(pThrd->asyncPool); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + (void)uv_timer_stop(timer); taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); @@ -2417,8 +2418,10 @@ static void destroyThrdObj(SCliThrd* pThrd) { tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + (void)uv_timer_stop(timer); taosMemoryFree(timer); } + (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); From 46a79c0be333809745ea9e69580bd5d71af0b934 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 17:50:35 +0800 Subject: [PATCH 06/61] fix indirect leak --- source/libs/transport/src/transCli.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c2ce00c1cf..803e50ce94 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -619,7 +619,11 @@ void* createConnPool(int size) { return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); } void* destroyConnPool(SCliThrd* pThrd) { - void* pool = pThrd->pool; + void* pool = pThrd->pool; + if (pool == NULL) { + return NULL; + } + SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conns)) { From 2875b0c753eb8b54491a89a894dcb75920e5ff44 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 10 Sep 2024 11:41:49 +0800 Subject: [PATCH 07/61] reduce mnode conn/app cache keep time to 3 seconds --- source/dnode/mnode/impl/src/mndProfile.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 3bdfa236d1..9eca3addd5 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -65,6 +65,8 @@ typedef struct { int64_t ipWhiteListVer; } SConnPreparedObj; +#define CACHE_OBJ_KEEP_TIME 3 // s + static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); @@ -161,9 +163,8 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType tstrncpy(connObj.user, user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); - int32_t keepTime = tsShellActivityTimer * 3; SConnObj *pConn = - taosCachePut(pMgmt->connCache, &connId, sizeof(uint32_t), &connObj, sizeof(connObj), keepTime * 1000); + taosCachePut(pMgmt->connCache, &connId, sizeof(uint32_t), &connObj, sizeof(connObj), CACHE_OBJ_KEEP_TIME * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr()); @@ -376,8 +377,7 @@ static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq *pReq) (void)memcpy(&app.summary, &pReq->summary, sizeof(pReq->summary)); app.lastAccessTimeMs = taosGetTimestampMs(); - const int32_t keepTime = tsShellActivityTimer * 3; - SAppObj *pApp = taosCachePut(pMgmt->appCache, &pReq->appId, sizeof(pReq->appId), &app, sizeof(app), keepTime * 1000); + SAppObj *pApp = taosCachePut(pMgmt->appCache, &pReq->appId, sizeof(pReq->appId), &app, sizeof(app), CACHE_OBJ_KEEP_TIME * 1000); if (pApp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to app %" PRIx64 " into cache since %s", pReq->appId, terrstr()); @@ -842,7 +842,6 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl int32_t numOfRows = 0; int32_t cols = 0; SConnObj *pConn = NULL; - int32_t keepTime = tsShellActivityTimer * 3; if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -856,7 +855,7 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl break; } - if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > ((int64_t)keepTime * 1000)) { + if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > ((int64_t)CACHE_OBJ_KEEP_TIME * 1000)) { continue; } From 2fa083c82feb626f4ec261d50f73b8ed0f17192a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 10 Sep 2024 19:03:27 +0800 Subject: [PATCH 08/61] opt transport --- source/libs/transport/src/transCli.c | 1 - source/libs/transport/src/transSvr.c | 16 +++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 803e50ce94..a0c7315fe7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2426,7 +2426,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(timer); } - (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 38c7edeeaf..3a597fe6ad 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -188,11 +188,13 @@ static void sendQuitToWorkThrd(SWorkThrd* pThrd); static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); -#define SRV_RELEASE_UV(loop) \ - do { \ - (void)uv_walk(loop, uvWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ +#define SRV_RELEASE_UV(loop) \ + do { \ + if (loop && uv_loop_alive(loop)) { \ + (void)uv_walk(loop, uvWalkCb, NULL); \ + (void)uv_run(loop, UV_RUN_DEFAULT); \ + (void)uv_loop_close(loop); \ + } \ } while (0); #define ASYNC_ERR_JRET(thrd) \ @@ -1615,7 +1617,6 @@ void destroyWorkThrdObj(SWorkThrd* pThrd) { } transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); - (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } @@ -1648,14 +1649,11 @@ void transCloseServer(void* arg) { (void)taosThreadJoin(srv->thread, NULL); SRV_RELEASE_UV(srv->loop); - (void)uv_loop_close(srv->loop); - for (int i = 0; i < srv->numOfThreads; i++) { destroyWorkThrd(srv->pThreadObj[i]); } } else { SRV_RELEASE_UV(srv->loop); - (void)uv_loop_close(srv->loop); } taosMemoryFree(srv->pThreadObj); From 25c0b928854a9ee958e0fb83d2bc82fddf796b1b Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 01:11:56 +0000 Subject: [PATCH 09/61] feat:Optimising the duration and keep --- include/libs/command/command.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 20 +++++--- source/libs/command/src/command.c | 52 ++++++--------------- tests/script/tsim/db/create_all_options.sim | 4 +- 4 files changed, 32 insertions(+), 45 deletions(-) diff --git a/include/libs/command/command.h b/include/libs/command/command.h index b788b03386..284f54e5ae 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -29,5 +29,6 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp); int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp); void qExplainFreeCtx(SExplainCtx *pCtx); +int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes); #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 20c5d09c9e..9c02fd6e5d 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -34,6 +34,7 @@ #include "systable.h" #include "thttp.h" #include "tjson.h" +#include "command.h" #define DB_VER_NUMBER 1 #define DB_RESERVE_SIZE 27 @@ -2321,18 +2322,25 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, (void)colDataSetVal(pColInfo, rows, (const char *)strictVstr, false); char durationVstr[128] = {0}; - int32_t len = sprintf(&durationVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.daysPerFile); + int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], pDb->cfg.daysPerFile); + varDataSetLen(durationVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); (void)colDataSetVal(pColInfo, rows, (const char *)durationVstr, false); - char keepVstr[128] = {0}; + char keepVstr[512] = {0}; + char keep0Str[128] = {0}; + char keep1Str[128] = {0}; + char keep2Str[128] = {0}; + + formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0); + formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1); + formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2); + if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, - pDb->cfg.daysToKeep0); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); } else { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, - pDb->cfg.daysToKeep2); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep0Str, keep1Str, keep2Str); } varDataSetLen(keepVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 138782495f..0e5f2f4cbd 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -344,27 +344,18 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { return TSDB_CACHE_MODEL_NONE_STR; } -static int32_t formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { +int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes) { int len = 0; if (timeInMinutes % 1440 == 0) { int days = timeInMinutes / 1440; - len = snprintf(NULL, 0, "%dd", days); - *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); - if(*buffer == NULL) return terrno; - sprintf(*buffer, "%dd", days); + len = sprintf(buffer, "%dd", days); } else if (timeInMinutes % 60 == 0) { int hours = timeInMinutes / 60; - len = snprintf(NULL, 0, "%dh", hours); - *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); - if(*buffer == NULL) return terrno; - sprintf(*buffer, "%dh", hours); + len = sprintf(buffer, "%dh", hours); } else { - len = snprintf(NULL, 0, "%dm", timeInMinutes); - *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); - if(*buffer == NULL) return terrno; - sprintf(*buffer, "%dm", timeInMinutes); + len = sprintf(buffer, "%dm", timeInMinutes); } - return TSDB_CODE_SUCCESS; + return len; } static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { @@ -404,25 +395,16 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, } else if (pCfg->hashPrefix < 0) { hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; } - char* durationStr = NULL; - char* keep0Str = NULL; - char* keep1Str = NULL; - char* keep2Str = NULL; - int32_t codeDuration = formatDurationOrKeep(&durationStr, pCfg->daysPerFile); - int32_t codeKeep0 = formatDurationOrKeep(&keep0Str, pCfg->daysToKeep0); - int32_t codeKeep1 = formatDurationOrKeep(&keep1Str, pCfg->daysToKeep1); - int32_t codeKeep2 = formatDurationOrKeep(&keep2Str, pCfg->daysToKeep2); - if(codeDuration != TSDB_CODE_SUCCESS || codeKeep0 != TSDB_CODE_SUCCESS || codeKeep1 != TSDB_CODE_SUCCESS || codeKeep2 != TSDB_CODE_SUCCESS) { - int32_t firstErrorCode = codeDuration != TSDB_CODE_SUCCESS ? codeDuration : - codeKeep0 != TSDB_CODE_SUCCESS ? codeKeep0 : - codeKeep1 != TSDB_CODE_SUCCESS ? codeKeep1 : codeKeep2; - taosMemoryFree(pRetentions); - taosMemoryFree(durationStr); - taosMemoryFree(keep0Str); - taosMemoryFree(keep1Str); - taosMemoryFree(keep2Str); - return firstErrorCode; - } + char durationStr[128] = {0}; + char keep0Str[128] = {0}; + char keep1Str[128] = {0}; + char keep2Str[128] = {0}; + + int32_t lenDuration = formatDurationOrKeep(durationStr, pCfg->daysPerFile); + int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pCfg->daysToKeep0); + int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pCfg->daysToKeep1); + int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pCfg->daysToKeep2); + if (IS_SYS_DBNAME(dbName)) { len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); } else { @@ -449,10 +431,6 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, } taosMemoryFree(pRetentions); - taosMemoryFree(durationStr); - taosMemoryFree(keep0Str); - taosMemoryFree(keep1Str); - taosMemoryFree(keep2Str); (varDataLen(buf2)) = len; diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index e4f29cc74e..e402223d93 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -92,10 +92,10 @@ endi if $data5_db != on then # strict return -1 endi -if $data6_db != 14400m then # duration +if $data6_db != 10d then # duration return -1 endi -if $data7_db != 5256000m,5256000m,5256000m then # keep +if $data7_db != 3650d,3650d,3650d then # keep return -1 endi if $data8_db != 256 then # buffer From fede52d5b7743ec553d10eaac72af5d694f6568d Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 10 Sep 2024 17:48:47 +0800 Subject: [PATCH 10/61] enh: (last) rm rwritebatch --- source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/tsdb/tsdbCache.c | 31 +++++++++++-------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 59d70889e5..c85c376c47 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -342,7 +342,6 @@ typedef struct { rocksdb_writeoptions_t *writeoptions; rocksdb_readoptions_t *readoptions; rocksdb_writebatch_t *writebatch; - rocksdb_writebatch_t *rwritebatch; STSchema *pTSchema; } SRocksCache; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3a3e20612e..f4a423bfa7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -214,10 +214,8 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { } rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); - rocksdb_writebatch_t *rwritebatch = rocksdb_writebatch_create(); pTsdb->rCache.writebatch = writebatch; - pTsdb->rCache.rwritebatch = rwritebatch; pTsdb->rCache.my_comparator = cmp; pTsdb->rCache.options = options; pTsdb->rCache.writeoptions = writeoptions; @@ -248,7 +246,6 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { rocksdb_close(pTsdb->rCache.db); rocksdb_flushoptions_destroy(pTsdb->rCache.flushoptions); rocksdb_writebatch_destroy(pTsdb->rCache.writebatch); - rocksdb_writebatch_destroy(pTsdb->rCache.rwritebatch); rocksdb_readoptions_destroy(pTsdb->rCache.readoptions); rocksdb_writeoptions_destroy(pTsdb->rCache.writeoptions); rocksdb_options_destroy(pTsdb->rCache.options); @@ -258,8 +255,8 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { taosMemoryFree(pTsdb->rCache.pTSchema); } -static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) { - rocksdb_writebatch_t *wb = read ? pTsdb->rCache.rwritebatch : pTsdb->rCache.writebatch; +static void rocksMayWrite(STsdb *pTsdb, bool force) { + rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; int count = rocksdb_writebatch_count(wb); if ((force && count > 0) || count >= ROCKS_BATCH_SIZE) { @@ -516,8 +513,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); - rocksMayWrite(pTsdb, true, false); - rocksMayWrite(pTsdb, true, true); + rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -667,8 +663,7 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); - rocksMayWrite(pTsdb, true, false); - rocksMayWrite(pTsdb, true, true); + rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); if (NULL != err) { @@ -862,7 +857,7 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra taosMemoryFree(pTSchema); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -903,7 +898,7 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { taosMemoryFree(pTSchema); - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -934,7 +929,7 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -973,7 +968,7 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1243,7 +1238,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosMemoryFreeClear(pToFree); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1582,7 +1577,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } // store result back to rocks cache - wb = pTsdb->rCache.rwritebatch; + wb = pTsdb->rCache.writebatch; char *value = NULL; size_t vlen = 0; code = tsdbCacheSerialize(pLastCol, &value, &vlen); @@ -1597,7 +1592,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } if (wb) { - rocksMayWrite(pTsdb, false, true); + rocksMayWrite(pTsdb, false); } _exit: @@ -1797,6 +1792,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache if (remainCols && TARRAY_SIZE(remainCols) > 0) { (void)taosThreadMutexLock(&pTsdb->lruMutex); + rocksMayWrite(pTsdb, false); + for (int i = 0; i < TARRAY_SIZE(remainCols);) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; LRUHandle *h = taosLRUCacheLookup(pCache, &idxKey->key, ROCKS_KEY_LEN); @@ -1944,7 +1941,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE taosMemoryFreeClear(pLastCol); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); _exit: (void)taosThreadMutexUnlock(&pTsdb->lruMutex); From cffa2a496ced1d33c425f36ea5a7577b101cbca9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 09:37:09 +0800 Subject: [PATCH 11/61] fix mem leak --- source/libs/transport/src/transSvr.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3a597fe6ad..48042b84c1 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -188,13 +188,11 @@ static void sendQuitToWorkThrd(SWorkThrd* pThrd); static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); -#define SRV_RELEASE_UV(loop) \ - do { \ - if (loop && uv_loop_alive(loop)) { \ - (void)uv_walk(loop, uvWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ - } \ +#define SRV_RELEASE_UV(loop) \ + do { \ + (void)uv_walk(loop, uvWalkCb, NULL); \ + (void)uv_run(loop, UV_RUN_DEFAULT); \ + (void)uv_loop_close(loop); \ } while (0); #define ASYNC_ERR_JRET(thrd) \ From abbcc4afa087dec2984e5c5b90987ddc81d4b472 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 11 Sep 2024 03:24:59 +0000 Subject: [PATCH 12/61] fix/TS-5404-use-temp-transid-when-timeout --- source/dnode/mnode/impl/src/mndTrans.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 63cab4168a..18ceddb7d0 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -64,6 +64,8 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); +int32_t tsMaxTransId = 0; + int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_TRANS, @@ -602,7 +604,9 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, tstrncpy(pTrans->opername, opername, TSDB_TRANS_OPER_LEN); } - pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); + int32_t sdbMaxId = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); + int32_t oldId = atomic_load_32(&tsMaxTransId); + pTrans->id = TMAX(sdbMaxId, oldId); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->conflict = conflict; @@ -1027,6 +1031,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); + atomic_store_32(&tsMaxTransId, pTrans->id); TAOS_RETURN(code); } mInfo("trans:%d, prepare finished", pTrans->id); From 44a955218cfcedb9b9445e058fcb3a87e4177c95 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 13:48:23 +0800 Subject: [PATCH 13/61] fix: (last) call rocksdb_write before rocksdb_multi_get --- source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/tsdb/tsdbCache.c | 111 +++++++++--------------- 2 files changed, 40 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c85c376c47..d4a33a5478 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -362,7 +362,6 @@ struct STsdb { SMemTable *imem; STsdbFS fs; // old SLRUCache *lruCache; - SCacheFlushState flushState; TdThreadMutex lruMutex; SLRUCache *biCache; TdThreadMutex biMutex; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f4a423bfa7..53e20d459f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -267,7 +267,6 @@ static void rocksMayWrite(STsdb *pTsdb, bool force) { tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, count, err); rocksdb_free(err); - // pTsdb->flushState.flush_count = 0; } rocksdb_writebatch_clear(wb); @@ -456,47 +455,23 @@ static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size TAOS_RETURN(TSDB_CODE_SUCCESS); } -static void tsdbCachePutBatch(SLastCol *pLastCol, const void *key, size_t klen, SCacheFlushState *state) { - int32_t code = 0; - STsdb *pTsdb = state->pTsdb; - SRocksCache *rCache = &pTsdb->rCache; - rocksdb_writebatch_t *wb = rCache->writebatch; - char *rocks_value = NULL; - size_t vlen = 0; - - code = tsdbCacheSerialize(pLastCol, &rocks_value, &vlen); - if (code) { - tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); - return; - } - - rocksdb_writebatch_put(wb, (char *)key, klen, rocks_value, vlen); - - taosMemoryFree(rocks_value); - - if (++state->flush_count >= ROCKS_BATCH_SIZE) { - char *err = NULL; - - rocksdb_write(rCache->db, rCache->writeoptions, wb, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - state->flush_count, err); - rocksdb_free(err); - } - - rocksdb_writebatch_clear(wb); - - state->flush_count = 0; - } -} +static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol); int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + STsdb *pTsdb = (STsdb *)ud; + + int32_t code = tsdbCachePutToRocksdb(pTsdb, (SLastKey *)key, pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, flush dirty lru failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; + } pLastCol->dirty = 0; + + rocksMayWrite(pTsdb, false); } return 0; @@ -511,7 +486,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { (void)taosThreadMutexLock(&pTsdb->lruMutex); - taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); + taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); @@ -602,7 +577,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + tsdbCacheFlushDirty(key, klen, pLastCol, ud); } for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { @@ -639,11 +614,11 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i SLastKey *pLastKey = &(SLastKey){.lflag = lflag, .uid = uid, .cid = cid}; LRUStatus status = taosLRUCacheInsert(pCache, pLastKey, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + TAOS_LRU_PRIORITY_LOW, pTsdb); if (status != TAOS_LRU_STATUS_OK) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); code = TSDB_CODE_FAILED; + pLastCol = NULL; } _exit: @@ -661,7 +636,7 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->lruCache; rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; - taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); + taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); @@ -733,6 +708,10 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, char **values_list = NULL; size_t *values_list_sizes = NULL; + + // was written by caller + // rocksMayWrite(pTsdb, true); // flush writebatch cache + TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, 2, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), NULL, _exit); @@ -857,7 +836,7 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra taosMemoryFree(pTSchema); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -898,7 +877,7 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { taosMemoryFree(pTSchema); - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -929,7 +908,7 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -968,7 +947,7 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1078,11 +1057,11 @@ static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLa TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(pLRULastCol, &charge)); LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, pLastKey, ROCKS_KEY_LEN, pLRULastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLRULastCol); code = TSDB_CODE_FAILED; + pLRULastCol = NULL; } _exit: @@ -1172,6 +1151,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray keys_list_sizes[i] = ROCKS_KEY_LEN; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + code = tsdbCacheGetValuesFromRocks(pTsdb, num_keys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes); if (code) { @@ -1238,7 +1219,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosMemoryFreeClear(pToFree); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1568,32 +1549,22 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); - taosMemoryFree(pLastCol); + pLastCol = NULL; TAOS_CHECK_EXIT(TSDB_CODE_FAILED); } // store result back to rocks cache - wb = pTsdb->rCache.writebatch; - char *value = NULL; - size_t vlen = 0; - code = tsdbCacheSerialize(pLastCol, &value, &vlen); + code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, pLastCol); if (code) { - tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); - } else { - SLastKey *key = &idxKey->key; - size_t klen = ROCKS_KEY_LEN; - rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); - taosMemoryFree(value); + tsdbError("vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + TAOS_CHECK_EXIT(code); } } - if (wb) { - rocksMayWrite(pTsdb, false); - } + rocksMayWrite(pTsdb, false); _exit: taosArrayDestroy(lastrowTmpIndexArray); @@ -1633,6 +1604,8 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA keys_list_sizes[i] = ROCKS_KEY_LEN; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + code = tsdbCacheGetValuesFromRocks(pTsdb, num_keys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes); if (code) { @@ -1681,11 +1654,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA } LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); - taosMemoryFreeClear(pLastCol); taosMemoryFreeClear(pToFree); TAOS_CHECK_EXIT(TSDB_CODE_FAILED); } @@ -1792,7 +1763,6 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache if (remainCols && TARRAY_SIZE(remainCols) > 0) { (void)taosThreadMutexLock(&pTsdb->lruMutex); - rocksMayWrite(pTsdb, false); for (int i = 0; i < TARRAY_SIZE(remainCols);) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; @@ -1907,6 +1877,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE keys_list_sizes[i] = klen; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, numKeys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), NULL, _exit); @@ -1941,7 +1913,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE taosMemoryFreeClear(pLastCol); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); _exit: (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1983,9 +1955,6 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { (void)taosThreadMutexInit(&pTsdb->lruMutex, NULL); - pTsdb->flushState.pTsdb = pTsdb; - pTsdb->flushState.flush_count = 0; - _err: if (code) { tsdbError("tsdb/cache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); From 64bb49e0dbfd2a3087ccdfe54fb28e4af6528e71 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Mon, 9 Sep 2024 18:44:16 +0800 Subject: [PATCH 14/61] fix:[TS-5839] Fix crash when use last function on a column with all null value with partition subclause. --- include/common/tcommon.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 65 +++++++++++++++++++------ tests/script/tsim/query/cache_last.sim | 2 +- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5714990dd5..753f84d774 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -110,6 +110,8 @@ typedef struct SFirstLastRes { int32_t pkBytes; int8_t pkType; STuplePos pos; + STuplePos nullTuplePos; + bool nullTupleSaved; char buf[]; } SFirstLastRes; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 16f31321ce..41e2cadace 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -152,6 +152,7 @@ int32_t getIrateInfoSize(int32_t pkBytes); int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t firstFunction(SqlFunctionCtx* pCtx); int32_t firstFunctionMerge(SqlFunctionCtx* pCtx); int32_t lastFunction(SqlFunctionCtx* pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 3ec63a6d34..976d15e7d8 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -3469,7 +3469,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLast, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, - .initFunc = functionSetup, + .initFunc = firstLastFunctionSetup, .processFunc = lastFunction, .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6d655528c7..a84e8b66ad 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2617,6 +2617,22 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } +int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (pResInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; + } + + SFirstLastRes * pRes = GET_ROWCELL_INTERBUF(pResInfo); + SInputColumnInfoData* pInput = &pCtx->input; + + pRes->nullTupleSaved = false; + pRes->nullTuplePos.pageId = -1; + return TSDB_CODE_SUCCESS; +} + static int32_t prepareBuf(SqlFunctionCtx* pCtx) { if (pCtx->subsidiaries.rowLen == 0) { int32_t rowLen = 0; @@ -2635,7 +2651,7 @@ static int32_t prepareBuf(SqlFunctionCtx* pCtx) { } static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, - SFirstLastRes* pInfo) { + SFirstLastRes* pInfo, bool noElements) { int32_t code = TSDB_CODE_SUCCESS; if (pCtx->subsidiaries.num <= 0) { @@ -2643,7 +2659,7 @@ static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowI } if (!pInfo->hasResult) { - code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + code = saveTupleData(pCtx, rowIndex, pSrcBlock, noElements ? &pInfo->nullTuplePos : &pInfo->pos); } else { code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); } @@ -2669,7 +2685,7 @@ static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t } pInfo->ts = currentTs; - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo, false); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2708,10 +2724,11 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; return TSDB_CODE_SUCCESS; } @@ -2802,10 +2819,11 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (numOfElems == 0) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2841,10 +2859,11 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; return TSDB_CODE_SUCCESS; } @@ -2983,10 +3002,11 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { // save selectivity value for column consisted of all null values if (numOfElems == 0) { - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; } return TSDB_CODE_SUCCESS; @@ -3031,7 +3051,7 @@ static bool firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOut static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, int32_t rowIndex) { if (firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput, pOutput->nullTupleSaved); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -3079,6 +3099,14 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer } } + if (numOfElems == 0) { + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pInfo->nullTupleSaved = true; + } + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -3099,6 +3127,11 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + + if (pResInfo->isNullRes) { + colDataSetNULL(pCol, pBlock->info.rows); + return setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); + } code = colDataSetVal(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); if (TSDB_CODE_SUCCESS != code) { return code; @@ -3134,12 +3167,16 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return TSDB_CODE_OUT_OF_RANGE; } - code = colDataSetVal(pCol, pBlock->info.rows, res, false); - if (TSDB_CODE_SUCCESS != code) { - return TSDB_CODE_OUT_OF_MEMORY; + if (pEntryInfo->numOfRes == 0) { + colDataSetNULL(pCol, pBlock->info.rows); + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); + } else { + code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + return TSDB_CODE_OUT_OF_MEMORY; + } + code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); } - code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); - taosMemoryFree(res); return code; } @@ -3185,7 +3222,7 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex pInfo->pkData = pInfo->buf + pInfo->bytes; } pInfo->ts = cts; - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo, false); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/tests/script/tsim/query/cache_last.sim b/tests/script/tsim/query/cache_last.sim index a06e6e1e6f..0a30bbd325 100644 --- a/tests/script/tsim/query/cache_last.sim +++ b/tests/script/tsim/query/cache_last.sim @@ -89,7 +89,7 @@ if $data10 != @ -> Merge (columns=3 width=24 input_order=unknown output_order= return -1 endi sql explain select count(*), last_row(f1), min(f1) from sta interval(1s); -if $data10 != @ -> Merge (columns=4 width=82 input_order=asc output_order=asc mode=sort)@ then +if $data10 != @ -> Merge (columns=4 width=106 input_order=asc output_order=asc mode=sort)@ then return -1 endi sql explain select distinct count(*), last_row(f1), min(f1) from tba1; From f9126f121187ddc3fabfbd1e876106ad4f72a492 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 14:11:51 +0800 Subject: [PATCH 15/61] enh:modify error code passing --- source/libs/sync/src/syncPipeline.c | 2 +- source/libs/sync/src/syncRespMgr.c | 2 +- source/libs/sync/src/syncSnapshot.c | 10 +++--- source/libs/tfs/src/tfs.c | 4 +-- source/libs/tfs/src/tfsDisk.c | 2 +- source/libs/transport/src/thttp.c | 4 +-- source/libs/transport/src/transCli.c | 52 +++++++++++++-------------- source/libs/transport/src/transComm.c | 16 ++++----- source/libs/transport/src/transSvr.c | 16 ++++----- source/util/src/talgo.c | 6 ++-- source/util/src/tbase58.c | 8 ++--- source/util/src/tbloomfilter.c | 6 ++-- source/util/src/tconfig.c | 4 +-- source/util/src/tgeosctx.c | 2 +- source/util/src/tjson.c | 2 +- source/util/src/tlist.c | 2 +- source/util/src/tlrucache.c | 2 +- source/util/src/tqueue.c | 8 ++--- source/util/src/tref.c | 6 ++-- source/util/src/tscalablebf.c | 4 +-- source/util/src/tworker.c | 10 +++--- 21 files changed, 84 insertions(+), 84 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index a6e9c7de32..86d4d83d29 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -1279,7 +1279,7 @@ int32_t syncLogBufferCreate(SSyncLogBuffer** ppBuf) { int32_t code = 0; SSyncLogBuffer* pBuf = taosMemoryCalloc(1, sizeof(SSyncLogBuffer)); if (pBuf == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exit); + TAOS_CHECK_GOTO(terrno, NULL, _exit); } pBuf->size = sizeof(pBuf->entries) / sizeof(pBuf->entries[0]); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index aa7a6da0a2..854d6bc314 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -25,7 +25,7 @@ int32_t syncRespMgrCreate(void *data, int64_t ttl, SSyncRespMgr **ppObj) { *ppObj = NULL; if ((pObj = taosMemoryCalloc(1, sizeof(SSyncRespMgr))) == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pObj->pRespHash = diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 2a3b164f03..0b77ab78a6 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -54,7 +54,7 @@ static int32_t syncSnapBufferCreate(SSyncSnapBuffer **ppBuf) { SSyncSnapBuffer *pBuf = taosMemoryCalloc(1, sizeof(SSyncSnapBuffer)); if (pBuf == NULL) { *ppBuf = NULL; - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pBuf->size = sizeof(pBuf->entries) / sizeof(void *); if (pBuf->size != TSDB_SYNC_SNAP_BUFFER_SIZE) return TSDB_CODE_SYN_INTERNAL_ERROR; @@ -74,7 +74,7 @@ int32_t snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex, SSyncSn SSyncSnapshotSender *pSender = taosMemoryCalloc(1, sizeof(SSyncSnapshotSender)); if (pSender == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pSender->start = false; @@ -285,7 +285,7 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (pSender->seq > SYNC_SNAPSHOT_SEQ_BEGIN) { pBlk = taosMemoryCalloc(1, sizeof(SyncSnapBlock)); if (pBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OUT; } @@ -422,7 +422,7 @@ int32_t snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId, SSyncSnapsh SSyncSnapshotReceiver *pReceiver = taosMemoryCalloc(1, sizeof(SSyncSnapshotReceiver)); if (pReceiver == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pReceiver->start = false; @@ -697,7 +697,7 @@ static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshot // copy snap info from leader void *data = taosMemoryCalloc(1, pMsg->dataLen); if (data == NULL) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } pInfo->data = data; data = NULL; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 4954e81837..9727256a58 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -36,7 +36,7 @@ int32_t tfsOpen(SDiskCfg *pCfg, int32_t ndisk, STfs **ppTfs) { pTfs = taosMemoryCalloc(1, sizeof(STfs)); if (pTfs == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (taosThreadSpinInit(&pTfs->lock, 0) != 0) { @@ -429,7 +429,7 @@ int32_t tfsOpendir(STfs *pTfs, const char *rname, STfsDir **ppDir) { int32_t code = 0; STfsDir *pDir = taosMemoryCalloc(1, sizeof(STfsDir)); if (pDir == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exit); + TAOS_CHECK_GOTO(terrno, NULL, _exit); } SDiskID diskId = {.id = 0, .level = 0}; diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 07fa8d79b9..dfdf1262c9 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -22,7 +22,7 @@ int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path, STfsDisk *pDisk = NULL; if ((pDisk = taosMemoryCalloc(1, sizeof(STfsDisk))) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if ((pDisk->path = taosStrdup(path)) == NULL) { diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index d19877dbf1..e0e2a0c323 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -446,7 +446,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { int32_t httpSendQuit(SHttpModule* http, int64_t chanId) { SHttpMsg* msg = taosMemoryCalloc(1, sizeof(SHttpMsg)); if (msg == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } msg->seq = atomic_fetch_add_64(&httpSeqNum, 1); msg->quit = 1; @@ -744,7 +744,7 @@ int64_t transInitHttpChanImpl() { int32_t code = 0; SHttpModule* http = taosMemoryCalloc(1, sizeof(SHttpModule)); if (http == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _ERROR; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 78320c450c..4dd3346f8b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -874,7 +874,7 @@ static int32_t allocConnRef(SCliConn* conn, bool update) { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); if (exh == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } exh->refId = transAddExHandle(transGetRefMgt(), exh); @@ -974,7 +974,7 @@ static int32_t cliCreateConn(SCliThrd* pThrd, SCliConn** pCliConn) { int32_t code = 0; SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); if (conn == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } // read/write stream handle @@ -996,7 +996,7 @@ static int32_t cliCreateConn(SCliThrd* pThrd, SCliConn** pCliConn) { if (timer == NULL) { timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); if (timer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _failed); + TAOS_CHECK_GOTO(terrno, NULL, _failed); } tDebug("no available timer, create a timer %p", timer); @@ -1951,7 +1951,7 @@ static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, SCliBatchList* pBatchList = taosMemoryCalloc(1, sizeof(SCliBatchList)); if (pBatchList == NULL) { tError("failed to create batch list, reason:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } QUEUE_INIT(&pBatchList->wq); pBatchList->port = port; @@ -1991,7 +1991,7 @@ static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliMsg* p SCliBatch* pBatch = taosMemoryCalloc(1, sizeof(SCliBatch)); if (pBatch == NULL) { tError("failed to create batch, reason:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } QUEUE_INIT(&pBatch->wq); @@ -2315,7 +2315,7 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { SCliThrd* pThrd = (SCliThrd*)taosMemoryCalloc(1, sizeof(SCliThrd)); if (pThrd == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + TAOS_CHECK_GOTO(terrno, NULL, _end); } QUEUE_INIT(&pThrd->msg); @@ -2323,7 +2323,7 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (pThrd->loop == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + TAOS_CHECK_GOTO(terrno, NULL, _end); } code = uv_loop_init(pThrd->loop); @@ -2362,7 +2362,7 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { for (int i = 0; i < timerSize; i++) { uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); if (timer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + TAOS_CHECK_GOTO(terrno, NULL, _end); } (void)uv_timer_init(pThrd->loop, timer); if (taosArrayPush(pThrd->timerList, &timer) == NULL) { @@ -2486,7 +2486,7 @@ int32_t cliSendQuit(SCliThrd* thrd) { int32_t code = 0; SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (msg == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } msg->type = Quit; @@ -2894,7 +2894,7 @@ int32_t transReleaseCliHandle(void* handle) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCtx->ahandle = tmsg.info.ahandle; @@ -2902,7 +2902,7 @@ int32_t transReleaseCliHandle(void* handle) { SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cmsg == NULL) { taosMemoryFree(pCtx); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } cmsg->msg = tmsg; cmsg->st = taosGetTimestampUs(); @@ -2923,7 +2923,7 @@ static int32_t transInitMsg(void* shandle, const SEpSet* pEpSet, STransMsg* pReq if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } epsetAssign(&pCtx->epSet, pEpSet); @@ -2937,7 +2937,7 @@ static int32_t transInitMsg(void* shandle, const SEpSet* pEpSet, STransMsg* pReq SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cliMsg == NULL) { taosMemoryFree(pCtx); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } cliMsg->ctx = pCtx; @@ -3071,7 +3071,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra STransMsg* pTransRsp = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransRsp == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); @@ -3081,7 +3081,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); if (sem == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } code = tsem_init(sem, 0, 0); @@ -3096,7 +3096,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra if (pCtx == NULL) { (void)tsem_destroy(sem); taosMemoryFree(sem); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } epsetAssign(&pCtx->epSet, pEpSet); @@ -3111,7 +3111,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra (void)tsem_destroy(sem); taosMemoryFree(sem); taosMemoryFree(pCtx); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } cliMsg->ctx = pCtx; @@ -3150,7 +3150,7 @@ int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { int32_t code = 0; tsem2_t* sem = taosMemoryCalloc(1, sizeof(tsem2_t)); if (sem == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(sem, 0, 0) != 0) { @@ -3159,7 +3159,7 @@ int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { STransSyncMsg* pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg)); if (pSyncMsg == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _EXIT); + TAOS_CHECK_GOTO(terrno, NULL, _EXIT); } taosInitRWLatch(&pSyncMsg->latch); @@ -3194,7 +3194,7 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransMsg == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN2); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); } SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); @@ -3206,7 +3206,7 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN2); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); } epsetAssign(&pCtx->epSet, pEpSet); @@ -3229,7 +3229,7 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cliMsg == NULL) { taosMemoryFree(pCtx); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN2); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); } cliMsg->ctx = pCtx; @@ -3294,7 +3294,7 @@ int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { for (int8_t i = 0; i < pTransInst->numOfThreads; i++) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } @@ -3303,7 +3303,7 @@ int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cliMsg == NULL) { taosMemoryFree(pCtx); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } @@ -3330,7 +3330,7 @@ int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { int32_t transAllocHandle(int64_t* refId) { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); if (exh == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } exh->refId = transAddExHandle(transGetRefMgt(), exh); @@ -3369,7 +3369,7 @@ int32_t transFreeConnById(void* shandle, int64_t transpointId) { SCliMsg* pCli = taosMemoryCalloc(1, sizeof(SCliMsg)); if (pCli == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + TAOS_CHECK_GOTO(terrno, NULL, _exception); } pCli->type = FreeById; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 1329489be6..05244dbce2 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -70,7 +70,7 @@ int32_t transDecompressMsg(char** msg, int32_t len) { char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STransMsgHead* pNewHead = (STransMsgHead*)buf; @@ -106,7 +106,7 @@ int transSockInfo2Str(struct sockaddr* sockname, char* dst) { int32_t transInitBuffer(SConnBuffer* buf) { buf->buf = taosMemoryCalloc(1, BUFFER_CAP); if (buf->buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } buf->cap = BUFFER_CAP; @@ -149,7 +149,7 @@ int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); if (*buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memcpy(*buf, p->buf, total); if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { @@ -249,7 +249,7 @@ int32_t transSetConnOption(uv_tcp_t* stream, int keepalive) { int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAsyncPool** pPool) { SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); if (pool == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; // return NULL; } int32_t code = 0; @@ -258,7 +258,7 @@ int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAs pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); if (pool->asyncs == NULL) { taosMemoryFree(pool); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int i = 0, err = 0; @@ -267,7 +267,7 @@ int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAs SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); if (item == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } item->pThrd = arg; @@ -560,7 +560,7 @@ int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue) { timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); if (timer == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } heap = heapCreate(timeCompare); @@ -852,7 +852,7 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { int32_t len = 0; char* pBuf = taosMemoryCalloc(1, pList->num * 36); if (pBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int i = 0; i < pList->num; i++) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 11aa468b19..f68561fedf 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -260,7 +260,7 @@ int32_t uvWhiteListToStr(SWhiteUserList* plist, char* user, char** ppBuf) { char* pBuf = taosMemoryCalloc(1, tlen + 64); if (pBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t len = sprintf(pBuf, "user: %s, ver: %" PRId64 ", ip: {%s}", user, plist->ver, tmp); @@ -299,7 +299,7 @@ int32_t uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* plist, if (ppUserList == NULL || *ppUserList == NULL) { SWhiteUserList* pUserList = taosMemoryCalloc(1, sizeof(SWhiteUserList)); if (pUserList == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pUserList->ver = ver; @@ -1104,7 +1104,7 @@ static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); if (pThrd->prepare == NULL) { tError("failed to init prepare"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } code = uv_prepare_init(pThrd->loop, pThrd->prepare); @@ -1155,7 +1155,7 @@ static int32_t addHandleToAcceptloop(void* arg) { srv->pAcceptAsync = taosMemoryCalloc(1, sizeof(uv_async_t)); if (srv->pAcceptAsync == NULL) { tError("failed to create async since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } code = uv_async_init(srv->loop, srv->pAcceptAsync, uvAcceptAsyncCb); @@ -1749,7 +1749,7 @@ int32_t transReleaseSrvHandle(void* handle) { SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (m == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return1; } @@ -1799,7 +1799,7 @@ int32_t transSendResponse(const STransMsg* msg) { SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (m == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return1; } @@ -1844,7 +1844,7 @@ int32_t transRegisterMsg(const STransMsg* msg) { SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (m == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return1; } @@ -1888,7 +1888,7 @@ int32_t transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (msg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 9c241516ca..f1ee40eccf 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -148,7 +148,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer if (NULL == buf) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); taosMemoryFreeClear(buf); @@ -393,7 +393,7 @@ int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompa char *buf = taosMemoryCalloc(1, size); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (base && size > 0) { @@ -454,7 +454,7 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c const int32_t THRESHOLD_SIZE = 6; char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t start = 0; start < numOfElem - 1; start += THRESHOLD_SIZE) { diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index b5d873f5ea..84ca86c084 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -43,7 +43,7 @@ int32_t base58_encode(const uint8_t *value, int32_t vlen, char **result) { size = (pe - pb) * 69 / 50 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } bfree = true; } @@ -65,7 +65,7 @@ int32_t base58_encode(const uint8_t *value, int32_t vlen, char **result) { uint8_t *pResult = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); if (!pResult) { if (bfree) taosMemoryFree(pbuf); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memset(pResult, '1', nz); while (pi != pbuf + size) pResult[nz++] = basis_58[*pi++]; @@ -117,7 +117,7 @@ int32_t base58_decode(const char *value, size_t inlen, int32_t *outlen, uint8_t size = (int32_t)(pe - pb) * 733 / 1000 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } bfree = true; } @@ -149,7 +149,7 @@ int32_t base58_decode(const char *value, size_t inlen, int32_t *outlen, uint8_t uint8_t *pResult = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); if (!pResult) { if (bfree) taosMemoryFree(pbuf); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memset(pResult, 0, nz); diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index c87c482167..841657e628 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -44,7 +44,7 @@ int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilte } SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pBF->expectedEntries = expectedEntries; @@ -65,7 +65,7 @@ int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilte pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } (*ppBF) = pBF; @@ -152,7 +152,7 @@ int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { int32_t lino = 0; SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (!pBF) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pBF->buffer = NULL; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index c6da1537f5..d20857453d 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -45,7 +45,7 @@ extern char **environ; int32_t cfgInit(SConfig **ppCfg) { SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); if (pCfg == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); @@ -1453,7 +1453,7 @@ struct SConfigIter { int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { SConfigIter *pIter = taosMemoryCalloc(1, sizeof(SConfigIter)); if (pIter == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pIter->pConf = pConf; diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index ddd04b1e88..8de8d26fad 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -93,7 +93,7 @@ int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx) { SGeosContext *tlGeosCtxObj = (SGeosContext *)taosMemoryCalloc(1, sizeof(SGeosContext)); if (!tlGeosCtxObj) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } if ((taosThreadSetSpecific(tlGeosCtxKey, (const void *)tlGeosCtxObj)) != 0) { taosMemoryFreeClear(tlGeosCtxObj); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 1d2b3f003c..4cf5917f7c 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -320,7 +320,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v } *pObj = taosMemoryCalloc(1, objSize); if (NULL == *pObj) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return func(pJsonObj, *pObj); } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 3bc24328cc..0fa73677be 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -84,7 +84,7 @@ int32_t tdListPrepend(SList *list, void *data) { int32_t tdListAppend(SList *list, const void *data) { SListNode *node = (SListNode *)taosMemoryCalloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memcpy((void *)(node->data), data, list->eleSize); diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 7faff88155..cfbd875890 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -112,7 +112,7 @@ static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) { table->lengthBits = 4; table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry *)); if (!table->list) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } table->elems = 0; diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 780a6c94f1..de21a61127 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -62,7 +62,7 @@ void taosSetQueueCapacity(STaosQueue *queue, int64_t size) { queue->itemLimit = int32_t taosOpenQueue(STaosQueue **queue) { *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); if (*queue == NULL) { - return (terrno = TSDB_CODE_OUT_OF_MEMORY); + return terrno; } int32_t code = taosThreadMutexInit(&(*queue)->mutex, NULL); @@ -162,7 +162,7 @@ int32_t taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize, void ** STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); if (pNode == NULL) { (void)atomic_sub_fetch_64(&tsQueueMemoryUsed, size + dataSize); - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pNode->dataSize = dataSize; @@ -260,7 +260,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { int32_t taosAllocateQall(STaosQall **qall) { *qall = taosMemoryCalloc(1, sizeof(STaosQall)); if (*qall == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return 0; } @@ -333,7 +333,7 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { int32_t taosOpenQset(STaosQset **qset) { *qset = taosMemoryCalloc(sizeof(STaosQset), 1); if (*qset == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexInit(&(*qset)->mutex, NULL); diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 0eac7b4427..e7161f7284 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -67,13 +67,13 @@ int32_t taosOpenRef(int32_t max, RefFp fp) { nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max); if (lockedBy == NULL) { taosMemoryFree(nodeList); - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexLock(&tsRefMutex); @@ -157,7 +157,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { pNode = taosMemoryCalloc(sizeof(SRefNode), 1); if (pNode == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } rid = atomic_add_fetch_64(&pSet->rid, 1); diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 4fac5a2b4f..ebc076e02e 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -38,7 +38,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf* } SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; @@ -218,7 +218,7 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { int32_t lino = 0; SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (!pSBf) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pSBf->hashFn1 = HASH_FUNCTION_1; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 1fc0d16d57..a1d2d8e38c 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -31,7 +31,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { pool->workers = taosMemoryCalloc(pool->max, sizeof(SQueueWorker)); if (pool->workers == NULL) { taosCloseQset(pool->qset); - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexInit(&pool->mutex, NULL); @@ -310,7 +310,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; pool->workers = taosMemoryCalloc(pool->max, sizeof(SWWorker)); if (pool->workers == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexInit(&pool->mutex, NULL); @@ -459,7 +459,7 @@ int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) case QWORKER_POOL: { SQWorkerPool *pPool = taosMemoryCalloc(1, sizeof(SQWorkerPool)); if (!pPool) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pPool->name = pCfg->name; pPool->min = pCfg->min; @@ -477,7 +477,7 @@ int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) case QUERY_AUTO_QWORKER_POOL: { SQueryAutoQWorkerPool *pPool = taosMemoryCalloc(1, sizeof(SQueryAutoQWorkerPool)); if (!pPool) { - return (terrno = TSDB_CODE_OUT_OF_MEMORY); + return terrno; } pPool->name = pCfg->name; pPool->min = pCfg->min; @@ -812,7 +812,7 @@ int32_t tQueryAutoQWorkerInit(SQueryAutoQWorkerPool *pool) { if (!pool->pCb) { pool->pCb = taosMemoryCalloc(1, sizeof(SQueryAutoQWorkerPoolCB)); - if (!pool->pCb) return TSDB_CODE_OUT_OF_MEMORY; + if (!pool->pCb) return terrno; pool->pCb->pPool = pool; pool->pCb->beforeBlocking = tQueryAutoQWorkerBeforeBlocking; pool->pCb->afterRecoverFromBlocking = tQueryAutoQWorkerRecoverFromBlocking; From 11b3faee8e5d3fe20d3d7fd488ed9ca056da89f2 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 11 Sep 2024 08:56:39 +0800 Subject: [PATCH 16/61] remove (void) from parser/planner/nodes/sma --- include/common/tname.h | 2 +- source/client/src/clientMain.c | 2 +- source/client/src/clientRawBlockWrite.c | 16 +-- source/client/src/clientSml.c | 7 +- source/common/src/tname.c | 3 +- source/dnode/mnode/impl/src/mndSma.c | 12 +- source/libs/nodes/src/nodesUtilFuncs.c | 15 ++- source/libs/parser/src/parAuthenticator.c | 5 +- source/libs/parser/src/parTranslater.c | 132 ++++++++++++---------- 9 files changed, 110 insertions(+), 84 deletions(-) diff --git a/include/common/tname.h b/include/common/tname.h index f0a68f28c4..c7c34310b9 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -37,7 +37,7 @@ typedef struct SName { char tname[TSDB_TABLE_NAME_LEN]; } SName; -SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName); +void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName); int32_t tNameExtractFullName(const SName* name, char* dst); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1c1fff9b7b..3d72c14fdd 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1526,7 +1526,7 @@ int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); SName tableName; - (void)toName(pTscObj->acctId, db, table, &tableName); + toName(pTscObj->acctId, db, table, &tableName); SVgroupInfo vgInfo; code = catalogGetTableHashVgroup(pCtg, &conn, &tableName, &vgInfo); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index d5785cce6b..0f8f0ec4e7 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -946,7 +946,8 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; - RAW_RETURN_CHECK(tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name)); + toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName); + RAW_RETURN_CHECK(tNameExtractFullName(&tableName, pReq.name)); SCmdMsgInfo pCmdMsg = {0}; pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; @@ -1021,7 +1022,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; SName pName = {0}; - (void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, + toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); // ignore the return value, always return pName STableMeta* pTableMeta = NULL; code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); @@ -1045,7 +1046,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; - if (tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name) != 0) { + toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName); + if (tNameExtractFullName(&tableName, pReq.name) != 0) { code = TSDB_CODE_INVALID_PARA; goto end; } @@ -1150,7 +1152,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -1163,7 +1165,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SName sName = {0}; tb_uid_t oldSuid = pCreateReq->ctb.suid; // pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); - (void)toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { code = TSDB_CODE_SUCCESS; @@ -1307,7 +1309,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); + toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo)); STableMeta* pTableMeta = NULL; @@ -1451,7 +1453,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); + toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo)); pArray = taosArrayInit(1, sizeof(void*)); RAW_NULL_CHECK(pArray); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 05678e1cbf..ad2655bc94 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -116,7 +116,7 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char return TSDB_CODE_SML_INVALID_DATA; } } else { - (void)toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore + toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore } pAuth.type = type; @@ -1113,7 +1113,10 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.commentLen = -1; pReq.igExists = true; - (void)tNameExtractFullName(pName, pReq.name); + code = tNameExtractFullName(pName, pReq.name); + if (TSDB_CODE_SUCCESS != code) { + goto end; + } pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 8d0f324509..4bb89464d0 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -87,12 +87,11 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in #endif -SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { +void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; snprintf(pName->dbname, sizeof(pName->dbname), "%s", pDbName); snprintf(pName->tname, sizeof(pName->tname), "%s", pTableName); - return pName; } int32_t tNameExtractFullName(const SName* name, char* dst) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 284d65cd9c..f91f7d49cc 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -368,7 +368,11 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, SEncoder encoder = {0}; int32_t contLen; SName name = {0}; - (void)tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + return NULL; + } SVDropTSmaReq req = {0}; req.indexUid = pSma->uid; @@ -1669,8 +1673,7 @@ static int32_t mndSetUpdateDbTsmaVersionPrepareLogs(SMnode *pMnode, STrans *pTra TAOS_RETURN(code); } - (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - TAOS_RETURN(code); + TAOS_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY)); } static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { @@ -1686,8 +1689,7 @@ static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTran TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - TAOS_RETURN(code); + TAOS_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); } static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1d6c19346a..8e2b739934 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -235,7 +235,10 @@ void nodesDestroyAllocatorSet() { int64_t refId = 0; while (NULL != pAllocator) { refId = pAllocator->self; - (void)taosRemoveRef(g_allocatorReqRefPool, refId); + int32_t code = taosRemoveRef(g_allocatorReqRefPool, refId); + if (TSDB_CODE_SUCCESS != code) { + nodesError("failed to remove ref at: %s:%d, rsetId:%d, refId:%"PRId64, __func__, __LINE__, g_allocatorReqRefPool, refId); + } pAllocator = taosIterateRef(g_allocatorReqRefPool, refId); } taosCloseRef(g_allocatorReqRefPool); @@ -328,7 +331,10 @@ void nodesDestroyAllocator(int64_t allocatorId) { return; } - (void)taosRemoveRef(g_allocatorReqRefPool, allocatorId); + int32_t code = taosRemoveRef(g_allocatorReqRefPool, allocatorId); + if (TSDB_CODE_SUCCESS != code) { + nodesError("failed to remove ref at: %s:%d, rsetId:%d, refId:%"PRId64, __func__, __LINE__, g_allocatorReqRefPool, allocatorId); + } } static int32_t makeNode(ENodeType type, int32_t size, SNode** ppNode) { @@ -1090,7 +1096,10 @@ void nodesDestroyNode(SNode* pNode) { pStmt->destroyParseFileCxt(&pStmt->pParFileCxt); } - (void)taosCloseFile(&pStmt->fp); + int32_t code = taosCloseFile(&pStmt->fp); + if (TSDB_CODE_SUCCESS != code) { + nodesError("failed to close file: %s:%d", __func__, __LINE__); + } break; } case QUERY_NODE_CREATE_DATABASE_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 60cc1024bf..7fbbd90d40 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -46,7 +46,7 @@ static int32_t setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const c int32_t code = tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName)); if (TSDB_CODE_SUCCESS != code) return code; } else { - (void)toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); + toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); } pAuth->type = type; pAuth->isView = isView; @@ -171,8 +171,9 @@ static EDealRes authSelectImpl(SNode* pNode, void* pContext) { #ifdef TD_ENTERPRISE SName name; STableMeta* pTableMeta = NULL; + toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name); int32_t code = getTargetMetaImpl( - pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name), &pTableMeta, true); + pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS == code && TSDB_VIEW_TABLE == pTableMeta->tableType) { isView = true; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a0b8c13e58..ebe6c82193 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -526,7 +526,8 @@ static int32_t getTargetMeta(STranslateContext* pCxt, const SName* pName, STable static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { SName name; - return getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta, false); + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + return getTargetMeta(pCxt, &name, pMeta, false); } static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCfg** pCfg) { @@ -557,7 +558,7 @@ static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, STableMeta** pMeta) { SParseContext* pParCxt = pCxt->pParseCxt; SName name; - (void)toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); int32_t code = TSDB_CODE_SUCCESS; if (pParCxt->async) { code = getTableMetaFromCache(pCxt->pMetaCache, &name, pMeta); @@ -635,7 +636,8 @@ static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pNam static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SVgroupInfo* pInfo) { SName name; - return getTableHashVgroupImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pInfo); + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + return getTableHashVgroupImpl(pCxt, &name, pInfo); } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, @@ -4016,7 +4018,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo SName tsmaTargetTbName = {0}; SVgroupInfo vgInfo = {0}; bool exists = false; - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); @@ -4678,9 +4680,8 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare // The SRealTableNode created through ROLLUP already has STableMeta. if (NULL == pRealTable->pMeta) { SName name; - code = getTargetMeta( - pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &(pRealTable->pMeta), true); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); + code = getTargetMeta( pCxt, &name, &(pRealTable->pMeta), true); if (TSDB_CODE_SUCCESS != code) { (void)generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); return code; @@ -6232,7 +6233,7 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames for (int j = 0; j < nTbls; ++j) { SName snameTb; char* tbName = taosArrayGetP(aTbnames, j); - (void)toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); + toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); SVgroupInfo vgInfo = {0}; bool bExists; int32_t code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); @@ -6261,7 +6262,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa int32_t code = 0; SRealTableNode* pRealTable = pInfo->pRealTable; char* tbName = taosArrayGetP(pInfo->aTbnames, 0); - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb); STableMeta* pMeta = NULL; TAOS_CHECK_RETURN(catalogGetCachedTableMeta(pCxt->pParseCxt->pCatalog, &snameTb, &pMeta)); @@ -6282,7 +6283,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) { STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i); SName tsmaTargetTbName = {0}; - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); @@ -8822,7 +8823,8 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm } if (TSDB_CODE_SUCCESS == code) { pReq->numOfFuncs = taosArrayGetSize(pReq->pFuncs); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + code = tNameExtractFullName(&tableName, pReq->name); } if (TSDB_CODE_SUCCESS == code) code = collectUseTable(&tableName, pCxt->pTables); @@ -8866,19 +8868,20 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); SName tableName; if (pStmt->withTsma) return TSDB_CODE_SUCCESS; - return doTranslateDropSuperTable( - pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), pClause->ignoreNotExists); + toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName); + return doTranslateDropSuperTable( pCxt, &tableName, pClause->ignoreNotExists); } static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { SName tableName; - return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), - pStmt->ignoreNotExists); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); } static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { SName tableName; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pAlterReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + int32_t code = tNameExtractFullName(&tableName, pAlterReq->name); if (TSDB_CODE_SUCCESS != code) return code; pAlterReq->alterType = pStmt->alterType; @@ -9394,10 +9397,12 @@ static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, pReq->name); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stb); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = tNameExtractFullName(&name, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { pReq->igExists = pStmt->ignoreExists; @@ -9545,10 +9550,12 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SCreateTagIndexReq* pReq) { SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->idxName); + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, pReq->idxName); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = tNameExtractFullName(&name, pReq->stbName); } if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); @@ -9584,8 +9591,8 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS int32_t code = 0; SName name; STableMeta* pMeta = NULL; - - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = getTargetMeta(pCxt, &name, &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -9627,7 +9634,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { SMDropSmaReq dropSmaReq = {0}; SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), dropSmaReq.name); + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, dropSmaReq.name); if (TSDB_CODE_SUCCESS != code) return code; dropSmaReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq); @@ -9702,11 +9710,11 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS SName name; if ('\0' != pStmt->subSTbName[0]) { pReq->subType = TOPIC_SUB_TYPE__TABLE; - (void)toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); (void)tNameGetFullDbName(&name, pReq->subDbName); if (TSDB_CODE_SUCCESS == code) { - (void)tNameExtractFullName(&name, pReq->subStbName); - if (pStmt->pQuery != NULL) { + code = tNameExtractFullName(&name, pReq->subStbName); + if (TSDB_CODE_SUCCESS == code && pStmt->pQuery != NULL) { code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } } @@ -9829,8 +9837,8 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt .mgmtEps = pParCxt->mgmtEpSet}; SName name; STableMeta* pMeta = NULL; - int32_t code = - getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false); + toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + int32_t code = getTargetMeta(pCxt, &name, &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -9952,7 +9960,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { int32_t origCode = code; SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); SViewMeta* pMeta = NULL; code = getViewMetaFromMetaCache(pCxt, &name, &pMeta); if (TSDB_CODE_SUCCESS != code) { @@ -10072,9 +10080,8 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt SName name; STableMeta* pMeta = NULL; int8_t tableType = 0; - int32_t code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &pMeta, true); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); + int32_t code = getTargetMeta(pCxt, &name, &pMeta, true); if (NULL != pMeta) { tableType = pMeta->tableType; taosMemoryFree(pMeta); @@ -11103,10 +11110,11 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* if ('\0' != pStmt->targetTabName[0]) { strcpy(name.dbname, pStmt->targetDbName); strcpy(name.tname, pStmt->targetTabName); - (void)tNameExtractFullName(&name, pReq->targetStbFullName); + code = tNameExtractFullName(&name, pReq->targetStbFullName); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildCreateStreamQuery(pCxt, pStmt, pReq); } - - code = buildCreateStreamQuery(pCxt, pStmt, pReq); if (TSDB_CODE_SUCCESS == code) { pReq->sql = taosStrdup(pCxt->pParseCxt->pSql); if (NULL == pReq->sql) { @@ -11312,7 +11320,7 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; SName name; char dbFName[TSDB_DB_FNAME_LEN]; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); (void)tNameGetFullDbName(&name, dbFName); int32_t code = validateCreateView(pCxt, pStmt); @@ -11363,7 +11371,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) return TSDB_CODE_OUT_OF_MEMORY; } dropReq.igNotExists = pStmt->ignoreNotExists; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -11399,7 +11407,8 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { int64_t s = taosReadFile(tfile, *buf, *len); if (s != *len) { - (void)taosCloseFile(&tfile); + int32_t code = taosCloseFile(&tfile); + qError("failed to close file: %s in %s:%d, err: %s", pName, __func__, __LINE__, tstrerror(code)); taosMemoryFreeClear(*buf); return TSDB_CODE_APP_ERROR; } @@ -11473,8 +11482,8 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, int32_t code = createRealTableForGrantTable(pStmt, &pTable); if (TSDB_CODE_SUCCESS == code) { SName name; - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), - &(pTable->pMeta), false); + toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name); + code = getTargetMeta(pCxt, &name, &(pTable->pMeta), false); if (code) { nodesDestroyNode((SNode*)pTable); return code; @@ -11516,8 +11525,8 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { if (0 != pStmt->tabName[0]) { SName name; STableMeta* pTableMeta = NULL; - code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -11552,8 +11561,8 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { if (0 != pStmt->tabName[0]) { SName name; STableMeta* pTableMeta = NULL; - code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -11667,7 +11676,7 @@ static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTabl int32_t code = getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pDbCfg); if (TSDB_CODE_SUCCESS == code) { SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = getTableCfg(pCxt, &name, (STableCfg**)&pStmt->pTableCfg); } return code; @@ -11678,7 +11687,7 @@ static int32_t translateShowCreateView(STranslateContext* pCxt, SShowCreateViewS return TSDB_CODE_OPS_NOT_SUPPORT; #else SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); return getViewMetaFromMetaCache(pCxt, &name, (SViewMeta**)&pStmt->pViewMeta); #endif } @@ -11949,10 +11958,11 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm SName name; SDbCfgInfo pDbInfo = {0}; int32_t code = TSDB_CODE_SUCCESS; - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); + code = tNameExtractFullName(&name, pReq->name); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); code = tNameExtractFullName(useTbName, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { @@ -12025,7 +12035,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm if (TSDB_CODE_SUCCESS == code) { memset(useTbName, 0, sizeof(SName)); memcpy(pStmt->originalTbName, pRecursiveTsma->tb, TSDB_TABLE_NAME_LEN); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName), pReq->stb); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName); + code = tNameExtractFullName(useTbName, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { numOfCols = pRecursiveTsma->pUsedCols->size; @@ -12146,7 +12157,7 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData if (TSDB_CODE_SUCCESS == code) { SName name = {0}; - (void)toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); + toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); code = collectUseTable(&name, cxt.pTargetTables); } @@ -12165,13 +12176,14 @@ static int32_t translateDropTSMA(STranslateContext* pCxt, SDropTSMAStmt* pStmt) SMDropSmaReq dropReq = {0}; SName name; STableTSMAInfo* pTsma = NULL; - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), dropReq.name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); + code = tNameExtractFullName(&name, dropReq.name); if (TSDB_CODE_SUCCESS == code) { dropReq.igNotExists = pStmt->ignoreNotExists; code = getTsma(pCxt, &name, &pTsma); } if (code == TSDB_CODE_SUCCESS) { - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); code = collectUseTable(&name, pCxt->pTargetTables); } if (TSDB_CODE_SUCCESS == code) @@ -13071,10 +13083,8 @@ static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) { int32_t code = 0; SName name; STableMeta* pTableMeta = NULL; - code = getTargetMeta(pCxt, - toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, - ((SValueNode*)pShow->pTbName)->literal, &name), - &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); goto _exit; @@ -13433,7 +13443,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = checkCreateTable(pCxt, pStmt, false); SVgroupInfo info = {0}; SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); if (TSDB_CODE_SUCCESS == code) { code = getTableHashVgroupImpl(pCxt, &name, &info); } @@ -13681,7 +13691,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla } if (TSDB_CODE_SUCCESS == code) { SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -14447,7 +14457,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { FOREACH(pNode, pStmt->pTables) { SDropTableClause* pClause = (SDropTableClause*)pNode; SName name; - (void)toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); + toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); int32_t code = buildDropTableVgroupHashmap(pCxt, pClause, &name, &tableType, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); @@ -14523,7 +14533,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTsmas = NULL; int32_t code = TSDB_CODE_SUCCESS; if (pCxt->pMetaCache) { - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (code != TSDB_CODE_SUCCESS) return code; if (pTsmas && pTsmas->size > 0) return TSDB_CODE_TSMA_MUST_BE_DROPPED; @@ -14707,7 +14717,7 @@ static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt SArray* pTsmas = NULL; SName tbName; int32_t code = 0; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); if (pCxt->pMetaCache) code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (TSDB_CODE_SUCCESS != code) return code; if (pTsmas && pTsmas->size > 0) { From b38d9da0e14225acec17a1039e9b18a7a2271b8f Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 06:26:22 +0000 Subject: [PATCH 17/61] feat:add return value --- source/dnode/mnode/impl/src/mndDb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9c02fd6e5d..6467dfcf11 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -2333,9 +2333,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, char keep1Str[128] = {0}; char keep2Str[128] = {0}; - formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0); - formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1); - formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2); + int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0); + int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1); + int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2); if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); From f5e2a36f2382585af829bb6da0542adbe12c6425 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 06:29:35 +0000 Subject: [PATCH 18/61] feat:update --- source/libs/command/src/command.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 0e5f2f4cbd..70a9b39229 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -345,12 +345,12 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { } int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes) { - int len = 0; + int32_t len = 0; if (timeInMinutes % 1440 == 0) { - int days = timeInMinutes / 1440; + int32_t days = timeInMinutes / 1440; len = sprintf(buffer, "%dd", days); } else if (timeInMinutes % 60 == 0) { - int hours = timeInMinutes / 60; + int32_t hours = timeInMinutes / 60; len = sprintf(buffer, "%dh", hours); } else { len = sprintf(buffer, "%dm", timeInMinutes); From 28df9fe2851243347cb54f9d36c50e0ec4094a04 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Sep 2024 14:30:59 +0800 Subject: [PATCH 19/61] refactor: remove void --- source/libs/stream/src/streamCheckStatus.c | 83 +++++++++++++++------- source/libs/stream/src/streamDispatch.c | 6 +- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 41124d8543..274ec5043c 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -27,7 +27,7 @@ static void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInf static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id); static void streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id); static void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id); -static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p); +static int32_t doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p); static void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList); static void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList); static int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, @@ -83,6 +83,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { SDataRange* pRange = &pTask->dataRange; STimeWindow* pWindow = &pRange->window; const char* idstr = pTask->id.idStr; + int32_t code = 0; SStreamTaskCheckReq req = { .streamId = pTask->id.streamId, @@ -102,11 +103,11 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pDispatch->taskId, pDispatch->nodeId, idstr); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64 - " window:%" PRId64 "-%" PRId64 "QID:0x%" PRIx64, + " window:%" PRId64 "-%" PRId64 " QID:0x%" PRIx64, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); - (void)streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, + code = streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -128,15 +129,19 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pVgInfo->taskId, pVgInfo->vgId, idstr); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 - " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d,QID:0x%" PRIx64, + " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, QID:0x%" PRIx64, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId); - (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + code = streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { // for sink task, set it ready directly. stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId); streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, idstr); processDownstreamReadyRsp(pTask); } + + if (code) { + stError("s-task:%s failed to send check msg to downstream, code:%s", idstr, tstrerror(code)); + } } void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp) { @@ -243,13 +248,13 @@ int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTaskCheckRsp* pRsp, SRpcHandleInfo* pRpcInfo, int32_t taskId) { SEncoder encoder; - int32_t code; + int32_t code = 0; int32_t len; tEncodeSize(tEncodeStreamTaskCheckRsp, pRsp, len, code); if (code < 0) { stError("vgId:%d failed to encode task check rsp, s-task:0x%x", pMeta->vgId, taskId); - return -1; + return TSDB_CODE_INVALID_MSG; } void* buf = rpcMallocCont(sizeof(SMsgHead) + len); @@ -257,13 +262,13 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, (uint8_t*)abuf, len); - (void)tEncodeStreamTaskCheckRsp(&encoder, pRsp); + code = tEncodeStreamTaskCheckRsp(&encoder, pRsp); tEncoderClear(&encoder); SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; - tmsgSendRsp(&rspMsg); - return 0; + + return code; } void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { @@ -316,8 +321,11 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { pInfo->pList = NULL; if (pInfo->checkRspTmr != NULL) { - (void)taosTmrStop(pInfo->checkRspTmr); + bool succ = taosTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; + if (!succ) { + stError("failed to stop checkrsp tmr"); // todo: add id + } } streamMutexDestroy(&pInfo->checkInfoLock); @@ -326,11 +334,17 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void processDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - (void)streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); + int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); + if (code) { + stError("s-task:%s failed to set event succ, code:%s", pTask->id.idStr, tstrerror(code)); + } int64_t checkTs = pTask->execInfo.checkTs; int64_t readyTs = pTask->execInfo.readyTs; - (void)streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); + code = streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); + if (code) { + stError("s-task:%s failed to record the downstream task status, code:%s", pTask->id.idStr, tstrerror(code)); + } if (pTask->status.taskStatus == TASK_STATUS__HALT) { if (!HAS_RELATED_FILLHISTORY_TASK(pTask) || (pTask->info.fillHistory != 0)) { @@ -341,9 +355,9 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { // halt it self for count window stream task until the related fill history task completed. stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr, pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus)); - int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); - if (code != 0) { - // todo: handle error + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); + if (code != 0) { // todo: handle error + stError("s-task:%s failed to handle halt event, code:%s", pTask->id.idStr, tstrerror(code)); } } @@ -352,7 +366,10 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { // todo: let's retry if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); - (void)streamLaunchFillHistoryTask(pTask); + code = streamLaunchFillHistoryTask(pTask); + if (code) { + stError("s-task:%s failed to launch history task, code:%s", pTask->id.idStr, tstrerror(code)); + } } } @@ -517,8 +534,9 @@ void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, streamMutexUnlock(&pInfo->checkInfoLock); } -void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { +int32_t doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { const char* id = pTask->id.idStr; + int32_t code = 0; SStreamTaskCheckReq req = { .streamId = pTask->id.streamId, @@ -536,10 +554,10 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher; setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId); - stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d)QID:0x%" PRIx64, id, + stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) QID:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); - (void)streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); + code = streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgs = taosArrayGetSize(vgInfo); @@ -554,13 +572,19 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { setCheckDownstreamReqInfo(&req, p->reqId, pVgInfo->taskId, pVgInfo->vgId); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 - " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%dQID:0x%" PRIx64, + " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d QID:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId); - (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + code = streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); break; } } } + + if (code) { + stError("s-task:%s failed to send check msg to downstream, code:%s", pTask->id.idStr, tstrerror(code)); + } else { + return code; + } } void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault, @@ -626,7 +650,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { continue; } - doSendCheckMsg(pTask, p); + int32_t code = doSendCheckMsg(pTask, p); } } @@ -676,7 +700,7 @@ void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList) { if (p != NULL) { p->rspTs = 0; p->status = -1; - doSendCheckMsg(pTask, p); + int32_t code = doSendCheckMsg(pTask, p); } } @@ -723,7 +747,10 @@ void rspMonitorFn(void* param, void* tmrId) { // not record the failed of the current task if try to close current vnode // otherwise, the put of message operation may incur invalid read of message queue. if (!pMeta->closeFlag) { - (void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + int32_t code = addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + if (code) { + stError("s-task:%s failed to create async record start failed task, code:%s", id, tstrerror(code)); + } } streamMetaReleaseTask(pMeta, pTask); @@ -805,7 +832,11 @@ void rspMonitorFn(void* param, void* tmrId) { streamTaskCompleteCheckRsp(pInfo, false, id); streamMutexUnlock(&pInfo->checkInfoLock); - (void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + int32_t code = addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + if (code) { + stError("s-task:%s failed to create async record start failed task, code:%s", id, tstrerror(code)); + } + streamMetaReleaseTask(pMeta, pTask); taosArrayDestroy(pNotReadyList); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9cac5578a8..e636683f02 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -193,13 +193,13 @@ int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock) // no need to do anything if failed int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { void* buf = NULL; - int32_t code = -1; + int32_t code = 0; SRpcMsg msg = {0}; int32_t tlen; tEncodeSize(tEncodeStreamTaskCheckReq, pReq, tlen, code); if (code < 0) { - return -1; + return code; } buf = rpcMallocCont(sizeof(SMsgHead) + tlen); @@ -217,8 +217,8 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, tEncoderClear(&encoder); return code; } - tEncoderClear(&encoder); + tEncoderClear(&encoder); initRpcMsg(&msg, TDMT_VND_STREAM_TASK_CHECK, buf, tlen + sizeof(SMsgHead)); stDebug("s-task:%s (level:%d) send check msg to s-task:0x%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr, pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); From 3821f88e6495e8d36e3f35efb2820c20196e8016 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Sep 2024 14:32:29 +0800 Subject: [PATCH 20/61] fix(stream): fix syntax error. --- source/libs/stream/src/streamCheckStatus.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 274ec5043c..d0604e6d21 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -582,9 +582,8 @@ int32_t doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { if (code) { stError("s-task:%s failed to send check msg to downstream, code:%s", pTask->id.idStr, tstrerror(code)); - } else { - return code; } + return code; } void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault, From 6c610e477755b9f4eafdc6bf6a2318224e28067c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 14:45:03 +0800 Subject: [PATCH 21/61] fix mem leak --- source/libs/transport/src/transCli.c | 1 - source/libs/transport/src/transSvr.c | 14 +++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a0c7315fe7..ce9adc836e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2422,7 +2422,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); - (void)uv_timer_stop(timer); taosMemoryFree(timer); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 48042b84c1..1db5d0a6d7 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1398,10 +1398,21 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, thrd->quit = false; thrd->pTransInst = shandle; thrd->pWhiteList = uvWhiteListCreate(); + if (thrd->pWhiteList == NULL) { + destroyWorkThrdObj(thrd); + code = TSDB_CODE_OUT_OF_MEMORY; + goto End; + } - srv->pThreadObj[i] = thrd; srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); + if (srv->pipe[i] == NULL) { + destroyWorkThrdObj(thrd); + code = TSDB_CODE_OUT_OF_MEMORY; + goto End; + } + thrd->pipe = &(srv->pipe[i][1]); // init read + srv->pThreadObj[i] = thrd; if ((code = addHandleToWorkloop(thrd, pipeName)) != 0) { goto End; @@ -1415,6 +1426,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, tError("failed to create worker-thread:%d", i); goto End; } + thrd->inited = 1; } #else From c38eb033a998804951e1d5d3577d93d51b6718d7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Sep 2024 15:57:06 +0800 Subject: [PATCH 22/61] refactor:update logs. --- source/libs/stream/src/streamCheckpoint.c | 2 +- source/libs/stream/src/streamDispatch.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index fa2bc65194..386c0d8222 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -237,7 +237,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (pActiveInfo->failedId >= checkpointId) { stError("s-task:%s vgId:%d checkpointId:%" PRId64 " transId:%d, has been marked failed, failedId:%" PRId64 - "discard the checkpoint-trigger block", + " discard the checkpoint-trigger block", id, vgId, checkpointId, transId, pActiveInfo->failedId); streamMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e636683f02..63411df11a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1498,8 +1498,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i // follower not handle the dispatch rsp if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) { - stError("s-task:%s vgId:%d is follower or task just re-launched, not handle the dispatch rsp, discard it", id, - vgId); + stError("s-task:%s vgId:%d is follower or just re-launched, not handle the dispatch rsp, discard it", id, vgId); streamMutexUnlock(&pMsgInfo->lock); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } From 9b418bee783bd7c53e36819bd408d05b590d6427 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 16:37:06 +0800 Subject: [PATCH 23/61] fix meta dead lock --- source/libs/executor/src/scanoperator.c | 139 ++++++++++++------------ 1 file changed, 72 insertions(+), 67 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 42e7e4ac3b..b0b6a91faa 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -90,7 +90,7 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order, bool* overlap) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; STimeWindow w = {0}; // 0 by default, which means it is not a interval operator of the upstream operator. @@ -101,7 +101,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc if (order == TSDB_ORDER_ASC) { w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); - if(w.ekey < pBlockInfo->window.skey) { + if (w.ekey < pBlockInfo->window.skey) { qError("w.ekey:%" PRId64 " < pBlockInfo->window.skey:%" PRId64, w.ekey, pBlockInfo->window.skey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -117,7 +117,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc break; } - if(w.ekey <= pBlockInfo->window.ekey) { + if (w.ekey <= pBlockInfo->window.ekey) { qError("w.ekey:%" PRId64 " <= pBlockInfo->window.ekey:%" PRId64, w.ekey, pBlockInfo->window.ekey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -128,7 +128,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc } } else { w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); - if(w.skey > pBlockInfo->window.ekey) { + if (w.skey > pBlockInfo->window.ekey) { qError("w.skey:%" PRId64 " > pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.ekey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -144,7 +144,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc break; } - if(w.skey >= pBlockInfo->window.skey){ + if (w.skey >= pBlockInfo->window.skey) { qError("w.skey:%" PRId64 " >= pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.skey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -227,7 +227,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset); - EFuncDataRequired reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); + EFuncDataRequired reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { notLoadBlock = false; break; @@ -399,7 +399,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca } } - if(*status != FUNC_DATA_REQUIRED_DATA_LOAD) { + if (*status != FUNC_DATA_REQUIRED_DATA_LOAD) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); qError("%s loadDataBlock invalid status:%d", GET_TASKID(pTaskInfo), *status); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; @@ -588,14 +588,14 @@ static void freeTableCachedValObj(STableCachedVal* pVal) { int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; - bool freeReader = false; - LRUHandle* h = NULL; - STableCachedVal val = {0}; - SMetaReader mr = {0}; - const char* idStr = pTask->id.str; - int32_t insertRet = TAOS_LRU_STATUS_OK; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool freeReader = false; + LRUHandle* h = NULL; + STableCachedVal val = {0}; + SMetaReader mr = {0}; + const char* idStr = pTask->id.str; + int32_t insertRet = TAOS_LRU_STATUS_OK; STableCachedVal* pVal = NULL; // currently only the tbname pseudo column @@ -625,8 +625,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // append null value before return to caller, since the caller will ignore this error code and proceed doSetNullValue(pBlock, pExpr, numOfExpr); } else { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(code), - idStr); + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(code), idStr); } pHandle->api.metaReaderFn.clearReader(&mr); return code; @@ -743,7 +742,7 @@ _end: if (NULL != pVal) { insertRet = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal, - sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); + sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (insertRet != TAOS_LRU_STATUS_OK) { qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr); } @@ -1416,7 +1415,7 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr if (!pRecorder) { return terrno; } - STableScanInfo* pTableScanInfo = pOptr->info; + STableScanInfo* pTableScanInfo = pOptr->info; *pRecorder = pTableScanInfo->base.readRecorder; *pOptrExplain = pRecorder; *len = sizeof(SFileBlockLoadRecorder); @@ -1499,7 +1498,7 @@ int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHa initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); QUERY_CHECK_NULL(pInfo->pResBlock, code, lino, _error, terrno); - + code = prepareDataBlockBuf(pInfo->pResBlock, &pInfo->base.matchInfo); QUERY_CHECK_CODE(code, lino, _error); @@ -1538,7 +1537,7 @@ int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHa _error: if (pInfo != NULL) { - pInfo->base.pTableListInfo = NULL; // this attribute will be destroy outside of this function + pInfo->base.pTableListInfo = NULL; // this attribute will be destroy outside of this function destroyTableScanOperatorInfo(pInfo); } @@ -1660,7 +1659,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (void**)&pReader, GET_TASKID(pTaskInfo), NULL); + (void**)&pReader, GET_TASKID(pTaskInfo), NULL); QUERY_CHECK_CODE(code, lino, _end); bool hasNext = false; @@ -1813,7 +1812,7 @@ static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ QUERY_CHECK_NULL(pInfo->pUpdateInfo, code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, - pInfo->pUpdateInfo->maxDataVersion); + pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; if (pRes) { @@ -2236,9 +2235,9 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS } uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; - TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; - TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t ver = pSrcBlock->info.version - 1; + TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; + int64_t ver = pSrcBlock->info.version - 1; if (pInfo->partitionSup.needCalc && (srcStartTsCol[0] != srcEndTsCol[0] || (hasPrimaryKeyCol(pInfo) && mode == STREAM_DELETE_DATA))) { @@ -2804,8 +2803,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { - code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, - pInfo->pRes, pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache); + code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, + pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { code = 0; @@ -3182,7 +3181,7 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) return; } - void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); + void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); if (!pUpInfo) { return; } @@ -3471,7 +3470,8 @@ FETCH_NEXT_BLOCK: (*ppRes) = pInfo->pDeleteDataRes; return code; } - qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, lino); + qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, + lino); blockDataCleanup(pInfo->pUpdateDataRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } break; @@ -3484,7 +3484,8 @@ FETCH_NEXT_BLOCK: (*ppRes) = pInfo->pUpdateRes; return code; } - qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, lino); + qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, + lino); blockDataCleanup(pInfo->pUpdateDataRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } break; @@ -3671,7 +3672,7 @@ static int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** for (int32_t i = 0; i < size; ++i) { STableKeyInfo* pkeyInfo = tableListGetInfo(pTableListInfo, i); QUERY_CHECK_NULL(pkeyInfo, code, lino, _end, terrno); - void* tmp = taosArrayPush(tableIdList, &pkeyInfo->uid); + void* tmp = taosArrayPush(tableIdList, &pkeyInfo->uid); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } @@ -4066,7 +4067,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* int32_t numOfOutput = taosArrayGetSize(pInfo->matchInfo.pList); pColIds = taosArrayInit(numOfOutput, sizeof(int16_t)); QUERY_CHECK_NULL(pColIds, code, lino, _error, terrno); - + for (int32_t i = 0; i < numOfOutput; ++i) { SColMatchItem* id = taosArrayGet(pInfo->matchInfo.pList, i); QUERY_CHECK_NULL(id, code, lino, _error, terrno); @@ -4268,7 +4269,7 @@ _error: } static int32_t doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, - SStorageAPI* pAPI) { + SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4280,7 +4281,6 @@ static int32_t doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pR qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); tDecoderClear(&(*mr).coder); - pAPI->metaReaderFn.clearReader(mr); goto _end; } @@ -4289,7 +4289,6 @@ static int32_t doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pR if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - pAPI->metaReaderFn.clearReader(mr); goto _end; } @@ -4697,6 +4696,12 @@ static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock* while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { code = doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); + if (code == TSDB_CODE_OUT_OF_MEMORY) { + break; + } else { + // ignore other error + } + ++count; if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); @@ -4814,7 +4819,7 @@ int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* p nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx); } } - //TODO wjm check pInfo->filterCtx.code + // TODO wjm check pInfo->filterCtx.code __optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdxNext : doTagScanFromMetaEntryNext; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); @@ -4925,7 +4930,7 @@ static int32_t fetchNextSubTableBlockFromReader(SOperatorInfo* pOperator, STmsSu pInfo->base.dataReader = pInput->pReader; while (true) { - bool hasNext = false; + bool hasNext = false; code = pAPI->tsdReader.tsdNextDataBlock(pInfo->base.dataReader, &hasNext); if (code != 0) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->base.dataReader); @@ -5036,7 +5041,7 @@ static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) { static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { int32_t code = setGroupStartEndIndex(pInfo); if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); return code; } STmsSubTablesMergeInfo* pSubTblsInfo = taosMemoryCalloc(1, sizeof(STmsSubTablesMergeInfo)); @@ -5065,8 +5070,7 @@ static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { } int32_t bufPageSize = pInfo->bufPageSize; int32_t inMemSize = (pSubTblsInfo->numSubTables - pSubTblsInfo->numTableBlocksInMem) * bufPageSize; - code = - createDiskbasedBuf(&pSubTblsInfo->pBlocksBuf, pInfo->bufPageSize, inMemSize, "blocksExternalBuf", tsTempDir); + code = createDiskbasedBuf(&pSubTblsInfo->pBlocksBuf, pInfo->bufPageSize, inMemSize, "blocksExternalBuf", tsTempDir); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pSubTblsInfo->aInputs); taosMemoryFree(pSubTblsInfo); @@ -5199,7 +5203,7 @@ static int32_t appendChosenRowToDataBlock(STmsSubTablesMergeInfo* pSubTblsInfo, SColumnInfoData* pSrcColInfo = taosArrayGet(pInputBlock->pDataBlock, i); QUERY_CHECK_NULL(pSrcColInfo, code, lino, _end, terrno); - bool isNull = colDataIsNull(pSrcColInfo, pInputBlock->info.rows, pInput->rowIdx, NULL); + bool isNull = colDataIsNull(pSrcColInfo, pInputBlock->info.rows, pInput->rowIdx, NULL); if (isNull) { code = colDataSetVal(pColInfo, pBlock->info.rows, NULL, true); @@ -5223,7 +5227,8 @@ _end: return code; } -static int32_t getSubTablesSortedBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t capacity, SSDataBlock** pResBlock) { +static int32_t getSubTablesSortedBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t capacity, + SSDataBlock** pResBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; STableMergeScanInfo* pInfo = pOperator->info; @@ -5580,9 +5585,9 @@ static int32_t getBlockForTableMergeScan(void* param, SSDataBlock** ppBlock) { } int32_t generateSortByTsPkInfo(SArray* colMatchInfo, int32_t order, SArray** ppSortArray) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; - SArray* pSortInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SArray* pSortInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); QUERY_CHECK_NULL(pSortInfo, code, lino, _end, terrno); SBlockOrderInfo biTs = {0}; SBlockOrderInfo biPk = {0}; @@ -5983,7 +5988,7 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla if (!execInfo) { return terrno; } - STableMergeScanInfo* pInfo = pOptr->info; + STableMergeScanInfo* pInfo = pOptr->info; execInfo->blockRecorder = pInfo->base.readRecorder; execInfo->sortExecInfo = pInfo->sortExecInfo; @@ -6126,24 +6131,24 @@ _error: // ==================================================================================================================== // TableCountScanOperator -static void destoryTableCountScanOperator(void* param); -static int32_t buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI); -static int32_t buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI); -static int32_t buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); -static int32_t buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName); -static int32_t buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes); -static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, - size_t perfdbTableNum); -static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, - size_t infodbTableNum, size_t perfdbTableNum); -static const char* GROUP_TAG_DB_NAME = "db_name"; -static const char* GROUP_TAG_STABLE_NAME = "stable_name"; +static void destoryTableCountScanOperator(void* param); +static int32_t buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI); +static int32_t buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI); +static int32_t buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); +static int32_t buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName); +static int32_t buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes); +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum); +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum); +static const char* GROUP_TAG_DB_NAME = "db_name"; +static const char* GROUP_TAG_STABLE_NAME = "stable_name"; int32_t tblCountScanGetGroupTagsSlotId(const SNodeList* scanCols, STableCountScanSupp* supp) { if (scanCols != NULL) { @@ -6478,7 +6483,7 @@ _end: } static int32_t buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -6526,7 +6531,7 @@ _end: } static int32_t buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; From 88250653642f48176eb25e3a84b03ba69f5f6862 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Sep 2024 16:58:29 +0800 Subject: [PATCH 24/61] refactor: update logs. --- source/libs/stream/src/streamDispatch.c | 29 +++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 63411df11a..b5c265af0d 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1477,20 +1477,25 @@ int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) { } int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { - const char* id = pTask->id.idStr; - int32_t vgId = pTask->pMeta->vgId; - SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; - int64_t now = taosGetTimestampMs(); - bool allRsp = false; - int32_t notRsp = 0; - int32_t numOfFailed = 0; - bool triggerDispatchRsp = false; + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; + SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; + int64_t now = taosGetTimestampMs(); + bool allRsp = false; + int32_t notRsp = 0; + int32_t numOfFailed = 0; + bool triggerDispatchRsp = false; + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; // we only set the dispatch msg info for current checkpoint trans + int64_t tmpCheckpointId = -1; + int32_t tmpTranId = -1; + streamMutexLock(&pTask->lock); triggerDispatchRsp = (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) && - (pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) && - (pTask->chkInfo.pActiveInfo->transId != pMsgInfo->transId); + (pInfo->activeId == pMsgInfo->checkpointId) && (pInfo->transId != pMsgInfo->transId); + tmpCheckpointId = pInfo->activeId; + tmpTranId = pInfo->transId; streamMutexUnlock(&pTask->lock); streamMutexLock(&pMsgInfo->lock); @@ -1556,8 +1561,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i streamTaskSetTriggerDispatchConfirmed(pTask, pRsp->downstreamNodeId); } else { stWarn("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 - " transId:%d discard, since expired", - pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId); + " transId:%d discard, current active checkpointId:%" PRId64 " active transId:%d, since expired", + pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId, tmpCheckpointId, tmpTranId); } } } From 1e388cace72d5f432165ed05439f9bc6bf7e8144 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Sep 2024 17:01:18 +0800 Subject: [PATCH 25/61] refactor: update logs. --- source/libs/stream/src/streamDispatch.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index b5c265af0d..eb6c9c1a66 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1486,16 +1486,18 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int32_t numOfFailed = 0; bool triggerDispatchRsp = false; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + int64_t tmpCheckpointId = -1; + int32_t tmpTranId = -1; + const char* pStatus = NULL; // we only set the dispatch msg info for current checkpoint trans - int64_t tmpCheckpointId = -1; - int32_t tmpTranId = -1; - streamMutexLock(&pTask->lock); - triggerDispatchRsp = (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) && - (pInfo->activeId == pMsgInfo->checkpointId) && (pInfo->transId != pMsgInfo->transId); + SStreamTaskState s = streamTaskGetStatus(pTask); + triggerDispatchRsp = (s.state == TASK_STATUS__CK) && (pInfo->activeId == pMsgInfo->checkpointId) && + (pInfo->transId != pMsgInfo->transId); tmpCheckpointId = pInfo->activeId; tmpTranId = pInfo->transId; + pStatus = s.name; streamMutexUnlock(&pTask->lock); streamMutexLock(&pMsgInfo->lock); @@ -1561,8 +1563,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i streamTaskSetTriggerDispatchConfirmed(pTask, pRsp->downstreamNodeId); } else { stWarn("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 - " transId:%d discard, current active checkpointId:%" PRId64 " active transId:%d, since expired", - pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId, tmpCheckpointId, tmpTranId); + " transId:%d discard, current status:%s, active checkpointId:%" PRId64 + " active transId:%d, since expired", + pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId, pStatus, tmpCheckpointId, tmpTranId); } } } From 0b6a934586e7d07b51b04d6054be096efc10faf7 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 11 Sep 2024 09:15:20 +0000 Subject: [PATCH 26/61] fix/TS-5404-use-temp-transid-when-timeout-lock --- source/dnode/mnode/impl/src/mndTrans.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 18ceddb7d0..724c9637e6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -64,7 +64,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); -int32_t tsMaxTransId = 0; +static int32_t tsMaxTransId = 0; int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { @@ -605,8 +605,9 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, } int32_t sdbMaxId = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); - int32_t oldId = atomic_load_32(&tsMaxTransId); - pTrans->id = TMAX(sdbMaxId, oldId); + sdbReadLock(pMnode->pSdb, SDB_TRANS); + pTrans->id = TMAX(sdbMaxId, tsMaxTransId + 1); + sdbUnLock(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->conflict = conflict; @@ -1031,7 +1032,9 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); - atomic_store_32(&tsMaxTransId, pTrans->id); + sdbReadLock(pMnode->pSdb, SDB_TRANS); + tsMaxTransId = TMAX(pTrans->id, tsMaxTransId); + sdbUnLock(pMnode->pSdb, SDB_TRANS); TAOS_RETURN(code); } mInfo("trans:%d, prepare finished", pTrans->id); From cf35e31289b86d885ae1cdd2812f6c0f13ce8fab Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 09:43:55 +0000 Subject: [PATCH 27/61] modify test case --- tests/script/tsim/db/alter_option.sim | 6 +++--- tests/script/tsim/db/basic6.sim | 6 +++--- tests/script/tsim/db/keep.sim | 4 ++-- tests/script/tsim/parser/alter.sim | 14 +++++++------- .../tsim/parser/alter__for_community_version.sim | 12 ++++++------ tests/script/tsim/parser/create_db.sim | 16 ++++++++-------- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 749b956a9a..4c18fe171c 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -72,10 +72,10 @@ endi if $data5_db != on then # strict return -1 endi -if $data6_db != 345600m then # duration +if $data6_db != 240d then # duration return -1 endi -if $data7_db != 1440000m,1440000m,1440000m then # keep +if $data7_db != 1000d,1000d,1000d then # keep return -1 endi if $data8_db != 256 then # buffer @@ -220,7 +220,7 @@ print ============== modify keep sql alter database db keep 2400 sql select * from information_schema.ins_databases print keep $data7_db -if $data7_db != 3456000m,3456000m,3456000m then +if $data7_db != 2400d,2400d,2400d then return -1 endi diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 5043574787..8d16c3e975 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -34,10 +34,10 @@ endi if $data24 != 1 then return -1 endi -if $data26 != 2880m then +if $data26 != 2d then return -1 endi -if $data27 != 14400m,14400m,14400m then +if $data27 != 10d,10d,10d then return -1 endi #if $data28 != 32 then @@ -78,7 +78,7 @@ endi if $data24 != 1 then return -1 endi -if $data26 != 21600m then +if $data26 != 15d then return -1 endi diff --git a/tests/script/tsim/db/keep.sim b/tests/script/tsim/db/keep.sim index f0653c4801..28c595cd21 100644 --- a/tests/script/tsim/db/keep.sim +++ b/tests/script/tsim/db/keep.sim @@ -45,7 +45,7 @@ print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data22 != 2 then return -1 endi -if $data27 != 86400m,86400m,86400m then +if $data27 != 60d,60d,60d then return -1 endi @@ -86,7 +86,7 @@ sql select * from information_schema.ins_databases if $data22 != 2 then return -1 endi -if $data27 != 43200m,43200m,43200m then +if $data27 != 30d,30d,30d then return -1 endi diff --git a/tests/script/tsim/parser/alter.sim b/tests/script/tsim/parser/alter.sim index 99e014a011..ceeb5df34f 100644 --- a/tests/script/tsim/parser/alter.sim +++ b/tests/script/tsim/parser/alter.sim @@ -40,7 +40,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 30240m,30240m,30240m then +if $data27 != 21d,21d,21d then return -1 endi sql alter database $db keep 11,12 @@ -48,7 +48,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,17280m,17280m then +if $data27 != 11d,12d,12d then return -1 endi sql alter database $db keep 20,20,20 @@ -56,7 +56,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 28800m,28800m,28800m then +if $data27 != 20d,20d,20d then return -1 endi sql alter database $db keep 10,10,10 @@ -64,7 +64,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 14400m,14400m,14400m then +if $data27 != 10d,10d,10d then return -1 endi sql alter database $db keep 10,10,11 @@ -72,7 +72,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 14400m,14400m,15840m then +if $data27 != 10d,10d,11d then return -1 endi sql alter database $db keep 11,12,13 @@ -80,7 +80,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,17280m,18720m then +if $data27 != 11d,12d,13d then return -1 endi sql alter database $db keep 365000,365000,365000 @@ -88,7 +88,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 525600000m,525600000m,525600000m then +if $data27 != 365000d,365000d,365000d then return -1 endi diff --git a/tests/script/tsim/parser/alter__for_community_version.sim b/tests/script/tsim/parser/alter__for_community_version.sim index 29c748d441..b3534204fc 100644 --- a/tests/script/tsim/parser/alter__for_community_version.sim +++ b/tests/script/tsim/parser/alter__for_community_version.sim @@ -23,7 +23,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 28800m,28800m,28800m then +if $data27 != 20d,20d,20d then return -1 endi @@ -49,7 +49,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 28800m,28800m,28800m then +if $data27 != 20d,20d,20d then return -1 endi sql alter database $db keep 10 @@ -57,7 +57,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 14400m,14400m,14400m then +if $data27 != 10d,10d,10d then return -1 endi sql alter database $db keep 11 @@ -65,7 +65,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,15840m,15840m then +if $data27 != 11d,11d,11d then return -1 endi sql alter database $db keep 13 @@ -73,7 +73,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 18720m,18720m,18720m then +if $data27 != 13d,13d,13d then return -1 endi sql alter database $db keep 365000 @@ -81,7 +81,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 525600000m,525600000m,525600000m then +if $data27 != 365000d,365000d,365000d then return -1 endi diff --git a/tests/script/tsim/parser/create_db.sim b/tests/script/tsim/parser/create_db.sim index 5cb659586c..1057e0b465 100644 --- a/tests/script/tsim/parser/create_db.sim +++ b/tests/script/tsim/parser/create_db.sim @@ -118,10 +118,10 @@ endi if $data24 != $replica then return -1 endi -if $data26 != 14400m then +if $data26 != 10d then return -1 endi -if $data27 != 525600m,525600m,525600m then +if $data27 != 365d,365d,365d then return -1 endi @@ -155,7 +155,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 56160m,56160m,56160m then +if $data27 != 39d,39d,39d then return -1 endi sql drop database dbk0 @@ -164,7 +164,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 56160m,57600m,57600m then +if $data27 != 39d,40d,40d then return -1 endi sql drop database dbka @@ -174,7 +174,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,15840m,15840m then +if $data27 != 11d,11d,11d then return -1 endi sql drop database dbk1 @@ -183,7 +183,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,17280m,18720m then +if $data27 != 11d,12d,13d then return -1 endi sql drop database dbk2 @@ -192,7 +192,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,15840m,18720m then +if $data27 != 11d,11d,13d then return -1 endi sql drop database dbk3 @@ -201,7 +201,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,18720m,18720m then +if $data27 != 11d,13d,13d then return -1 endi sql drop database dbk4 From 4d103c09d490f67fd5b6fb836d91449a8731433c Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 19:22:04 +0800 Subject: [PATCH 28/61] fix: ignore tsdbCacheFlushDirty errcode --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 53e20d459f..74bd837775 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -577,7 +577,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCacheFlushDirty(key, klen, pLastCol, ud); + (void)tsdbCacheFlushDirty(key, klen, pLastCol, ud); } for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { From 9923079693fb3eb24186812bed79f7519980ea95 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 11 Sep 2024 19:57:47 +0800 Subject: [PATCH 29/61] TD-31198: add 32/64 bit ODBC driver into Windows packages --- cmake/taosadapter_CMakeLists.txt.in | 2 +- packaging/tools/tdengine.iss | 41 +++++++++++++---------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index ca6b5a3e5f..8085c55e3e 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -55,7 +55,8 @@ Source: favicon.ico; DestDir: "{app}\include"; Flags: igNoreversion; Source: {#MyAppSourceDir}{#MyAppDLLName}; DestDir: "{win}\System32"; Flags: igNoreversion recursesubdirs createallsubdirs 64bit;Check:IsWin64; Source: {#MyAppSourceDir}{#MyAppCfgName}; DestDir: "{app}\cfg"; Flags: igNoreversion recursesubdirs createallsubdirs onlyifdoesntexist uninsneveruninstall Source: {#MyAppSourceDir}{#MyAppDriverName}; DestDir: "{app}\driver"; Flags: igNoreversion recursesubdirs createallsubdirs -Source: {#MyAppSourceDir}\taos_odbc\*; DestDir: "{app}\taos_odbc\"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}\taos_odbc\*; DestDir: "{app}\taos_odbc"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}\*.dll; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs ;Source: {#MyAppSourceDir}{#MyAppConnectorName}; DestDir: "{app}\connector"; Flags: igNoreversion recursesubdirs createallsubdirs ;Source: {#MyAppSourceDir}{#MyAppExamplesName}; DestDir: "{app}\examples"; Flags: igNoreversion recursesubdirs createallsubdirs Source: {#MyAppSourceDir}{#MyAppIncludeName}; DestDir: "{app}\include"; Flags: igNoreversion recursesubdirs createallsubdirs @@ -69,7 +70,9 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt [run] Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden -Filename: "{cmd}"; Parameters: "/c odbcconf /F ""C:\TDengine\taos_odbc\win_odbcinst.in"""; WorkingDir: "{app}"; Flags: runhidden; StatusMsg: "Configuring ODBC" + +Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64" +Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86" [UninstallRun] RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden @@ -100,32 +103,26 @@ begin end; function DeleteOdbcDsnRegistry: Boolean; -var - Names: TArrayOfString; - I: Integer; - Value: String; begin - if RegGetValueNames(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', Names) then - begin - for I := 0 to GetArrayLength(Names) - 1 do - begin - if RegQueryStringValue(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', Names[I], Value) then - begin - if Value = 'TDengine' then - begin - RegDeleteKeyIncludingSubkeys(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\' + Names[I]); - RegDeleteValue(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources\', Names[I]); - end; - end; - end; - end; + RegDeleteKeyIncludingSubkeys(HKCU, 'SOFTWARE\ODBC\ODBC.INI\TAOS_ODBC_DSN'); + RegDeleteKeyIncludingSubkeys(HKCU, 'SOFTWARE\ODBC\ODBC.INI\TAOS_ODBC_WS_DSN') + + RegDeleteValue(HKCU, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', 'TAOS_ODBC_DSN'); + RegDeleteValue(HKCU, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', 'TAOS_ODBC_WS_DSN'); + Result := True; end; function DeleteOdbcDriverRegistry: Boolean; begin - RegDeleteKeyIncludingSubkeys(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\TDengine'); - RegDeleteValue(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\ODBC Drivers', 'TDengine'); + // Delete 64-bit ODBC driver registry + RegDeleteKeyIncludingSubkeys(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\TAOS_ODBC_DRIVER'); + RegDeleteValue(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\ODBC Drivers', 'TAOS_ODBC_DRIVER'); + + // Delete 32-bit ODBC driver registry + RegDeleteKeyIncludingSubkeys(HKLM64, 'SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\TAOS_ODBC_DRIVER'); + RegDeleteValue(HKLM64, 'SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\ODBC Drivers', 'TAOS_ODBC_DRIVER'); + Result := True; end; From e226898e1751d35726a97209bc0b726536d5177e Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 11 Sep 2024 20:25:04 +0800 Subject: [PATCH 30/61] semphore --- include/os/osSemaphore.h | 30 ++++---- source/client/src/clientImpl.c | 4 - source/client/src/clientMonitor.c | 8 +- source/common/src/cos.c | 2 +- source/common/src/tmisce.c | 1 - source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgCache.c | 2 +- source/libs/executor/src/exchangeoperator.c | 1 - source/libs/transport/src/transCli.c | 8 +- source/os/src/osSemaphore.c | 85 ++++++++++++++++----- source/util/src/tsched.c | 8 +- 11 files changed, 93 insertions(+), 58 deletions(-) diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 5b46706790..d893f42740 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -27,11 +27,11 @@ extern "C" { // typedef struct tsem_s *tsem_t; typedef dispatch_semaphore_t tsem_t; -int tsem_init(tsem_t *sem, int pshared, unsigned int value); -int tsem_wait(tsem_t *sem); -int tsem_timewait(tsem_t *sim, int64_t milis); -int tsem_post(tsem_t *sem); -int tsem_destroy(tsem_t *sem); +int32_t tsem_init(tsem_t *sem, int pshared, unsigned int value); +int32_t tsem_wait(tsem_t *sem); +int32_t tsem_timewait(tsem_t *sim, int64_t milis); +int32_t tsem_post(tsem_t *sem); +int32_t tsem_destroy(tsem_t *sem); #define tsem2_t tsem_t #define tsem2_init tsem_init @@ -45,11 +45,11 @@ int tsem_destroy(tsem_t *sem); #define tsem_t HANDLE -int tsem_init(tsem_t *sem, int pshared, unsigned int value); -int tsem_wait(tsem_t *sem); -int tsem_timewait(tsem_t *sim, int64_t milis); -int tsem_post(tsem_t *sem); -int tsem_destroy(tsem_t *sem); +int32_t tsem_init(tsem_t *sem, int pshared, unsigned int value); +int32_t tsem_wait(tsem_t *sem); +int32_t tsem_timewait(tsem_t *sim, int64_t milis); +int32_t tsem_post(tsem_t *sem); +int32_t tsem_destroy(tsem_t *sem); #define tsem2_t tsem_t #define tsem2_init tsem_init @@ -61,11 +61,11 @@ int tsem_destroy(tsem_t *sem); #else #define tsem_t sem_t -#define tsem_init sem_init -int tsem_wait(tsem_t *sem); -int tsem_timewait(tsem_t *sim, int64_t milis); -#define tsem_post sem_post -#define tsem_destroy sem_destroy +int32_t tsem_init(tsem_t *sem, int pshared, unsigned int value); +int32_t tsem_wait(tsem_t *sem); +int32_t tsem_timewait(tsem_t *sim, int64_t milis); +int32_t tsem_post(tsem_t *sem); +int32_t tsem_destroy(tsem_t *sem); typedef struct tsem2_t { TdThreadMutex mutex; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d77b8dcbb7..000ec60718 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2871,7 +2871,6 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t s } int32_t code = tsem_init(¶m->sem, 0, 0); if (TSDB_CODE_SUCCESS != code) { - terrno = code; taosMemoryFree(param); return NULL; } @@ -2879,7 +2878,6 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t s taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, source); code = tsem_wait(¶m->sem); if (TSDB_CODE_SUCCESS != code) { - terrno = code; taosMemoryFree(param); return NULL; } @@ -2910,7 +2908,6 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, } int32_t code = tsem_init(¶m->sem, 0, 0); if (TSDB_CODE_SUCCESS != code) { - terrno = code; taosMemoryFree(param); return NULL; } @@ -2918,7 +2915,6 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid); code = tsem_wait(¶m->sem); if (TSDB_CODE_SUCCESS != code) { - terrno = code; taosMemoryFree(param); return NULL; } diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index b890b61668..90893ba106 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -34,19 +34,19 @@ static void processFileInTheEnd(TdFilePtr pFile, char* path) { return; } if (taosFtruncateFile(pFile, 0) != 0) { - tscError("failed to truncate file:%s, errno:%d", path, terrno); + tscError("failed to truncate file:%s, terrno:%d", path, terrno); return; } if (taosUnLockFile(pFile) != 0) { - tscError("failed to unlock file:%s, errno:%d", path, terrno); + tscError("failed to unlock file:%s, terrno:%d", path, terrno); return; } if (taosCloseFile(&(pFile)) != 0) { - tscError("failed to close file:%s, errno:%d", path, errno); + tscError("failed to close file:%s, terrno:%d", path, terrno); return; } if (taosRemoveFile(path) != 0) { - tscError("failed to remove file:%s, errno:%d", path, errno); + tscError("failed to remove file:%s, terrno:%d", path, terrno); return; } } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index d3b3f1f87d..82b1fad2ce 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -1412,7 +1412,7 @@ static int32_t s3GetObjectToFileByEp(const char *object_name, const char *fileNa TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName); + uError("[s3] open file error, terrno:%d, fileName:%s", terrno, fileName); TAOS_RETURN(terrno); } diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index f3f4b29617..e40db7b4cf 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -197,7 +197,6 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t code = taosGetAppName(tmp, NULL); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(code, NULL, _exit); } TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0b0cb6dc91..11e1b1221b 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -849,7 +849,7 @@ int32_t catalogInit(SCatalogCfg* cfg) { } if (tsem_init(&gCtgMgmt.queue.reqSem, 0, 0)) { - qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + qError("tsem_init failed, terror:%s", tstrerror(terrno)); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 01201a2480..fe353b0eda 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -3239,7 +3239,7 @@ void *ctgUpdateThreadFunc(void *param) { while (true) { if (tsem_wait(&gCtgMgmt.queue.reqSem)) { - qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + qError("ctg tsem_wait failed, error:%s", tstrerror(terrno)); } if (atomic_load_8((int8_t *)&gCtgMgmt.queue.stopQueue)) { diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 457aa8ff2b..5352ea37ae 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -565,7 +565,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pSourceDataInfo->status = EX_SOURCE_DATA_READY; code = tsem_post(&pExchangeInfo->ready); if (code != TSDB_CODE_SUCCESS) { - code = TAOS_SYSTEM_ERROR(code); qError("failed to invoke post when fetch rsp is ready, code:%s, %p", tstrerror(code), pExchangeInfo); return code; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 78320c450c..ea30f0c112 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3087,7 +3087,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra code = tsem_init(sem, 0, 0); if (code != 0) { taosMemoryFree(sem); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); @@ -3250,9 +3250,8 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, } code = tsem2_timewait(pSyncMsg->pSem, timeoutMs); - if (code < 0) { - pRsp->code = TSDB_CODE_TIMEOUT_ERROR; - code = TSDB_CODE_TIMEOUT_ERROR; + if (code != 0) { + pRsp->code = code; } else { memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); pSyncMsg->pRsp->pCont = NULL; @@ -3260,7 +3259,6 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, epsetAssign(pEpSet, &pSyncMsg->epSet); *epUpdated = 1; } - code = 0; } _RETURN: (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index e959174f11..4496ba011f 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -48,7 +48,10 @@ int32_t taosGetPId() { return GetCurrentProcessId(); } int32_t taosGetAppName(char* name, int32_t* len) { char filepath[1024] = {0}; - GetModuleFileName(NULL, filepath, MAX_PATH); + if (GetModuleFileName(NULL, filepath, MAX_PATH) == 0) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } char* sub = strrchr(filepath, '.'); if (sub != NULL) { *sub = '\0'; @@ -70,7 +73,12 @@ int32_t taosGetAppName(char* name, int32_t* len) { } int32_t tsem_wait(tsem_t* sem) { - return WaitForSingleObject(*sem, INFINITE); + DWORD ret = WaitForSingleObject(*sem, INFINITE); + if(ret == WAIT_OBJECT_0) { + return 0; + } else { + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + } } int32_t tsem_timewait(tsem_t* sem, int64_t timeout_ms) { @@ -78,61 +86,65 @@ int32_t tsem_timewait(tsem_t* sem, int64_t timeout_ms) { if (result == WAIT_OBJECT_0) { return 0; // Semaphore acquired } else if (result == WAIT_TIMEOUT) { - return -1; // Timeout reached + return TSDB_CODE_TIMEOUT_ERROR; // Timeout reached } else { - return result; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } } // Inter-process sharing is not currently supported. The pshared parameter is invalid. -int tsem_init(tsem_t* sem, int pshared, unsigned int value) { +int32_t tsem_init(tsem_t* sem, int pshared, unsigned int value) { *sem = CreateSemaphore(NULL, value, LONG_MAX, NULL); - return (*sem != NULL) ? 0 : -1; + return (*sem != NULL) ? 0 : TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } -int tsem_post(tsem_t* sem) { +int32_t tsem_post(tsem_t* sem) { if (ReleaseSemaphore(*sem, 1, NULL)) return 0; - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } -int tsem_destroy(tsem_t* sem) { +int32_t tsem_destroy(tsem_t* sem) { if (CloseHandle(*sem)) return 0; - return -1; + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } #elif defined(_TD_DARWIN_64) #include -int tsem_init(tsem_t *psem, int flags, unsigned int count) { +int32_t tsem_init(tsem_t *psem, int flags, unsigned int count) { *psem = dispatch_semaphore_create(count); - if (*psem == NULL) return -1; + if (*psem == NULL) return TAO_SYTAOS_SYSTEM_ERROR(errno); return 0; } -int tsem_destroy(tsem_t *psem) { - if (psem == NULL || *psem == NULL) return -1; +int32_t tsem_destroy(tsem_t *psem) { + // if (psem == NULL || *psem == NULL) return -1; // dispatch_release(*psem); // *psem = NULL; return 0; } -int tsem_post(tsem_t *psem) { +int32_t tsem_post(tsem_t *psem) { if (psem == NULL || *psem == NULL) return -1; - dispatch_semaphore_signal(*psem); + (void)dispatch_semaphore_signal(*psem); return 0; } -int tsem_wait(tsem_t *psem) { +int32_t tsem_wait(tsem_t *psem) { if (psem == NULL || *psem == NULL) return -1; dispatch_semaphore_wait(*psem, DISPATCH_TIME_FOREVER); return 0; } -int tsem_timewait(tsem_t *psem, int64_t milis) { +int32_t tsem_timewait(tsem_t *psem, int64_t milis) { if (psem == NULL || *psem == NULL) return -1; dispatch_time_t time = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(milis * USEC_PER_SEC)); - return dispatch_semaphore_wait(*psem, time); + if(dispatch_semaphore_wait(*psem, time) == 0) { + return 0; + } else { + return TSDB_CODE_TIMEOUT_ERROR; + } } bool taosCheckPthreadValid(TdThread thread) { return thread != 0; } @@ -216,6 +228,14 @@ int32_t taosGetAppName(char* name, int32_t* len) { return 0; } +int32_t tsem_init(tsem_t *psem, int flags, unsigned int count) { + if(sem_init(psem, flags, count) == 0) { + return 0; + } else { + return TAOS_SYSTEM_ERROR(errno); + } +} + int32_t tsem_timewait(tsem_t* sem, int64_t ms) { int ret = 0; @@ -233,13 +253,16 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) { while ((ret = sem_timedwait(sem, &ts)) == -1 && errno == EINTR) { continue; } + if(ETIMEDOUT == errno) { + return TSDB_CODE_TIMEOUT_ERROR; + } if (-1 == ret) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return ret; + return 0; } int32_t tsem_wait(tsem_t* sem) { @@ -285,6 +308,22 @@ int tsem2_init(tsem2_t* sem, int pshared, unsigned int value) { return 0; } +int32_t tsem_post(tsem_t* psem) { + if (sem_post(psem) == 0) { + return 0; + } else { + return TAOS_SYSTEM_ERROR(errno); + } +} + +int32_t tsem_destroy(tsem_t *sem) { + if (sem_destroy(sem) == 0) { + return 0; + } else { + return TAOS_SYSTEM_ERROR(errno); + } +} + int tsem2_post(tsem2_t *sem) { int32_t code = taosThreadMutexLock(&sem->mutex); if (code) { @@ -364,7 +403,11 @@ int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) { ret = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts); if (ret != 0) { (void)taosThreadMutexUnlock(&sem->mutex); - return ret; + if (errno == ETIMEDOUT) { + return TSDB_CODE_TIMEOUT_ERROR; + } else { + return TAOS_SYSTEM_ERROR(errno); + } } } } diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 34c74660fc..1686b41038 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -150,7 +150,7 @@ void *taosProcessSchedQueue(void *scheduler) { while (1) { if ((ret = tsem_wait(&pSched->fullSem)) != 0) { - uFatal("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); + uFatal("wait %s fullSem failed(%s)", pSched->label, strerror(terrno)); } if (atomic_load_8(&pSched->stop)) { break; @@ -169,7 +169,7 @@ void *taosProcessSchedQueue(void *scheduler) { } if ((ret = tsem_post(&pSched->emptySem)) != 0) { - uFatal("post %s emptySem failed(%s)", pSched->label, strerror(errno)); + uFatal("post %s emptySem failed(%s)", pSched->label, strerror(terrno)); } if (msg.fp) @@ -197,7 +197,7 @@ int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { } if ((ret = tsem_wait(&pSched->emptySem)) != 0) { - uFatal("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); + uFatal("wait %s emptySem failed(%s)", pSched->label, strerror(terrno)); } if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { @@ -212,7 +212,7 @@ int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { } if ((ret = tsem_post(&pSched->fullSem)) != 0) { - uFatal("post %s fullSem failed(%s)", pSched->label, strerror(errno)); + uFatal("post %s fullSem failed(%s)", pSched->label, strerror(terrno)); } return ret; } From dca6b21e88aef06a0495c4298b169f031c774304 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 12 Sep 2024 09:34:53 +0800 Subject: [PATCH 31/61] fix: timewait --- source/os/src/osSemaphore.c | 21 ++++++++++----------- source/os/test/osSemaphoreTests.cpp | 16 ++++++++++++---- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 4496ba011f..f6d339c89e 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -114,7 +114,7 @@ int32_t tsem_destroy(tsem_t* sem) { int32_t tsem_init(tsem_t *psem, int flags, unsigned int count) { *psem = dispatch_semaphore_create(count); - if (*psem == NULL) return TAO_SYTAOS_SYSTEM_ERROR(errno); + if (*psem == NULL) return TAOS_SYSTEM_ERROR(errno); return 0; } @@ -250,16 +250,15 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) { ts.tv_sec += ts.tv_nsec / 1000000000; ts.tv_nsec %= 1000000000; - while ((ret = sem_timedwait(sem, &ts)) == -1 && errno == EINTR) { - continue; - } - if(ETIMEDOUT == errno) { - return TSDB_CODE_TIMEOUT_ERROR; - } - - if (-1 == ret) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + while ((ret = sem_timedwait(sem, &ts)) == -1) { + if(errno == EINTR) { + continue; + } else if(errno == ETIMEDOUT) { + return TSDB_CODE_TIMEOUT_ERROR; + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } } return 0; diff --git a/source/os/test/osSemaphoreTests.cpp b/source/os/test/osSemaphoreTests.cpp index 9347254b10..1576d2845d 100644 --- a/source/os/test/osSemaphoreTests.cpp +++ b/source/os/test/osSemaphoreTests.cpp @@ -59,18 +59,26 @@ TEST(osSemaphoreTests, Destroy) { TEST(osSemaphoreTests, WaitTime0) { tsem_t sem; (void)tsem_init(&sem, 0, 0); - EXPECT_NE(tsem_timewait(&sem, 1000), 0); + EXPECT_EQ(tsem_timewait(&sem, 1000), TSDB_CODE_TIMEOUT_ERROR); (void)tsem_destroy(&sem); } TEST(osSemaphoreTests, WaitTime1) { tsem_t sem; (void)tsem_init(&sem, 0, 1); - EXPECT_EQ(tsem_timewait(&sem, 1000), 0); - EXPECT_NE(tsem_timewait(&sem, 1000), 0); + EXPECT_EQ(tsem_timewait(&sem, 10000), 0); + EXPECT_EQ(tsem_timewait(&sem, 1000), TSDB_CODE_TIMEOUT_ERROR); (void)tsem_destroy(&sem); } +TEST(osSemaphoreTests, WaitTime2) { + tsem2_t sem; + (void)tsem2_init(&sem, 0, 1); + EXPECT_EQ(tsem2_timewait(&sem, 10000), 0); + EXPECT_EQ(tsem2_timewait(&sem, 1000), TSDB_CODE_TIMEOUT_ERROR); + (void)tsem2_destroy(&sem); +} + TEST(osSemaphoreTests, WaitAndPost) { tsem_t sem; int result = tsem_init(&sem, 0, 0); @@ -217,7 +225,7 @@ TEST(osSemaphoreTests, Performance4_1) { (void)tsem_post(&sem); }).detach(); - (void)tsem_timewait(&sem, 1000); + EXPECT_EQ(tsem_timewait(&sem, 1000),0); (void)tsem_destroy(&sem); } From f9d38ef7bae2dc05f3187b95d5e194d05387d512 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Thu, 12 Sep 2024 09:36:37 +0800 Subject: [PATCH 32/61] fix:[TD-32041] Fix rand function wrong behaviour. --- source/libs/scalar/src/sclfunc.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4e8107a5b2..3b9cad68ad 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2888,13 +2888,16 @@ int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut } int32_t randFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (!IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0]))) { - int32_t seed; - GET_TYPED_DATA(seed, int32_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData); - taosSeedRand(seed); - } + int32_t seed; int32_t numOfRows = inputNum == 1 ? pInput[0].numOfRows : TMAX(pInput[0].numOfRows, pInput[1].numOfRows); for (int32_t i = 0; i < numOfRows; ++i) { + // for constant seed, only set seed once + if ((pInput[0].numOfRows == 1 && i == 0) || (pInput[0].numOfRows != 1)) { + if (!IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) && !colDataIsNull_s(pInput[0].columnData, i)) { + GET_TYPED_DATA(seed, int32_t, GET_PARAM_TYPE(&pInput[0]), colDataGetData(pInput[0].columnData, i)); + taosSeedRand(seed); + } + } double random_value = (double)(taosRand() % RAND_MAX) / RAND_MAX; colDataSetDouble(pOutput->columnData, i, &random_value); } From 28b3c92dddd0f9d3f91ef3b77e8d051e5abb2494 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 02:36:07 +0000 Subject: [PATCH 33/61] fix/TS-5404-use-write-lock --- source/dnode/mnode/impl/src/mndTrans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 724c9637e6..43f8ffe61b 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1032,7 +1032,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); - sdbReadLock(pMnode->pSdb, SDB_TRANS); + sdbWriteLock(pMnode->pSdb, SDB_TRANS); tsMaxTransId = TMAX(pTrans->id, tsMaxTransId); sdbUnLock(pMnode->pSdb, SDB_TRANS); TAOS_RETURN(code); From 41dfc643b768f9ca1e8538a3b5cd6512f8737dda Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Thu, 12 Sep 2024 13:45:17 +0800 Subject: [PATCH 34/61] enh:modify error code pass --- source/client/src/clientHb.c | 16 +++--- source/client/src/clientImpl.c | 29 +++++------ source/client/src/clientMain.c | 2 +- source/client/src/clientMsgHandler.c | 30 +++++------ source/client/src/clientSml.c | 74 ++++++++++++++-------------- source/client/src/clientSmlJson.c | 8 +-- source/client/src/clientSmlLine.c | 16 +++--- source/client/src/clientSmlTelnet.c | 2 +- source/client/src/clientStmt.c | 33 ++++++------- source/client/src/clientStmt2.c | 25 +++++----- source/client/src/clientTmq.c | 22 ++++----- 11 files changed, 126 insertions(+), 131 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 47b80c03d1..8297ac362e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -180,7 +180,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == vgInfo->vgHash) { tscError("hash init[%d] failed", rsp->vgNum); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } @@ -188,7 +188,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { tscError("hash push failed, errno:%d", errno); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -689,7 +689,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { (void)releaseRequest(*rid); if (NULL == taosArrayPush(hbBasic->queryDesc, &desc)) { taosArrayDestroy(desc.subDesc); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pIter = taosHashIterate(pObj->pRequests, pIter); @@ -777,7 +777,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient pKv->value = qUserAuth; pKv->valueLen += sizeof(SUserAuthVersion); } else { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } goto _return; } @@ -785,7 +785,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient // key/user not exist, add user SUserAuthVersion *user = taosMemoryMalloc(sizeof(SUserAuthVersion)); if (!user) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } tstrncpy(user->user, pTscObj->user, TSDB_USER_LEN); @@ -1436,7 +1436,7 @@ int32_t appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key, SAppHbMgr **pAppHbMg TSC_ERR_RET(hbMgrInit()); *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (*pAppHbMgr == NULL) { - TSC_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + TSC_ERR_JRET(terrno); } // init stat (*pAppHbMgr)->startTime = taosGetTimestampMs(); @@ -1456,14 +1456,14 @@ int32_t appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key, SAppHbMgr **pAppHbMg (*pAppHbMgr)->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); if ((*pAppHbMgr)->activeInfo == NULL) { - TSC_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + TSC_ERR_JRET(terrno); } // taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq); TSC_ERR_JRET(taosThreadMutexLock(&clientHbMgr.lock)); if (taosArrayPush(clientHbMgr.appHbMgrs, &(*pAppHbMgr)) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; (void)taosThreadMutexUnlock(&clientHbMgr.lock); goto _return; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d77b8dcbb7..484d9bb7e5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -228,7 +228,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen); @@ -247,7 +247,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*pRequest)->allocatorRefId = -1; @@ -258,7 +258,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -595,7 +595,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr if (NULL == taosArrayPush(nodeList, &load)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } @@ -619,7 +619,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } if (NULL == taosArrayAddBatch(nodeList, pData, mnodeNum)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tscDebug("0x%" PRIx64 " %s policy, use mnode list, num:%d", pRequest->requestId, policy, mnodeNum); @@ -646,7 +646,7 @@ int32_t buildQnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } if (NULL == taosArrayAddBatch(nodeList, pData, qNodeNum)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tscDebug("0x%" PRIx64 " qnode policy, use qnode list, num:%d", pRequest->requestId, qNodeNum); goto _return; @@ -665,7 +665,7 @@ int32_t buildQnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } if (NULL == taosArrayAddBatch(nodeList, pData, mnodeNum)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tscDebug("0x%" PRIx64 " qnode policy, use mnode list, num:%d", pRequest->requestId, mnodeNum); @@ -699,7 +699,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } if (NULL == taosArrayPush(pDbVgList, &pRes->pRes)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -791,7 +791,7 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* } if (NULL == taosArrayPush(pDbVgList, &pVgList)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -903,8 +903,7 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, pArray = taosArrayInit(tbNum, sizeof(STbSVersion)); if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t i = 0; i < tbNum; ++i) { @@ -915,7 +914,7 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, } STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion, .tver = tbInfo->tversion}; if (NULL == taosArrayPush(pArray, &tbSver)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -1662,7 +1661,7 @@ static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInf void* pReq = taosMemoryMalloc(contLen); if (NULL == pReq) { taosMemoryFree(*pMsgSendInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (-1 == tSerializeSConnectReq(pReq, contLen, &connectReq)) { @@ -2050,7 +2049,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { taosReleaseConv(idx, conv, C2M); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pResultInfo->convertBuf[i] = p; @@ -2441,7 +2440,6 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR if (pResultInfo->decompBuf == NULL) { pResultInfo->decompBuf = taosMemoryMalloc(payloadLen); if (pResultInfo->decompBuf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to prepare the decompress buffer, size:%d", payloadLen); return terrno; } @@ -2450,7 +2448,6 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR if (pResultInfo->decompBufSize < payloadLen) { char* p = taosMemoryRealloc(pResultInfo->decompBuf, payloadLen); if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to prepare the decompress buffer, size:%d", payloadLen); return terrno; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1c1fff9b7b..4cd8afe880 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -216,7 +216,7 @@ int32_t fetchWhiteListCallbackFn(void *param, SDataBuf *pMsg, int32_t code) { taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pInfo); tFreeSGetUserWhiteListRsp(&wlRsp); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int i = 0; i < wlRsp.numWhiteLists; ++i) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 54020b77ef..f8d443aaad 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -511,19 +511,19 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(SHOW_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); - TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, terrno); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD1_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD2_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD3_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); int32_t numOfCfg = taosArrayGetSize(pVars); code = blockDataEnsureCapacity(pBlock, numOfCfg); @@ -531,26 +531,26 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { SVariablesInfo* pInfo = taosArrayGet(pVars, i); - TSDB_CHECK_NULL(pInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pInfo, code, line, END, terrno); char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - TSDB_CHECK_NULL(pColInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); code = colDataSetVal(pColInfo, i, name, false); TSDB_CHECK_CODE(code, line, END); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - TSDB_CHECK_NULL(pColInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); code = colDataSetVal(pColInfo, i, value, false); TSDB_CHECK_CODE(code, line, END); char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(scope, pInfo->scope, TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - TSDB_CHECK_NULL(pColInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); code = colDataSetVal(pColInfo, i, scope, false); TSDB_CHECK_CODE(code, line, END); } @@ -662,29 +662,29 @@ static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) { pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData)); - TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, terrno); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = COMPACT_DB_RESULT_FIELD1_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = COMPACT_DB_RESULT_FIELD3_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); code = blockDataEnsureCapacity(pBlock, 1); TSDB_CHECK_CODE(code, line, END); SColumnInfoData* pResultCol = taosArrayGet(pBlock->pDataBlock, 0); - TSDB_CHECK_NULL(pResultCol, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pResultCol, code, line, END, terrno); SColumnInfoData* pIdCol = taosArrayGet(pBlock->pDataBlock, 1); - TSDB_CHECK_NULL(pIdCol, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pIdCol, code, line, END, terrno); SColumnInfoData* pReasonCol = taosArrayGet(pBlock->pDataBlock, 2); - TSDB_CHECK_NULL(pReasonCol, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pReasonCol, code, line, END, terrno); char result[COMPACT_DB_RESULT_FIELD1_LEN] = {0}; char reason[COMPACT_DB_RESULT_FIELD3_LEN] = {0}; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 05678e1cbf..fa6a00b197 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -178,7 +178,7 @@ int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, if (tag->cols == NULL) { uError("SML:smlBuildTableInfo failed to allocate memory"); taosMemoryFree(tag); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *tInfo = tag; @@ -210,7 +210,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); if (measure == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); @@ -251,11 +251,11 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (i < pTableMeta->tableInfo.numOfColumns) { if(taosArrayPush((*sMeta)->cols, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else { if(taosArrayPush((*sMeta)->tags, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } @@ -328,7 +328,7 @@ END: int32_t smlJoinMeasureTag(SSmlLineInfo *elements) { elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); if (elements->measureTag == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(elements->measureTag, elements->measure, elements->measureLen); (void)memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); @@ -455,14 +455,14 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); if (elements->colArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } if (taosArrayPush(elements->colArray, kvTs) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(elements->colArray, kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } info->preLine = *elements; @@ -937,7 +937,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (hashTmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } int32_t i = 0; @@ -997,7 +997,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO field.bytes = getBytes(kv->type, kv->length); (void)memcpy(field.name, kv->key, kv->keyLen); if (taosArrayPush(results, &field) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); @@ -1046,7 +1046,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); if (pReq.pColumns == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (int32_t i = 0; i < pReq.numOfColumns; ++i) { @@ -1059,7 +1059,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1106,7 +1106,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); if (taosArrayPush(pReq.pTags, &field) == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1124,7 +1124,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -1185,7 +1185,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); if (measure == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } (void)memcpy(measure, superTable, superTableLen); @@ -1205,12 +1205,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); if (pColumns == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); if (pTags == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosArrayDestroy(pColumns); goto end; } @@ -1252,7 +1252,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (hashTmp == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (uint16_t i = pTableMeta->tableInfo.numOfColumns; @@ -1278,14 +1278,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); if (pColumns == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); if (pTags == NULL){ taosArrayDestroy(pColumns); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { @@ -1297,14 +1297,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (taosArrayPush(pColumns, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } else { if (taosArrayPush(pTags, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1374,14 +1374,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); if (pColumns == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); if (pTags == NULL){ taosArrayDestroy(pColumns); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { @@ -1393,14 +1393,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (taosArrayPush(pColumns, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } else { if (taosArrayPush(pTags, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1497,7 +1497,7 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { if (taosArrayPush(metaArray, kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; @@ -1547,7 +1547,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { if(taosArrayPush(metaArray, kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; @@ -1646,7 +1646,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { uError("create SSmlHandle hash obj failed"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAILED; } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); @@ -1665,7 +1665,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAILED; } @@ -1681,7 +1681,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (!kvHash) { uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1703,7 +1703,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { if (taosArrayPush(colsArray, &kvHash) == NULL) { taosHashCleanup(kvHash); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; } @@ -1799,12 +1799,12 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); if (info->pRequest->dbList == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); if (data == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -1817,7 +1817,7 @@ static int32_t smlInsertData(SSmlHandle *info) { int measureLen = tableData->sTableNameLen; char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); if (measure == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); @@ -1828,11 +1828,11 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); if (info->pRequest->tableList == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } if (taosArrayPush(info->pRequest->tableList, &pName) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 10ff234552..ece1ddf61f 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -526,7 +526,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ return ret; } if (taosArrayPush(preLineKV, &kv) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { @@ -873,7 +873,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ cJSON_Delete(valueJson); elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ret = smlParseValueFromJSONObj(valueJson, &kv); if (ret != TSDB_CODE_SUCCESS) { @@ -901,7 +901,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ cJSON_Delete(tagsJson); uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ret = smlParseTagsFromJSON(info, tagsJson, elements); if (unlikely(ret)) { @@ -965,7 +965,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { payloadNum = payloadNum << 1; void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); if (tmp == NULL) { - ret = TSDB_CODE_OUT_OF_MEMORY; + ret = terrno; return ret; } info->lines = (SSmlLineInfo *)tmp; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 14a334e54c..e620ca9b0c 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -150,7 +150,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } void *data = taosMemoryMalloc(pVal->length); if(data == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); pVal->value = data; @@ -271,7 +271,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(keyLen); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); @@ -280,7 +280,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(valueLen); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, value, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); @@ -294,7 +294,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; if(taosArrayPush(preLineKV, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { @@ -422,7 +422,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.keyLen); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, key, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); @@ -433,7 +433,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.length); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); @@ -459,11 +459,11 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); if (currElement->colArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index a25ef4ec95..e8601e33bc 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -132,7 +132,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .keyEscaped = false, .valueEscaped = false}; if (taosArrayPush(preLineKV, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { return TSDB_CODE_SML_INVALID_DATA; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 52b56abb91..175628cc7d 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -15,7 +15,7 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void** } else if (pTblBuf->buffIdx < taosArrayGetSize(pTblBuf->pBufList)) { pTblBuf->pCurBuff = taosArrayGetP(pTblBuf->pBufList, pTblBuf->buffIdx++); if (NULL == pTblBuf->pCurBuff) { - return TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); + return TAOS_GET_TERRNO(terrno); } *pBuf = pTblBuf->pCurBuff; pTblBuf->buffOffset = pTblBuf->buffUnit; @@ -186,7 +186,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { pRes->fields = taosMemoryMalloc(size); pRes->userFields = taosMemoryMalloc(size); if (NULL == pRes->fields || NULL == pRes->userFields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); (void)memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); @@ -204,7 +204,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); } @@ -212,7 +212,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); } @@ -306,7 +306,7 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { }; if (taosHashPut(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid), &cache, sizeof(cache))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (pStmt->sql.autoCreateTbl) { @@ -369,7 +369,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { if (NULL == pStmt->sql.pBindInfo) { pStmt->sql.pBindInfo = taosMemoryMalloc(pTableCtx->boundColsInfo.numOfBound * sizeof(*pStmt->sql.pBindInfo)); if (NULL == pStmt->sql.pBindInfo) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -612,7 +612,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } pStmt->exec.pCurrBlock = pNewBlock; @@ -702,7 +702,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } pStmt->exec.pCurrBlock = pNewBlock; @@ -722,7 +722,6 @@ int32_t stmtResetStmt(STscStmt* pStmt) { pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_OUT_OF_MEMORY; STMT_ERR_RET(terrno); } @@ -739,7 +738,7 @@ int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) { SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i); *p = taosArrayInit(20, POINTER_BYTES); if (*p == NULL) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } } @@ -812,15 +811,15 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { pTblBuf->buffSize = pTblBuf->buffUnit * 1000; pTblBuf->pBufList = taosArrayInit(100, POINTER_BYTES); if (NULL == pTblBuf->pBufList) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } void* buff = taosMemoryMalloc(pTblBuf->buffSize); if (NULL == buff) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pTblBuf->pBufList, &buff) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pTblBuf->pCurBuff = buff; @@ -948,11 +947,11 @@ int32_t stmtInitStbInterlaceTableInfo(STscStmt* pStmt) { for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -1184,11 +1183,11 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray** for (int32_t i = 0; i < 100; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index e7e08c0982..01a7c4be4c 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -643,7 +643,6 @@ static int32_t stmtResetStmt(STscStmt2* pStmt) { pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_OUT_OF_MEMORY; STMT_ERR_RET(terrno); } @@ -660,7 +659,7 @@ static int32_t stmtAsyncOutput(STscStmt2* pStmt, void* param) { SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i); *p = taosArrayInit(20, POINTER_BYTES); if (*p == NULL) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } } @@ -733,15 +732,15 @@ static int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { pTblBuf->buffSize = pTblBuf->buffUnit * 1000; pTblBuf->pBufList = taosArrayInit(100, POINTER_BYTES); if (NULL == pTblBuf->pBufList) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } void* buff = taosMemoryMalloc(pTblBuf->buffSize); if (NULL == buff) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pTblBuf->pBufList, &buff) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pTblBuf->pCurBuff = buff; @@ -796,7 +795,7 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { } pStmt->sql.siInfo.pTableCols = taosArrayInit(STMT_TABLE_COLS_NUM, POINTER_BYTES); if (NULL == pStmt->sql.siInfo.pTableCols) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; (void)stmtClose(pStmt); return NULL; } @@ -889,11 +888,11 @@ static int32_t stmtInitStbInterlaceTableInfo(STscStmt2* pStmt) { for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -1121,11 +1120,11 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt2* pStmt, SArray** for (int32_t i = 0; i < 100; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } @@ -1160,7 +1159,7 @@ static int32_t stmtCacheBlock(STscStmt2* pStmt) { }; if (taosHashPut(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid), &cache, sizeof(cache))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (pStmt->sql.autoCreateTbl) { @@ -1554,7 +1553,7 @@ static int32_t createParseContext(const SRequestObj* pRequest, SParseContext** p *pCxt = taosMemoryCalloc(1, sizeof(SParseContext)); if (*pCxt == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } **pCxt = (SParseContext){.requestId = pRequest->requestId, @@ -1671,7 +1670,7 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { } else { SSqlCallbackWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); if (pWrapper == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } else { pWrapper->pRequest = pRequest; pRequest->pWrapper = pWrapper; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 783815d97f..cd443f88e5 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1094,7 +1094,7 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { if (*topics == NULL) { *topics = tmq_list_new(); if (*topics == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } taosRLockLatch(&tmq->lock); @@ -1353,7 +1353,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAIL; } @@ -1394,7 +1394,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } if (taosArrayPush(req.topicNames, &topicFName) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFree(topicFName); goto FAIL; } @@ -1404,7 +1404,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req); buf = taosMemoryMalloc(tlen); if (buf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAIL; } @@ -2017,7 +2017,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFreeClear(msg); return code; } @@ -2713,7 +2713,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { tscError("failed to allocate memory for sync commit"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(&pInfo->sem, 0, 0) != 0) { tscError("failed to init sem for sync commit"); @@ -2785,7 +2785,7 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { tscError("consumer:0x%" PRIx64 " failed to prepare seek operation", tmq->consumerId); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(&pInfo->sem, 0, 0) != 0) { @@ -2922,7 +2922,7 @@ FAIL: int32_t syncAskEp(tmq_t* pTmq) { SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); - if (pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pInfo == NULL) return terrno; if (tsem2_init(&pInfo->sem, 0, 0) != 0) { taosMemoryFree(pInfo); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -3427,7 +3427,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a pCommon->pList = taosArrayInit(4, sizeof(tmq_topic_assignment)); if (pCommon->pList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } if (tsem2_init(&pCommon->rsp, 0, 0) != 0) { @@ -3445,7 +3445,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); if (pParam == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -3649,7 +3649,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ if (pParam == NULL) { taosMemoryFree(msg); taosMemoryFree(sendInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(&pParam->sem, 0, 0) != 0) { taosMemoryFree(msg); From 2c53c08726de3fb3171437a027ca2fcd54831533 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 12 Sep 2024 11:31:10 +0800 Subject: [PATCH 35/61] fix(stream):fix issue when ts less than 0 --- source/libs/stream/src/streamUpdate.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 40de393570..9a1a30ac7b 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -155,7 +155,7 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b } pInfo->pTsBuckets = NULL; pInfo->pTsSBFs = NULL; - pInfo->minTS = -1; + pInfo->minTS = INT64_MIN; pInfo->interval = adjustInterval(interval, precision); pInfo->watermark = adjustWatermark(pInfo->interval, interval, watermark); pInfo->numSBFs = 0; @@ -181,7 +181,7 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b QUERY_CHECK_CODE(code, lino, _end); } - TSKEY dumy = 0; + TSKEY dumy = INT64_MIN; for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) { void* tmp = taosArrayPush(pInfo->pTsBuckets, &dumy); if (!tmp) { @@ -231,11 +231,7 @@ _end: static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - if (ts < 0) { - code = TSDB_CODE_FAILED; - QUERY_CHECK_CODE(code, lino, _end); - } - if (pInfo->minTS < 0) { + if (pInfo->minTS == INT64_MIN) { pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval); } int64_t index = (int64_t)((ts - pInfo->minTS) / pInfo->interval); @@ -349,7 +345,7 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); - if (ts < maxTs - pInfo->watermark) { + if (ts < maxTs - pInfo->watermark && maxTs != INT64_MIN) { // this window has been closed. if (pInfo->pCloseWinSBF) { code = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen, &res); From 78e9bb95ac2d6f2f9788f9aadb2f0ca740672961 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 12 Sep 2024 14:02:19 +0800 Subject: [PATCH 36/61] osString --- include/os/osString.h | 5 +- source/client/src/clientTmq.c | 34 +++++++++----- source/libs/index/src/indexComm.c | 48 ++++++++++++------- source/libs/index/test/utilUT.cc | 8 +++- source/libs/parser/src/parInsertSml.c | 10 ++-- source/libs/parser/src/parInsertSql.c | 8 ++-- source/libs/parser/src/parInsertStmt.c | 20 ++++---- source/libs/parser/src/parUtil.c | 2 +- source/libs/parser/src/parser.c | 4 +- source/os/src/osString.c | 65 +++++++++++++++++++++++--- 10 files changed, 144 insertions(+), 60 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index 80755de031..a64fb34f1e 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -65,7 +65,10 @@ typedef enum { M2C = 0, C2M } ConvType; char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); -int64_t taosStr2int64(const char *str); +int32_t taosStr2int64(const char *str, int64_t *val); +int32_t taosStr2int16(const char *str, int16_t *val); +int32_t taosStr2int32(const char *str, int32_t *val); +int32_t taosStr2int8(const char *str, int8_t *val); int32_t taosConvInit(void); void taosConvDestroy(); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 783815d97f..73aab57ed6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -298,6 +298,7 @@ void tmq_conf_destroy(tmq_conf_t* conf) { } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + int32_t code = 0; if (conf == NULL || key == NULL || value == NULL) { return TMQ_CONF_INVALID; } @@ -324,8 +325,9 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "auto.commit.interval.ms") == 0) { - int64_t tmp = taosStr2int64(value); - if (tmp < 0 || EINVAL == errno || ERANGE == errno) { + int64_t tmp; + code = taosStr2int64(value, &tmp); + if (tmp < 0 || code != 0) { return TMQ_CONF_INVALID; } conf->autoCommitInterval = (tmp > INT32_MAX ? INT32_MAX : tmp); @@ -333,8 +335,9 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "session.timeout.ms") == 0) { - int64_t tmp = taosStr2int64(value); - if (tmp < 6000 || tmp > 1800000) { + int64_t tmp; + code = taosStr2int64(value, &tmp); + if (tmp < 6000 || tmp > 1800000 || code != 0) { return TMQ_CONF_INVALID; } conf->sessionTimeoutMs = tmp; @@ -342,8 +345,9 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "heartbeat.interval.ms") == 0) { - int64_t tmp = taosStr2int64(value); - if (tmp < 1000 || tmp >= conf->sessionTimeoutMs) { + int64_t tmp; + code = taosStr2int64(value, &tmp); + if (tmp < 1000 || tmp >= conf->sessionTimeoutMs || code != 0) { return TMQ_CONF_INVALID; } conf->heartBeatIntervalMs = tmp; @@ -351,8 +355,9 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "max.poll.interval.ms") == 0) { - int64_t tmp = taosStr2int64(value); - if (tmp < 1000 || tmp > INT32_MAX) { + int32_t tmp; + code = taosStr2int32(value, &tmp); + if (tmp < 1000 || code != 0) { return TMQ_CONF_INVALID; } conf->maxPollIntervalMs = tmp; @@ -414,8 +419,9 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "td.connect.port") == 0) { - int64_t tmp = taosStr2int64(value); - if (tmp <= 0 || tmp > 65535) { + int64_t tmp; + code = taosStr2int64(value, &tmp); + if (tmp <= 0 || tmp > 65535 || code != 0) { return TMQ_CONF_INVALID; } @@ -435,7 +441,9 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } if (strcasecmp(key, "msg.consume.excluded") == 0) { - conf->sourceExcluded = (taosStr2int64(value) != 0) ? TD_REQ_FROM_TAOX : 0; + int64_t tmp; + code = taosStr2int64(value, &tmp); + conf->sourceExcluded = (0 == code && tmp != 0) ? TD_REQ_FROM_TAOX : 0; return TMQ_CONF_OK; } @@ -444,7 +452,9 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "msg.enable.batchmeta") == 0) { - conf->enableBatchMeta = (taosStr2int64(value) != 0) ? true : false; + int64_t tmp; + code = taosStr2int64(value, &tmp); + conf->enableBatchMeta = (0 == code && tmp != 0) ? true : false; return TMQ_CONF_OK; } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 2c681a1df3..8b2e6002aa 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -132,36 +132,52 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d } #if 1 if (dtype == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t va = taosStr2int64(a); - int64_t vb = taosStr2int64(b); + int64_t va; + taosStr2int64(a, &va); + int64_t vb; + taosStr2int64(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) { - uint8_t va = taosStr2int64(a); - uint8_t vb = taosStr2int64(b); + uint8_t va; + taosStr2int8(a, &va); + uint8_t vb; + taosStr2int8(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_TINYINT) { - int8_t va = taosStr2int64(a); - int8_t vb = taosStr2int64(b); + int8_t va; + taosStr2int8(a, &va); + int8_t vb; + taosStr2int8(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_SMALLINT) { - int16_t va = taosStr2int64(a); - int16_t vb = taosStr2int64(b); + int16_t va; + taosStr2int16(a, &va); + int16_t vb; + taosStr2int16(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_USMALLINT) { - uint16_t va = taosStr2int64(a); - uint16_t vb = taosStr2int64(b); + uint16_t va; + taosStr2int16(a, &va); + uint16_t vb; + taosStr2int16(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_INT) { - int32_t va = taosStr2int64(a); - int32_t vb = taosStr2int64(b); + int32_t va; + taosStr2int32(a, &va); + int32_t vb; + taosStr2int32(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_UINT) { - uint32_t va = taosStr2int64(a); - uint32_t vb = taosStr2int64(b); + uint32_t va; + taosStr2int32(a, &va); + uint32_t vb; + taosStr2int32(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_BIGINT) { - int64_t va = taosStr2int64(a); - int64_t vb = taosStr2int64(b); + int64_t va; + taosStr2int64(a, &va); + int64_t vb; + taosStr2int64(b, &vb); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_UBIGINT) { uint64_t va, vb; diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index 323a6b4afa..299b62b6fb 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -328,13 +328,17 @@ TEST_F(UtilEnv, testFill) { int64_t val = i; char buf[65] = {0}; idxInt2str(val, buf, 1); - EXPECT_EQ(val, taosStr2int64(buf)); + int64_t ret = 0; + taosStr2int64(buf, &ret); + EXPECT_EQ(val, ret); } for (int i = 0; i < 1000000; i++) { int64_t val = 0 - i; char buf[65] = {0}; idxInt2str(val, buf, -1); - EXPECT_EQ(val, taosStr2int64(buf)); + int64_t ret = 0; + taosStr2int64(buf, &ret); + EXPECT_EQ(val, ret); } } TEST_F(UtilEnv, TempResult) { diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 23025dcab9..929ec01f6a 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -143,13 +143,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem goto end; } if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); goto end; } char buf[512] = {0}; - (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(terrno)); taosMemoryFree(p); code = buildSyntaxErrMsg(msg, buf, kv->value); goto end; @@ -257,7 +257,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 goto end; } if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(pUcs4); ret = TSDB_CODE_PAR_VALUE_TOO_LONG; goto end; @@ -404,14 +404,14 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc goto end; } if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, pColSchema->bytes, kv->value); (void)buildInvalidOperationMsg(&pBuf, "value too long"); ret = TSDB_CODE_PAR_VALUE_TOO_LONG; goto end; } - ret = buildInvalidOperationMsg(&pBuf, strerror(errno)); + ret = buildInvalidOperationMsg(&pBuf, strerror(terrno)); goto end; } pVal->value.pData = pUcs4; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 662e24a679..d15d305c95 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -682,12 +682,12 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, return TSDB_CODE_OUT_OF_MEMORY; } if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(terrno)); taosMemoryFree(p); return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z); } @@ -1616,11 +1616,11 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, } if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len)) { taosMemoryFree(pUcs4); - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); + snprintf(buf, tListLen(buf), "%s", strerror(terrno)); return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); } pVal->value.pData = pUcs4; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index b67b7fc583..ff701e7fa2 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -194,13 +194,13 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); goto end; } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(terrno)); taosMemoryFree(p); code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); goto end; @@ -274,11 +274,11 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); + snprintf(buf, tListLen(buf), "%s", strerror(terrno)); return buildSyntaxErrMsg(pMsgBuf, buf, NULL); } @@ -534,13 +534,13 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c goto end; } if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); goto end; } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(terrno)); taosMemoryFree(p); code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); goto end; @@ -610,11 +610,11 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_ } if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); + snprintf(buf, tListLen(buf), "%s", strerror(terrno)); return buildSyntaxErrMsg(pMsgBuf, buf, NULL); } @@ -740,11 +740,11 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM /*if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {*/ if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { - if (errno == E2BIG) { + if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); + snprintf(buf, tListLen(buf), "%s", strerror(terrno)); return buildSyntaxErrMsg(pMsgBuf, buf, NULL); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 5c4ced7004..94bd53ea68 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -469,7 +469,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi val.type = TSDB_DATA_TYPE_NCHAR; if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, - strerror(errno)); + strerror(terrno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); taosMemoryFree(tmp); goto end; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 35e626872e..2de61ab669 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -198,7 +198,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { int32_t output = 0; if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, &output)) { - return errno; + return terrno; } varDataSetLen(pVal->datum.p, output); pVal->node.resType.bytes = output + VARSTR_HEADER_SIZE; @@ -486,7 +486,7 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { int32_t output = 0; if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, &output)) { - return errno; + return terrno; } varDataSetLen(pVal->datum.p, output); pVal->node.resType.bytes = output + VARSTR_HEADER_SIZE; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 1c99355e34..e2e0c04515 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -38,6 +38,8 @@ char *tstrdup(const char *str) { } #ifdef WINDOWS + +// No errors are expected to occur char *strsep(char **stringp, const char *delim) { char *s; const char *spanp; @@ -84,9 +86,57 @@ char *stpncpy(char *dest, const char *src, int n) { } #endif -int64_t taosStr2int64(const char *str) { - char *endptr = NULL; - return strtoll(str, &endptr, 10); +int32_t taosStr2int64(const char *str, int64_t *val) { + if (str == NULL || val == NULL) { + return TSDB_CODE_INVALID_PARA; + } + char *endptr = NULL; + int64_t ret = strtoll(str, &endptr, 10); + if (errno == ERANGE && (ret == LLONG_MAX || ret == LLONG_MIN)) { + return TAOS_SYSTEM_ERROR(errno); + } else { + *val = ret; + return 0; + } +} + +int32_t taosStr2int16(const char *str, int16_t *val) { + int64_t tmp = 0; + int32_t code = taosStr2int64(str, &tmp); + if (code) { + return code; + } else if (tmp > INT16_MAX || tmp < INT16_MIN) { + return TSDB_CODE_INVALID_PARA; + } else { + *val = (int16_t)tmp; + return 0; + } +} + +int32_t taosStr2int32(const char *str, int32_t *val) { + int64_t tmp = 0; + int32_t code = taosStr2int64(str, &tmp); + if (code) { + return code; + } else if (tmp > INT32_MAX || tmp < INT32_MIN) { + return TSDB_CODE_INVALID_PARA; + } else { + *val = (int32_t)tmp; + return 0; + } +} + +int32_t taosStr2int8(const char *str, int8_t *val) { + int64_t tmp = 0; + int32_t code = taosStr2int64(str, &tmp); + if (code) { + return code; + } else if (tmp > INT8_MAX || tmp < INT8_MIN) { + return TSDB_CODE_INVALID_PARA; + } else { + *val = (int8_t)tmp; + return 0; + } } int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { @@ -253,12 +303,12 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); - return -1; + terrno = TSDB_CODE_APP_ERROR; + return false; #else (void)memset(ucs4, 0, ucs4_max_len); int32_t idx = -1; - int32_t code = 0; iconv_t conv = taosAcquireConv(&idx, M2C); if ((iconv_t)-1 == conv || (iconv_t)0 == conv) { return false; @@ -267,9 +317,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 size_t ucs4_input_len = mbsLength; size_t outLeft = ucs4_max_len; if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) { - code = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseConv(idx, conv, M2C); - terrno = code; return false; } @@ -277,6 +326,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 if (len != NULL) { *len = (int32_t)(ucs4_max_len - outLeft); if (*len < 0) { + // can not happen + terrno = TSDB_CODE_APP_ERROR; return false; } } From d01670d49a8ec2fb4891d1ff1476ee3786df39b3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Sep 2024 14:06:25 +0800 Subject: [PATCH 37/61] fix(stream): fix the error in the checkpoint-trigger confirm condition. --- source/libs/stream/src/streamDispatch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index eb6c9c1a66..32fb67dd98 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1494,7 +1494,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i streamMutexLock(&pTask->lock); SStreamTaskState s = streamTaskGetStatus(pTask); triggerDispatchRsp = (s.state == TASK_STATUS__CK) && (pInfo->activeId == pMsgInfo->checkpointId) && - (pInfo->transId != pMsgInfo->transId); + (pInfo->transId == pMsgInfo->transId); tmpCheckpointId = pInfo->activeId; tmpTranId = pInfo->transId; pStatus = s.name; From a7269e4536caba4b31b0c7b2932ff5a925a85090 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 12 Sep 2024 14:17:14 +0800 Subject: [PATCH 38/61] fix: remove void function call --- include/util/tqueue.h | 2 +- source/client/src/clientMonitor.c | 2 +- source/libs/catalog/src/catalog.c | 22 +++++-- source/libs/catalog/src/ctgAsync.c | 12 +++- source/libs/catalog/src/ctgCache.c | 31 ++++++--- source/libs/catalog/src/ctgRemote.c | 58 +++++++++++++---- source/libs/catalog/src/ctgUtil.c | 27 ++++++-- source/libs/executor/src/dataDeleter.c | 4 +- source/libs/executor/src/dataDispatcher.c | 4 +- source/libs/executor/src/dataInserter.c | 10 ++- .../libs/executor/src/dynqueryctrloperator.c | 27 ++++++-- source/libs/executor/src/groupcacheoperator.c | 63 ++++++++++++++++--- source/libs/executor/src/mergejoin.c | 5 +- source/libs/executor/src/mergejoinoperator.c | 4 +- source/libs/qworker/src/qwUtil.c | 5 +- source/libs/qworker/src/qworker.c | 13 +++- source/libs/scheduler/inc/schInt.h | 4 +- source/libs/scheduler/src/schJob.c | 29 +++++++-- source/libs/scheduler/src/schTask.c | 12 +++- source/libs/scheduler/src/schUtil.c | 11 +++- source/libs/scheduler/src/scheduler.c | 6 +- source/util/src/thash.c | 4 +- source/util/src/tqueue.c | 6 +- source/util/src/tref.c | 2 +- source/util/src/tsimplehash.c | 2 +- source/util/src/ttimer.c | 3 + 26 files changed, 287 insertions(+), 81 deletions(-) diff --git a/include/util/tqueue.h b/include/util/tqueue.h index f7eaf794b0..5ae642b69f 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -79,7 +79,7 @@ void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); int32_t taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize, void **item); void taosFreeQitem(void *pItem); int32_t taosWriteQitem(STaosQueue *queue, void *pItem); -int32_t taosReadQitem(STaosQueue *queue, void **ppItem); +void taosReadQitem(STaosQueue *queue, void **ppItem); bool taosQueueEmpty(STaosQueue *queue); void taosUpdateItemSize(STaosQueue *queue, int32_t items); int32_t taosQueueItemSize(STaosQueue *queue); diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 9ed6512352..6904ea850c 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -730,7 +730,7 @@ static void* monitorThreadFunc(void* param) { } MonitorSlowLogData* slowLogData = NULL; - (void)taosReadQitem(monitorQueue, (void**)&slowLogData); + taosReadQitem(monitorQueue, (void**)&slowLogData); if (slowLogData != NULL) { if (slowLogData->type == SLOW_LOG_READ_BEGINNIG && quitCnt == 0) { if (slowLogData->pFile != NULL) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0b0cb6dc91..43c12f8164 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -706,7 +706,9 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); if (code) { qError("clear cache enqueue failed, error:%s", tstrerror(code)); - (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + if (taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer)) { + qError("reset catalog cache monitor timer error, timer stoppped"); + } } goto _return; @@ -714,7 +716,9 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { } qTrace("reset catalog timer"); - (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + if (taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer)) { + qError("reset catalog cache monitor timer error, timer stoppped"); + } _return: @@ -1517,10 +1521,16 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SC _return: if (pJob) { - (void)taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); + int32_t code2 = taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); + if (TSDB_CODE_SUCCESS) { + qError("release catalog job refId %" PRId64 "falied, error:%s", pJob->refId, tstrerror(code2)); + } if (code) { - (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + code2 = taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + if (TSDB_CODE_SUCCESS) { + qError("remove catalog job refId %" PRId64 "falied, error:%s", pJob->refId, tstrerror(code2)); + } } } @@ -1967,7 +1977,9 @@ void catalogDestroy(void) { } if (gCtgMgmt.cacheTimer) { - (void)taosTmrStop(gCtgMgmt.cacheTimer); + if (taosTmrStop(gCtgMgmt.cacheTimer)) { + qTrace("stop catalog cache timer may failed"); + } gCtgMgmt.cacheTimer = NULL; taosTmrCleanUp(gCtgMgmt.timer); gCtgMgmt.timer = NULL; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index e35aaeb0b1..9940474891 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1007,7 +1007,11 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(terrno); } - (void)taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); + void* p = taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); + if (NULL == p) { + ctgError("acquire job from ref failed, refId:%" PRId64 ", error: %s", pJob->refId, tstrerror(terrno)); + CTG_ERR_JRET(terrno); + } double el = (taosGetTimestampUs() - st) / 1000.0; qDebug("qid:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", @@ -1406,7 +1410,11 @@ int32_t ctgCallUserCb(void* param) { qDebug("qid:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); - (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + int64_t refId = pJob->refId; + int32_t code = taosRemoveRef(gCtgMgmt.jobPool, refId); + if (code) { + qError("qid:0x%" PRIx64 " remove ctg job %" PRId64 " from jobPool failed, error:%s", pJob->queryId, refId, tstrerror(code)); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 01201a2480..0e54ac77a2 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2915,21 +2915,27 @@ void ctgClearMetaCache(SCtgCacheOperation *operation) { if (CTG_CACHE_LOW(remainSize, cacheMaxSize)) { qDebug("catalog finish meta clear, remainSize:%" PRId64 ", cacheMaxSize:%dMB", remainSize, cacheMaxSize); - (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + if (taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer)) { + qError("reset catalog cache monitor timer error, timer stoppped"); + } return; } if (!roundDone) { qDebug("catalog all meta cleared, remainSize:%" PRId64 ", cacheMaxSize:%dMB, to clear handle", remainSize, cacheMaxSize); ctgClearFreeCache(operation); - (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + if (taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer)) { + qError("reset catalog cache monitor timer error, timer stoppped"); + } return; } int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); if (code) { qError("clear cache enqueue failed, error:%s", tstrerror(code)); - (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + if (taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer)) { + qError("reset catalog cache monitor timer error, timer stoppped"); + } } } @@ -2993,7 +2999,10 @@ int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) { pCtgCache->pTsmas = NULL; ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName); - (void)taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); + code = taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); + if (TSDB_CODE_SUCCESS != code) { + ctgError("remove table %s.%s from tsmaCache failed, error:%s", msg->dbFName, msg->tbName, tstrerror(code)); + } CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); } else { @@ -3191,6 +3200,7 @@ void ctgCleanupCacheQueue(void) { SCtgQNode *nodeNext = NULL; SCtgCacheOperation *op = NULL; bool stopQueue = false; + int32_t code = 0; while (true) { node = gCtgMgmt.queue.head->next; @@ -3209,7 +3219,10 @@ void ctgCleanupCacheQueue(void) { } if (op->syncOp) { - (void)tsem_post(&op->rspSem); + code = tsem_post(&op->rspSem); + if (code) { + qError("tsem_post failed when cleanup cache queue, error:%s", tstrerror(code)); + } } else { taosMemoryFree(op); } @@ -3234,12 +3247,13 @@ void ctgCleanupCacheQueue(void) { void *ctgUpdateThreadFunc(void *param) { setThreadName("catalog"); + int32_t code = 0; qInfo("catalog update thread started"); while (true) { if (tsem_wait(&gCtgMgmt.queue.reqSem)) { - qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + qError("ctg tsem_wait failed, error:%s", tstrerror(terrno)); } if (atomic_load_8((int8_t *)&gCtgMgmt.queue.stopQueue)) { @@ -3256,7 +3270,10 @@ void *ctgUpdateThreadFunc(void *param) { (void)(*gCtgCacheOperation[operation->opId].func)(operation); // ignore any error if (operation->syncOp) { - (void)tsem_post(&operation->rspSem); + code = tsem_post(&operation->rspSem); + if (code) { + ctgError("tsem_post failed for syncOp update, error:%s", tstrerror(code)); + } } else { taosMemoryFreeClear(operation); } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 00b1d7ad79..c9114ce90e 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -470,7 +470,10 @@ _return: taosMemoryFree(pMsg->pEpSet); if (pJob) { - (void)taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); + int32_t code2 = taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); + if (code2) { + qError("release ctg job refId:%" PRId64 " failed, error:%s", cbParam->refId, tstrerror(code2)); + } } CTG_API_LEAVE(code); @@ -1087,11 +1090,16 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n int32_t reqType = TDMT_MND_GET_TABLE_INDEX; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; - (void)tNameExtractFullName(name, tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); - int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp); + int32_t code = tNameExtractFullName(name, tbFName); + if (code) { + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + CTG_ERR_RET(code); + } + + code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get index msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); CTG_ERR_RET(code); @@ -1403,17 +1411,23 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_VND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - (void)tNameExtractFullName(pTableName, tbFName); + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; + int32_t code = tNameExtractFullName(pTableName, tbFName); + if (code) { + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + CTG_ERR_RET(code); + } + SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; ctgDebug("try to get table cfg from vnode, vgId:%d, ep num:%d, ep %s:%d, tbFName:%s", vgroupInfo->vgId, vgroupInfo->epSet.numOfEps, pEp->fqdn, pEp->port, tbFName); - int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen, mallocFp); + code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get tb cfg msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); CTG_ERR_RET(code); @@ -1471,15 +1485,20 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_MND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - (void)tNameExtractFullName(pTableName, tbFName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; + int32_t code = tNameExtractFullName(pTableName, tbFName); + if (code) { + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + CTG_ERR_RET(code); + } + ctgDebug("try to get table cfg from mnode, tbFName:%s", tbFName); - int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen, mallocFp); + code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get tb cfg msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); CTG_ERR_RET(code); @@ -1583,11 +1602,15 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char fullName[TSDB_TABLE_FNAME_LEN]; - (void)tNameExtractFullName(pName, fullName); + int32_t code = tNameExtractFullName(pName, fullName); + if (code) { + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pName->type, pName->dbname, pName->tname); + CTG_ERR_RET(code); + } ctgDebug("try to get view info from mnode, viewFName:%s", fullName); - int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](fullName, &msg, 0, &msgLen, mallocFp); + code = queryBuildMsg[TMSG_INDEX(reqType)](fullName, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build view-meta msg failed, code:%x, viewFName:%s", code, fullName); CTG_ERR_RET(code); @@ -1640,11 +1663,15 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; - (void)tNameExtractFullName(name, tbFName); + int32_t code = tNameExtractFullName(name, tbFName); + if (code) { + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + CTG_ERR_RET(code); + } ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); - int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp); + code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get index msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); CTG_ERR_RET(code); @@ -1697,7 +1724,12 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c int32_t msgLen = 0; int32_t reqType = TDMT_VND_GET_STREAM_PROGRESS; char tbFName[TSDB_TABLE_FNAME_LEN]; - (void)tNameExtractFullName(pTbName, tbFName); + int32_t code = tNameExtractFullName(pTbName, tbFName); + if (code) { + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTbName->type, pTbName->dbname, pTbName->tname); + CTG_ERR_RET(code); + } + SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; @@ -1705,7 +1737,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c ctgDebug("try to get stream progress from vnode, vgId:%d, ep num:%d, ep %s:%d, target:%s", vgroupInfo->vgId, vgroupInfo->epSet.numOfEps, pEp->fqdn, pEp->port, tbFName); - int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](bInput, &msg, 0, &msgLen, mallocFp); + code = queryBuildMsg[TMSG_INDEX(reqType)](bInput, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get stream progress failed, code:%s, tbFName:%s", tstrerror(code), tbFName); CTG_ERR_RET(code); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index daa2199421..545e3e1371 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -433,6 +433,7 @@ void ctgFreeHandle(SCatalog* pCtg) { void ctgClearHandleMeta(SCatalog* pCtg, int64_t* pClearedSize, int64_t* pCleardNum, bool* roundDone) { int64_t cacheSize = 0; + int32_t code = 0; void* pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache* dbCache = pIter; @@ -447,7 +448,10 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t* pClearedSize, int64_t* pCleardN continue; } - (void)taosHashRemove(dbCache->tbCache, key, len); + code = taosHashRemove(dbCache->tbCache, key, len); + if (code) { + qError("taosHashRemove table cache failed, key:%s, len:%d, error:%s", (char*)key, (int32_t)len, tstrerror(code)); + } cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); @@ -1201,8 +1205,12 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d SVgroupInfo* vgInfo = NULL; char tbFullName[TSDB_TABLE_FNAME_LEN]; - (void)tNameExtractFullName(pTableName, tbFullName); - + code = tNameExtractFullName(pTableName, tbFullName); + if (code) { + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + CTG_ERR_RET(code); + } + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1965,7 +1973,12 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char tbFName[TSDB_TABLE_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - (void)tNameExtractFullName(&req->pRawReq->tbName, tbFName); + code = tNameExtractFullName(&req->pRawReq->tbName, tbFName); + if (code) { + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + CTG_ERR_RET(code); + } + (void)tNameGetFullDbName(&req->pRawReq->tbName, dbFName); while (true) { @@ -2151,7 +2164,11 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) if (IS_SYS_DBNAME(req->pRawReq->tbName.dbname)) { (void)snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); } else { - (void)tNameExtractFullName(&req->pRawReq->tbName, viewFName); + code = tNameExtractFullName(&req->pRawReq->tbName, viewFName); + if (code) { + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + CTG_ERR_RET(code); + } } int32_t len = strlen(viewFName) + 1; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 60bfb58ef5..57f4289ebf 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -190,7 +190,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw } SDataDeleterBuf* pBuf = NULL; - (void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); + taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { TAOS_MEMCPY(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); taosFreeQitem(pBuf); @@ -248,7 +248,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { taosMemoryFree(pDeleter->pParam); while (!taosQueueEmpty(pDeleter->pDataBlocks)) { SDataDeleterBuf* pBuf = NULL; - (void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); + taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { taosMemoryFreeClear(pBuf->pData); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 9bbc5a94eb..8acd569358 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -233,7 +233,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow } SDataDispatchBuf* pBuf = NULL; - (void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); + taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { TAOS_MEMCPY(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); @@ -291,7 +291,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; - (void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); + taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { taosMemoryFreeClear(pBuf->pData); taosFreeQitem(pBuf); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 5ed97c8db6..b29bef1f1e 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -57,6 +57,7 @@ typedef struct SSubmitRspParam { int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { SSubmitRspParam* pParam = (SSubmitRspParam*)param; SDataInserterHandle* pInserter = pParam->pInserter; + int32_t code2 = 0; if (code) { pInserter->submitRes.code = code; @@ -106,7 +107,14 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { _return: - (void)tsem_post(&pInserter->ready); + code2 = tsem_post(&pInserter->ready); + if (code2 < 0) { + qError("tsem_post inserter ready failed, error:%s", tstrerror(code2)); + if (TSDB_CODE_SUCCESS == code) { + pInserter->submitRes.code = code2; + } + } + taosMemoryFree(pMsg->pData); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 44e8a3cb8a..3d23130e13 100644 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -561,7 +561,8 @@ static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPost static int32_t handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) { SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post; - + int32_t code = 0; + pPost->isStarted = false; if (pStbJoin->basic.batchFetch) { @@ -571,7 +572,11 @@ static int32_t handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDy if (pPost->leftNeedCache) { uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); if (num && --(*num) <= 0) { - (void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); + code = tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); + if (code) { + qError("tSimpleHashRemove leftCurrUid %" PRId64 " from leftCache failed, error:%s", pPost->leftCurrUid, tstrerror(code)); + QRY_ERR_RET(code); + } QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, true)); } } @@ -579,7 +584,11 @@ static int32_t handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDy if (!pPost->rightNeedCache) { void* v = tSimpleHashGet(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); if (NULL != v) { - (void)tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); + code = tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); + if (code) { + qError("tSimpleHashRemove rightCurrUid %" PRId64 " from rightCache failed, error:%s", pPost->rightCurrUid, tstrerror(code)); + QRY_ERR_RET(code); + } QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, false)); } } @@ -660,7 +669,11 @@ static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnc break; default: if (1 == (*pNum)) { - (void)tSimpleHashRemove(pOnceHash, pKey, keySize); + code = tSimpleHashRemove(pOnceHash, pKey, keySize); + if (code) { + qError("tSimpleHashRemove failed in addToJoinTableHash, error:%s", tstrerror(code)); + QRY_ERR_RET(code); + } } (*pNum)++; break; @@ -811,8 +824,12 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) { uint64_t* pUid = NULL; int32_t iter = 0; + int32_t code = 0; while (NULL != (pUid = tSimpleHashIterate(pStbJoin->ctx.prev.onceTable, pUid, &iter))) { - (void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid)); + code = tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid)); + if (code) { + qError("tSimpleHashRemove failed in postProcessStbJoinTableHash, error:%s", tstrerror(code)); + } } pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache); diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 1796aa7b64..2540579413 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -30,7 +30,9 @@ static void removeGroupCacheFile(SGroupCacheFileInfo* pFileInfo) { if (pFileInfo->fd.fd) { - (void)taosCloseFile(&pFileInfo->fd.fd); + if (taosCloseFile(&pFileInfo->fd.fd) < 0) { + qError("close group cache file failed, fd:%p, error:%s", pFileInfo->fd.fd, tstrerror(terrno)); + } pFileInfo->fd.fd = NULL; (void)taosThreadMutexDestroy(&pFileInfo->fd.mutex); } @@ -92,7 +94,9 @@ static void logGroupCacheExecInfo(SGroupCacheOperatorInfo* pGrpCacheOperator) { static void freeSGcSessionCtx(void* p) { SGcSessionCtx* pSession = p; if (pSession->semInit) { - (void)tsem_destroy(&pSession->waitSem); + if (tsem_destroy(&pSession->waitSem) < 0) { + qError("tsem_destroy session waitSem failed, error:%s", tstrerror(terrno)); + } } } @@ -262,6 +266,9 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S } static FORCE_INLINE void releaseFdToFileCtx(SGroupCacheFileFd* pFd) { + if (NULL == pFd) { + return; + } (void)taosThreadMutexUnlock(&pFd->mutex); } @@ -274,6 +281,8 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC SGroupCacheData* pGroup = NULL; while (NULL != pHead) { + pFd = NULL; + if (pGCache->batchFetch) { pFileCtx = &pHead->pCtx->fileCtx; } else { @@ -290,7 +299,11 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC int64_t blkId = pHead->basic.blkId; pHead = pHead->next; - (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + code = taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + if (code) { + qError("taosHashRemove blk %" PRId64 " from diryBlk failed, error:%s", blkId, tstrerror(code)); + goto _return; + } continue; } @@ -304,13 +317,19 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC } if (deleted) { + releaseFdToFileCtx(pFd); + qTrace("FileId:%d-%d-%d already be deleted, skip write", pCtx->id, pGroup ? pGroup->vgId : GROUP_CACHE_DEFAULT_VGID, pHead->basic.fileId); int64_t blkId = pHead->basic.blkId; pHead = pHead->next; - (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + code = taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + if (code) { + qError("taosHashRemove blk %" PRId64 " from diryBlk failed, error:%s", blkId, tstrerror(code)); + goto _return; + } continue; } @@ -336,7 +355,11 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC int64_t blkId = pHead->basic.blkId; pHead = pHead->next; - (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + code = taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + if (code) { + qError("taosHashRemove blk %" PRId64 " from diryBlk failed, error:%s", blkId, tstrerror(code)); + goto _return; + } } _return: @@ -1036,7 +1059,11 @@ static int32_t getCacheBlkFromDownstreamOperator(struct SOperatorInfo* pOperator } SGcSessionCtx* pWaitCtx = *ppWaitCtx; pWaitCtx->newFetch = true; - (void)taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId)); + code = taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId)); + if (code) { + qError("taosHashRemove session %" PRId64 " from waitSession failed, error: %s", pSessionId, tstrerror(code)); + return code; + } QRY_ERR_RET(tsem_post(&pWaitCtx->waitSem)); return code; @@ -1125,14 +1152,22 @@ static int32_t groupCacheSessionWait(struct SOperatorInfo* pOperator, SGcDownstr QRY_ERR_JRET(taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES)); - (void)tsem_wait(&pSession->waitSem); + code = tsem_wait(&pSession->waitSem); + if (code) { + qError("tsem_wait failed, error:%s", tstrerror(code)); + QRY_ERR_JRET(code); + } if (pSession->newFetch) { pSession->newFetch = false; return getCacheBlkFromDownstreamOperator(pOperator, pCtx, sessionId, pSession, ppRes); } - (void)taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId)); + code = taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId)); + if (code) { + qError("taosHashRemove session %" PRId64 " from waitSession failed, error: %s", sessionId, tstrerror(code)); + QRY_ERR_JRET(code); + } bool got = false; return getBlkFromSessionCacheImpl(pOperator, sessionId, pSession, ppRes, &got); @@ -1278,14 +1313,22 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); if (ppBlock) { QRY_ERR_RET(releaseBaseBlockToList(pCtx, *ppBlock)); - (void)taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); + code = taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); + if (code) { + qError("taosHashRemove session %" PRId64 " from pReadBlk failed, error: %s", pGcParam->sessionId, tstrerror(code)); + QRY_ERR_RET(code); + } } } code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes); if (NULL == *ppRes) { qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows); - (void)taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); + code = taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); + if (code) { + qError("taosHashRemove session %" PRId64 " from pSessions failed, error: %s", pGcParam->sessionId, tstrerror(code)); + QRY_ERR_RET(code); + } } else { pSession->resRows += (*ppRes)->info.rows; qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows); diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index 302dd31788..61809b99e0 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -2413,7 +2413,10 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) { pGrp->readIdx = 0; //pGrp->endIdx = pGrp->blk->info.rows - 1; } else { - (void)taosArrayPop(pCache->grps); + if (NULL == taosArrayPop(pCache->grps)) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + pGrp = taosArrayGet(pCache->grps, 0); if (NULL == pGrp) { MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 90c2248c12..d0354c12d5 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -1375,7 +1375,9 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol _return: if (pTable->noKeepEqGrpRows || !keepGrp || (!pTable->multiEqGrpRows && !restart)) { - (void)taosArrayPop(pTable->eqGrps); + if (NULL == taosArrayPop(pTable->eqGrps)) { + code = terrno; + } } else { pTable->grpTotalRows += pGrp->endIdx - pGrp->beginIdx + 1; } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 4b9067a191..aef348827a 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -581,7 +581,10 @@ void qwDestroyImpl(void *pMgmt) { int32_t schStatusCount = 0; qDebug("start to destroy qworker, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt); - (void)taosTmrStop(mgmt->hbTimer); //ignore error + if (taosTmrStop(mgmt->hbTimer)) { + qTrace("stop qworker hb timer may failed"); + } + mgmt->hbTimer = NULL; taosTmrCleanUp(mgmt->timer); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 1a9d3e7ba9..041347791e 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1187,7 +1187,9 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { int32_t schNum = taosHashGetSize(mgmt->schHash); if (schNum <= 0) { QW_UNLOCK(QW_READ, &mgmt->schLock); - (void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error + if (taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer)) { + qError("reset qworker hb timer error, timer stoppped"); + } (void)qwRelease(refId); // ignore error return; } @@ -1199,7 +1201,9 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { taosMemoryFree(rspList); taosArrayDestroy(pExpiredSch); QW_ELOG("calloc %d SQWHbInfo failed, code:%x", schNum, terrno); - (void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error + if (taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer)) { + qError("reset qworker hb timer error, timer stoppped"); + } (void)qwRelease(refId); // ignore error return; } @@ -1255,7 +1259,10 @@ _return: taosMemoryFreeClear(rspList); taosArrayDestroy(pExpiredSch); - (void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error + if (taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer)) { + qError("reset qworker hb timer error, timer stoppped"); + } + (void)qwRelease(refId); // ignore error } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 3a25f37895..a3be94d4c8 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -414,7 +414,9 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_START_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - (void)taosArrayPush((_task)->profile.execTime, &us); \ + if (NULL == taosArrayPush((_task)->profile.execTime, &us)) { \ + qError("taosArrayPush task execTime failed, error:%s", tstrerror(terrno)); \ + } \ if (0 == (_task)->execId) { \ (_task)->profile.startTs = us; \ } \ diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 8e2fbb878d..57a21643ef 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -512,6 +512,7 @@ int32_t schNotifyUserFetchRes(SSchJob *pJob) { } void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { + int32_t code = 0; SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); if (SCH_OP_NULL == pJob->opStatus.op) { @@ -526,7 +527,10 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { if (SCH_JOB_IN_SYNC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - (void)tsem_post(&pJob->rspSem); // ignore error + code = tsem_post(&pJob->rspSem); + if (code) { + ctgError("tsem_post failed for syncOp, error:%s", tstrerror(code)); + } } else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); (void)schNotifyUserExecRes(pJob); // ignore error @@ -771,7 +775,10 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); - (void)tsem_destroy(&pJob->rspSem); // ignore error + int32_t code = tsem_destroy(&pJob->rspSem); + if (code) { + qError("tsem_destroy failed, error:%s", tstrerror(code)); + } taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -790,7 +797,12 @@ int32_t schJobFetchRows(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - (void)tsem_wait(&pJob->rspSem); // ignore error + code = tsem_wait(&pJob->rspSem); + if (code) { + qError("tsem_wait for fetch rspSem failed, error:%s", tstrerror(code)); + SCH_RET(code); + } + SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } } else { @@ -895,7 +907,10 @@ _return: } else if (pJob->refId < 0) { schFreeJobImpl(pJob); } else { - (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error + code = taosRemoveRef(schMgmt.jobRef, pJob->refId); + if (code) { + SCH_JOB_DLOG("taosRemoveRef job refId:0x%" PRIx64 " from jobRef, error:%s", pJob->refId, tstrerror(code)); + } } SCH_RET(code); @@ -909,7 +924,11 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { if (pReq->syncReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - (void)tsem_wait(&pJob->rspSem); // ignore error + code = tsem_wait(&pJob->rspSem); + if (code) { + qError("qid:0x%" PRIx64 " tsem_wait sync rspSem failed, error:%s", pReq->pDag->queryId, tstrerror(code)); + SCH_ERR_RET(code); + } } SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 59b8954a48..8e9d46de31 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -422,7 +422,9 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { schDropTaskOnExecNode(pJob, pTask); if (pTask->delayTimer) { - (void)taosTmrStopA(&pTask->delayTimer); // ignore error + if (!taosTmrStopA(&pTask->delayTimer)) { + SCH_TASK_WLOG("stop task delayTimer failed, may stopped, status:%d", pTask->status); + } } taosHashClear(pTask->execNodes); (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error @@ -1319,7 +1321,9 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - (void)taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); + if (taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer)) { + SCH_TASK_ELOG("taosTmrReset delayExec timer failed, handle:%p", schMgmt.timer); + } return TSDB_CODE_SUCCESS; } @@ -1350,7 +1354,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { SCH_LOCK_TASK(pTask); if (pTask->delayTimer) { - (void)taosTmrStopA(&pTask->delayTimer); + if (!taosTmrStopA(&pTask->delayTimer)) { + SCH_TASK_WLOG("stop delayTimer failed, status:%d", pTask->delayTimer); + } } schDropTaskOnExecNode(pJob, pTask); SCH_UNLOCK_TASK(pTask); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 3f610ed387..612486c806 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -86,6 +86,7 @@ void schFreeHbTrans(SSchHbTrans *pTrans) { } void schCleanClusterHb(void *pTrans) { + int32_t code = 0; SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); SSchHbTrans *hb = taosHashIterate(schMgmt.hbConnections, NULL); @@ -93,7 +94,10 @@ void schCleanClusterHb(void *pTrans) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL); schFreeHbTrans(hb); - (void)taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + code = taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + if (code) { + qError("taosHashRemove hb connection failed, error:%s", tstrerror(code)); + } } hb = taosHashIterate(schMgmt.hbConnections, hb); @@ -116,7 +120,10 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { schFreeHbTrans(hb); - (void)taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + code = taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + if (code) { + SCH_TASK_WLOG("taosHashRemove hb connection failed, error:%s", tstrerror(code)); + } } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3ce5cd5714..a62c59e547 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -184,6 +184,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { } void schedulerDestroy(void) { + int32_t code = 0; atomic_store_8((int8_t *)&schMgmt.exit, 1); if (schMgmt.jobRef >= 0) { @@ -195,7 +196,10 @@ void schedulerDestroy(void) { if (refId == 0) { break; } - (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error + code = taosRemoveRef(schMgmt.jobRef, pJob->refId); + if (code) { + qWarn("taosRemoveRef job refId:%" PRId64 " failed, error:%s", pJob->refId, tstrerror(code)); + } pJob = taosIterateRef(schMgmt.jobRef, refId); } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 7780be3fb7..758e283bc3 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -488,10 +488,10 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { if (pe->num == 0) { taosHashEntryWUnlock(pHashObj, pe); taosHashRUnlock(pHashObj); - return -1; + return TSDB_CODE_NOT_FOUND; } - int code = -1; + int code = TSDB_CODE_NOT_FOUND; SHashNode *pNode = pe->next; SHashNode *prevNode = NULL; diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 780a6c94f1..a802737461 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -229,9 +229,8 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { return code; } -int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { +void taosReadQitem(STaosQueue *queue, void **ppItem) { STaosQnode *pNode = NULL; - int32_t code = 0; (void)taosThreadMutexLock(&queue->mutex); @@ -247,14 +246,11 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { if (queue->qset) { (void)atomic_sub_fetch_32(&queue->qset->numOfItems, 1); } - code = 1; uTrace("item:%p is read out from queue:%p, items:%d mem:%" PRId64, *ppItem, queue, queue->numOfItems, queue->memOfItems); } (void)taosThreadMutexUnlock(&queue->mutex); - - return code; } int32_t taosAllocateQall(STaosQall **qall) { diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 0eac7b4427..a1c161a42c 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -426,7 +426,7 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { } else { uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; - code = -1; + code = terrno; } taosUnlockList(pSet->lockedBy + hash); diff --git a/source/util/src/tsimplehash.c b/source/util/src/tsimplehash.c index d14e72822f..b16fde86cf 100644 --- a/source/util/src/tsimplehash.c +++ b/source/util/src/tsimplehash.c @@ -297,7 +297,7 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key, size_t keyLen) { } int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) { - int32_t code = TSDB_CODE_FAILED; + int32_t code = TSDB_CODE_INVALID_PARA; if (!pHashObj || !key) { return code; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 60edb2f045..d5310b2e7e 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -492,6 +492,9 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han if (timer == NULL) { *pTmrId = taosTmrStart(fp, mseconds, param, handle); + if (NULL == *pTmrId) { + stopped = true; + } return stopped; } From fba5560f1bfb29e33bbdf43da8b8d22072eacf68 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 12 Sep 2024 14:24:11 +0800 Subject: [PATCH 39/61] fix(stmt2/close): wait asnyc cb completed to free itself --- source/client/src/clientStmt2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index e7e08c0982..e716e06005 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1667,7 +1667,6 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { STMT_ERR_RET(stmtCleanExecInfo(pStmt, (code ? false : true), false)); ++pStmt->sql.runTimes; - } else { SSqlCallbackWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); if (pWrapper == NULL) { @@ -1718,6 +1717,7 @@ int stmtClose2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); if (pStmt->options.asyncExecFn) { + (void)tsem_wait(&pStmt->asyncQuerySem); (void)tsem_destroy(&pStmt->asyncQuerySem); } taosMemoryFree(stmt); From e0a275f1d41ee12853b502b3375f671c1b791d47 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Sep 2024 14:49:35 +0800 Subject: [PATCH 40/61] fix(stream): do some internal refactor and rewrite the error code for encode. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 13 ++++++++----- source/libs/stream/src/streamCheckStatus.c | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c00d9a93bb..c05e9f7b60 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -586,6 +586,10 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve bool isLeader, bool restored) { int32_t code = 0; int32_t vgId = pMeta->vgId; + int32_t numOfTasks = 0; + int32_t taskId = -1; + int64_t streamId = -1; + bool added = false; if (tsDisableStream) { tqInfo("vgId:%d stream disabled, not deploy stream tasks", vgId); @@ -613,13 +617,12 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve } // 2.save task, use the latest commit version as the initial start version of stream task. - int32_t taskId = pTask->id.taskId; - int64_t streamId = pTask->id.streamId; - bool added = false; + taskId = pTask->id.taskId; + streamId = pTask->id.streamId; streamMetaWLock(pMeta); code = streamMetaRegisterTask(pMeta, sversion, pTask, &added); - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + numOfTasks = streamMetaGetNumOfTasks(pMeta); streamMetaWUnLock(pMeta); if (code < 0) { @@ -654,7 +657,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve tqDebug("vgId:%d not leader, not launch stream task s-task:0x%x", vgId, taskId); } } else { - tqWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store", vgId, taskId); + tqWarn("vgId:%d failed to add s-task:0x%x, since already exists in meta store, total:%d", vgId, taskId, numOfTasks); tFreeStreamTask(pTask); } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index d0604e6d21..31d1bddd01 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -268,6 +268,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; tmsgSendRsp(&rspMsg); + code = (code >= 0)? 0:code; return code; } From 9ad2b9d96a21ba3c98da904e0c13c41b290b9fc9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 12 Sep 2024 14:58:05 +0800 Subject: [PATCH 41/61] move sem to client main to wait for set tbname etc. --- source/client/src/clientMain.c | 13 +++++++++---- source/client/src/clientStmt2.c | 9 ++++----- tests/script/api/stmt2-nohole.c | 12 ++++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1c1fff9b7b..a28c17cb25 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1317,8 +1317,8 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); if (TSDB_CODE_SUCCESS != refreshMeta(pRequest->pTscObj, pRequest)) { - tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, - tstrerror(code), pRequest->requestId); + tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), + pRequest->requestId); } pRequest->prevCode = code; doAsyncQuery(pRequest, true); @@ -1369,7 +1369,7 @@ typedef struct SAsyncFetchParam { void *param; } SAsyncFetchParam; -static int32_t doAsyncFetch(void* pParam) { +static int32_t doAsyncFetch(void *pParam) { SAsyncFetchParam *param = pParam; taosAsyncFetchImpl(param->pReq, param->fp, param->param); taosMemoryFree(param); @@ -1393,7 +1393,7 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { return; } - SAsyncFetchParam* pParam = taosMemoryCalloc(1, sizeof(SAsyncFetchParam)); + SAsyncFetchParam *pParam = taosMemoryCalloc(1, sizeof(SAsyncFetchParam)); if (!pParam) { fp(param, res, terrno); return; @@ -1983,6 +1983,11 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col return terrno; } + STscStmt2 *pStmt = (STscStmt2 *)stmt; + if (pStmt->options.asyncExecFn) { + (void)tsem_wait(&pStmt->asyncQuerySem); + } + int32_t code = 0; for (int i = 0; i < bindv->count; ++i) { if (bindv->tbnames && bindv->tbnames[i]) { diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index e716e06005..51533fc56c 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1263,10 +1263,6 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); - if (pStmt->options.asyncExecFn) { - (void)tsem_wait(&pStmt->asyncQuerySem); - } - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; @@ -1703,6 +1699,10 @@ int stmtClose2(TAOS_STMT2* stmt) { pStmt->bindThreadInUse = false; } + if (pStmt->options.asyncExecFn) { + (void)tsem_wait(&pStmt->asyncQuerySem); + } + STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64 ", parseSqlNum=>%" PRId64 ", pStmt->stat.bindDataNum=>%" PRId64 ", settbnameAPI:%u, bindAPI:%u, addbatchAPI:%u, execAPI:%u" @@ -1717,7 +1717,6 @@ int stmtClose2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); if (pStmt->options.asyncExecFn) { - (void)tsem_wait(&pStmt->asyncQuerySem); (void)tsem_destroy(&pStmt->asyncQuerySem); } taosMemoryFree(stmt); diff --git a/tests/script/api/stmt2-nohole.c b/tests/script/api/stmt2-nohole.c index b29dd3e826..5954f3660b 100644 --- a/tests/script/api/stmt2-nohole.c +++ b/tests/script/api/stmt2-nohole.c @@ -14,12 +14,12 @@ int64_t genReqid() { return count; } -sem_t sem; +// sem_t sem; void stmtAsyncQueryCb(void* param, TAOS_RES* pRes, int code) { int affected_rows = taos_affected_rows(pRes); printf("\033[31maffected rows:%d\033[0m\n", affected_rows); - (void)sem_post(&sem); + //(void)sem_post(&sem); return; /* SSP_CB_PARAM* qParam = (SSP_CB_PARAM*)param; @@ -319,7 +319,7 @@ _bind_again: taos_stmt2_free_fields(stmt, fields); */ // if (taos_stmt_execute(stmt) != 0) { - (void)sem_init(&sem, 0, 0); + //(void)sem_init(&sem, 0, 0); start = clock(); // if (taos_stmt2_exec(stmt, NULL, stmtAsyncQueryCb, NULL) != 0) { if (taos_stmt2_exec(stmt, NULL) != 0) { @@ -330,9 +330,9 @@ _bind_again: end = clock(); printf("exec time:%f\n", (double)(end - start) / CLOCKS_PER_SEC); - sem_wait(&sem); - (void)sem_destroy(&sem); - if (++run_time < 2) { + // sem_wait(&sem); + //(void)sem_destroy(&sem); + if (++run_time < 20) { goto _bind_again; } taos_stmt2_close(stmt); From e7cfe21e7213e0e10213b297ecbfd9b3b86e3918 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 12 Sep 2024 15:17:09 +0800 Subject: [PATCH 42/61] stmt2/async: new flag semWaited for the current batch --- source/client/inc/clientStmt2.h | 8 ++++---- source/client/src/clientMain.c | 3 ++- source/client/src/clientStmt2.c | 2 ++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 74eb198930..4e9a09c082 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -150,10 +150,10 @@ typedef struct { SStmtExecInfo exec; SStmtBindInfo bInfo; - int64_t reqid; - int32_t errCode; - tsem_t asyncQuerySem; - + int64_t reqid; + int32_t errCode; + tsem_t asyncQuerySem; + bool semWaited; SStmtStatInfo stat; } STscStmt2; /* diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a28c17cb25..7cfbeb1372 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1984,8 +1984,9 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col } STscStmt2 *pStmt = (STscStmt2 *)stmt; - if (pStmt->options.asyncExecFn) { + if (pStmt->options.asyncExecFn && !pStmt->semWaited) { (void)tsem_wait(&pStmt->asyncQuerySem); + pStmt->semWaited = true; } int32_t code = 0; diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 51533fc56c..1739dced8f 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -819,6 +819,7 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { if (pStmt->options.asyncExecFn) { (void)tsem_init(&pStmt->asyncQuerySem, 0, 1); } + pStmt->semWaited = false; STMT_LOG_SEQ(STMT_INIT); @@ -1678,6 +1679,7 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { pRequest->body.queryFp = asyncQueryCb; ((SSyncQueryParam*)(pRequest)->body.interParam)->userParam = pStmt; + pStmt->semWaited = false; launchAsyncQuery(pRequest, pStmt->sql.pQuery, NULL, pWrapper); } From aecab6e2caf2b4d71567bfaa66e6457918db5861 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 12 Sep 2024 15:27:11 +0800 Subject: [PATCH 43/61] stmt2/close: wait only when !semWaited (i.e. execed current batch) --- source/client/src/clientStmt2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 1739dced8f..ae59319cec 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1701,7 +1701,7 @@ int stmtClose2(TAOS_STMT2* stmt) { pStmt->bindThreadInUse = false; } - if (pStmt->options.asyncExecFn) { + if (pStmt->options.asyncExecFn && !pStmt->semWaited) { (void)tsem_wait(&pStmt->asyncQuerySem); } From aa80aa39bcfa818cd70302e5716c7ad2bc053638 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 12 Sep 2024 15:34:57 +0800 Subject: [PATCH 44/61] fix(query):use window start key when ts column is null --- include/libs/function/functionMgt.h | 1 + source/libs/executor/src/timewindowoperator.c | 10 ++++++++-- source/libs/function/src/functionMgt.c | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 233e201ac3..519207377b 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -280,6 +280,7 @@ bool fmIsSkipScanCheckFunc(int32_t funcId); bool fmIsPrimaryKeyFunc(int32_t funcId); bool fmIsProcessByRowFunc(int32_t funcId); bool fmisSelectGroupConstValueFunc(int32_t funcId); +bool fmIsElapsedFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType, int32_t pkBytes); int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 640ce5c98b..0cd506d15a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -293,7 +293,9 @@ void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY pr SPoint point2 = (SPoint){.key = curTs, .val = &v2}; SPoint point = (SPoint){.key = windowKey, .val = &v}; - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + if (!fmIsElapsedFunc(pCtx[k].functionId)) { + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + } if (type == RESULT_ROW_START_INTERP) { pCtx[k].start.key = point.key; @@ -639,7 +641,11 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num int64_t prevTs = *(int64_t*)pTsKey->pData; if (groupId == pBlock->info.id.groupId) { - doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey, + TSKEY curTs = pBlock->info.window.skey; + if (tsCols != NULL) { + curTs = tsCols[startPos]; + } + doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, curTs, startPos, w.ekey, RESULT_ROW_END_INTERP, pSup); } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 1a927a1576..886772b36c 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -276,6 +276,13 @@ bool fmisSelectGroupConstValueFunc(int32_t funcId) { return FUNCTION_TYPE_GROUP_CONST_VALUE == funcMgtBuiltins[funcId].type; } +bool fmIsElapsedFunc(int32_t funcId) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return false; + } + return FUNCTION_TYPE_ELAPSED == funcMgtBuiltins[funcId].type; +} + bool fmIsBlockDistFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; From 726b13372192d298a61791dadbaf9d1e2877919d Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Thu, 12 Sep 2024 07:46:30 +0000 Subject: [PATCH 45/61] modify error code passing --- source/dnode/mnode/impl/src/mndArbGroup.c | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 12 +-- source/dnode/mnode/impl/src/mndDb.c | 4 +- source/dnode/mnode/impl/src/mndDef.c | 18 ++--- source/dnode/mnode/impl/src/mndDnode.c | 30 +++---- source/dnode/mnode/impl/src/mndFunc.c | 16 ++-- source/dnode/mnode/impl/src/mndInfoSchema.c | 2 +- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/mnode/impl/src/mndPerfSchema.c | 4 +- source/dnode/mnode/impl/src/mndPrivilege.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 10 +-- source/dnode/mnode/impl/src/mndQnode.c | 8 +- source/dnode/mnode/impl/src/mndQuery.c | 4 +- source/dnode/mnode/impl/src/mndScheduler.c | 4 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 32 ++++---- source/dnode/mnode/impl/src/mndSnode.c | 6 +- source/dnode/mnode/impl/src/mndStb.c | 32 ++++---- source/dnode/mnode/impl/src/mndStream.c | 6 +- source/dnode/mnode/impl/src/mndStreamHb.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 4 +- source/dnode/mnode/impl/src/mndSync.c | 4 +- source/dnode/mnode/impl/src/mndTrans.c | 2 +- source/dnode/mnode/impl/src/mndUser.c | 88 ++++++++++----------- source/dnode/mnode/impl/src/mndVgroup.c | 6 +- source/dnode/mnode/sdb/src/sdb.c | 2 +- source/dnode/mnode/sdb/src/sdbFile.c | 8 +- source/dnode/mnode/sdb/src/sdbHash.c | 1 - 28 files changed, 156 insertions(+), 157 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 3860f10f8f..1c9dc4c77c 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -678,7 +678,7 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { mndArbGroupSetAssignedLeader(&newGroup, candidateIndex); if (taosArrayPush(pUpdateArray, &newGroup) == NULL) { taosArrayDestroy(pUpdateArray); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } sdbRelease(pSdb, pArbGroup); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 606b93035f..f71ab95d03 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -216,7 +216,7 @@ static int32_t buildMqHbRsp(SRpcMsg *pMsg, SMqHbRsp *rsp){ } void *buf = rpcMallocCont(tlen); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if(tSerializeSMqHbRsp(buf, tlen, rsp) <= 0){ @@ -261,7 +261,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e rsp->topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp)); if (rsp->topics == NULL) { taosRUnLockLatch(&pConsumer->lock); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } // handle all topics subscribed by this consumer @@ -318,7 +318,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t j = 0; j < vgNum; j++) { @@ -339,7 +339,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } if (taosArrayPush(rsp->topics, &topicEp) == NULL) { @@ -347,7 +347,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } taosRUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); @@ -362,7 +362,7 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, rsp); void *buf = rpcMallocCont(tlen); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SMqRspHead *pHead = buf; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6467dfcf11..efa98e3b40 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1558,7 +1558,7 @@ static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bo } if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } @@ -1729,7 +1729,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs int32_t code = 0; pRsp->pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); if (pRsp->pVgroupInfos == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index d93b275e48..61606b0364 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -137,7 +137,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { if (sz != 0) { pObj->tasks = taosArrayInit(sz, sizeof(void *)); if (pObj->tasks == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } @@ -161,14 +161,14 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { if (taosArrayPush(pArray, &pTask) == NULL) { taosMemoryFree(pTask); taosArrayDestroy(pArray); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } } } if (taosArrayPush(pObj->tasks, &pArray) == NULL) { taosArrayDestroy(pArray); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } } @@ -297,24 +297,24 @@ int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, if (updateType == CONSUMER_ADD_REB){ pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); if(pConsumer->rebNewTopics == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } char* topicTmp = taosStrdup(topic); if (taosArrayPush(pConsumer->rebNewTopics, &topicTmp) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } }else if (updateType == CONSUMER_REMOVE_REB) { pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); if(pConsumer->rebRemovedTopics == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } char* topicTmp = taosStrdup(topic); if (taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } }else if (updateType == CONSUMER_INSERT_SUB){ @@ -330,7 +330,7 @@ int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup); if (pConsumer->rebNewTopics == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } pConsumer->assignedTopics = subscribe->topicNames; @@ -612,7 +612,7 @@ int32_t tCloneSubscribeObj(const SMqSubscribeObj *pSub, SMqSubscribeObj **ppSub) int32_t code = 0; SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); if (pSubNew == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } (void)memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 71952b3bb8..a432df60a5 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -461,7 +461,7 @@ int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { } if(taosArrayPush(pDnodeInfo, &dInfo) == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; sdbCancelFetch(pSdb, pIter); break; } @@ -618,7 +618,7 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { } pReq = rpcMallocCont(contLen); if (pReq == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } (void)tSerializeSDnodeInfoReq(pReq, contLen, &infoReq); @@ -980,7 +980,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); if (NULL == rsp.dnodeList) { mError("failed to alloc epSet while process dnode list req"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1006,7 +1006,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1064,7 +1064,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo)); if (NULL == rsp.variables) { mError("failed to alloc SVariablesInfo array while process show variables req"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1074,7 +1074,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); (void)strcpy(info.scope, "server"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1082,7 +1082,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsTimezoneStr); (void)strcpy(info.scope, "both"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1090,7 +1090,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsLocale); (void)strcpy(info.scope, "both"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1098,7 +1098,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsCharset); (void)strcpy(info.scope, "both"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1106,7 +1106,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsEnableMonitor); (void)strcpy(info.scope, "server"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1114,7 +1114,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsMonitorInterval); (void)strcpy(info.scope, "server"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1122,7 +1122,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogThreshold); (void)strcpy(info.scope, "server"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1130,7 +1130,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogMaxLen); (void)strcpy(info.scope, "server"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1140,14 +1140,14 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", scopeStr); (void)strcpy(info.scope, "server"); if (taosArrayPush(rsp.variables, &info) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 326f2ffa95..9a617fb894 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -265,7 +265,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre func.codeSize = pCreate->codeLen; func.pCode = taosMemoryMalloc(func.codeSize); if (func.pCode == NULL || func.pCode == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -528,13 +528,13 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs; retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo)); if (retrieveRsp.pFuncInfos == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto RETRIEVE_FUNC_OVER; } retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo)); if (retrieveRsp.pFuncExtraInfos == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto RETRIEVE_FUNC_OVER; } @@ -563,28 +563,28 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { funcInfo.codeSize = pFunc->codeSize; funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize); if (funcInfo.pCode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; goto RETRIEVE_FUNC_OVER; } (void)memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); if (funcInfo.commentSize > 0) { funcInfo.pComment = taosMemoryCalloc(1, funcInfo.commentSize); if (funcInfo.pComment == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; goto RETRIEVE_FUNC_OVER; } (void)memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); } } if (taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; goto RETRIEVE_FUNC_OVER; } SFuncExtraInfo extraInfo = {0}; extraInfo.funcVersion = pFunc->funcVersion; extraInfo.funcCreatedTime = pFunc->createdTime; if (taosArrayPush(retrieveRsp.pFuncExtraInfos, &extraInfo) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; goto RETRIEVE_FUNC_OVER; } @@ -594,7 +594,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto RETRIEVE_FUNC_OVER; } diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index eb7cb7d505..f72f17684e 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -60,7 +60,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) { TAOS_CHECK_RETURN(mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas)); if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index aa03fbf2d9..a0dd30f11a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -559,7 +559,7 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle step.initFp = initFp; step.cleanupFp = cleanupFp; if (taosArrayPush(pMnode->pSteps, &step) == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } TAOS_RETURN(0); diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index da5e201c05..f36ddf7493 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -58,7 +58,7 @@ int32_t mndPerfsInitMeta(SHashObj *hash) { TAOS_CHECK_RETURN(mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)); if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } } @@ -129,7 +129,7 @@ int32_t mndInitPerfs(SMnode *pMnode) { int32_t code = 0; pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (pMnode->perfsMeta == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index ce082ad45d..a13fe5d69f 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -36,7 +36,7 @@ int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteLis pWhiteListRsp->numWhiteLists = 1; pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); if (pWhiteListRsp->pWhiteLists == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memset(pWhiteListRsp->pWhiteLists, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 3bdfa236d1..04c474dd56 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -318,7 +318,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + TAOS_CHECK_GOTO(terrno, NULL, _OVER); } contLen = tSerializeSConnectRsp(pRsp, contLen, &connectRsp); @@ -555,7 +555,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb hbRsp.info = taosArrayInit(kvNum, sizeof(SKv)); if (NULL == hbRsp.info) { mError("taosArrayInit %d rsp kv failed", kvNum); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; tFreeClientHbRsp(&hbRsp); TAOS_RETURN(code); } @@ -699,7 +699,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { batchRsp.svrTimestamp = taosGetTimestampSec(); batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); if (batchRsp.rsps == NULL) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } batchRsp.monitorParas.tsEnableMonitor = tsEnableMonitor; batchRsp.monitorParas.tsMonitorInterval = tsMonitorInterval; @@ -732,7 +732,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { } void *buf = rpcMallocCont(tlen); if (!buf) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } tlen = tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); if (tlen < 0) { @@ -820,7 +820,7 @@ static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { } void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } contLen = tSerializeSServerVerRsp(pRsp, contLen, &rsp); if (contLen < 0) { diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 54052590a1..41d292a62f 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -249,7 +249,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } (void)tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); @@ -380,7 +380,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } (void)tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); @@ -483,7 +483,7 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) { SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); if (NULL == qnodeList) { mError("failed to alloc epSet while process qnode list req"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } @@ -532,7 +532,7 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) { int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index c743aafd13..4816df6323 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -103,7 +103,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { batchRsp.pRsps = taosArrayInit(msgNum, sizeof(SBatchRspMsg)); if (NULL == batchRsp.pRsps) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -147,7 +147,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } pRsp = rpcMallocCont(rspSize); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } if (tSerializeSBatchRsp(pRsp, rspSize, &batchRsp) < 0) { diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 93cd351543..4f72b26a5e 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -846,13 +846,13 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); if (pVgEp == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->vgId = pVgroup->vgId; if (taosArrayPush(pSub->unassignedVgs, &pVgEp) == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFree(pVgEp); goto END; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index b8d44da8c4..55687c00ba 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -243,7 +243,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pShow = mndCreateShowObj(pMnode, &retrieveReq); if (pShow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; mError("failed to process show-meta req since %s", tstrerror(code)); TAOS_RETURN(code); } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 284d65cd9c..f7948764dd 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1279,7 +1279,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool info.expr = taosMemoryMalloc(pSma->exprLen + 1); if (info.expr == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pIter); return code; @@ -1289,7 +1289,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool info.expr[pSma->exprLen] = 0; if (NULL == taosArrayPush(rsp->pIndex, &info)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFree(info.expr); sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pIter); @@ -1326,7 +1326,7 @@ static int32_t mndProcessGetSmaReq(SRpcMsg *pReq) { int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1361,7 +1361,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { rsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); if (NULL == rsp.pIndex) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1376,7 +1376,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -1699,7 +1699,7 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "create-tsma"); if (!pTrans) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } mndTransSetDbName(pTrans, pCxt->pDb->name, NULL); @@ -1797,14 +1797,14 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { if (pCxt->pCreateSmaReq->pVgroupVerList) { pCxt->pCreateStreamReq->pVgroupVerList = taosArrayDup(pCxt->pCreateSmaReq->pVgroupVerList, NULL); if (!pCxt->pCreateStreamReq->pVgroupVerList) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } } if (LIST_LENGTH(pProjects) > 0) { createStreamReq.pCols = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SField)); if (!createStreamReq.pCols) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } } @@ -1978,7 +1978,7 @@ static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { STransAction dropStreamRedoAction = {0}; STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "drop-tsma"); if (!pTrans) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } SMDropStreamReq dropStreamReq = {0}; @@ -2337,7 +2337,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa funcInfo.funcId = pFuncNode->funcId; funcInfo.colId = ((SColumnNode *)pFuncNode->pParameterList->pHead->pNode)->colId; if (!taosArrayPush(pInfo->pFuncs, &funcInfo)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosArrayDestroy(pInfo->pFuncs); nodesDestroyNode(pNode); return code; @@ -2351,7 +2351,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa if (code == TSDB_CODE_SUCCESS && pDestStb->numOfTags > 0) { pInfo->pTags = taosArrayInit(pDestStb->numOfTags, sizeof(SSchema)); if (!pInfo->pTags) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } else { for (int32_t i = 0; i < pDestStb->numOfTags; ++i) { if (NULL == taosArrayPush(pInfo->pTags, &pDestStb->pTags[i])) { @@ -2364,7 +2364,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa if (code == TSDB_CODE_SUCCESS) { pInfo->pUsedCols = taosArrayInit(pDestStb->numOfColumns - 3, sizeof(SSchema)); if (!pInfo->pUsedCols) - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; else { // skip _wstart, _wend, _duration for (int32_t i = 1; i < pDestStb->numOfColumns - 2; ++i) { @@ -2438,7 +2438,7 @@ static int32_t mndGetTSMA(SMnode *pMnode, char *tsmaFName, STableTSMAInfoRsp *rs TAOS_RETURN(code); } if (NULL == taosArrayPush(rsp->pTsmas, &pTsma)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; tFreeAndClearTableTSMAInfo(pTsma); } *exist = true; @@ -2524,7 +2524,7 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt TAOS_RETURN(code); } if (NULL == taosArrayPush(pRsp->pTsmas, &pTsma)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; tFreeAndClearTableTSMAInfo(pTsma); sdbCancelFetch(pSdb, pIter); TAOS_RETURN(code); @@ -2566,7 +2566,7 @@ static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq) { rsp.pTsmas = taosArrayInit(4, POINTER_BYTES); if (NULL == rsp.pTsmas) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -2588,7 +2588,7 @@ static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq) { int32_t contLen = tSerializeTableTSMAInfoRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index c5e38ed048..4616f50a79 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -220,7 +220,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } (void)tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); @@ -248,7 +248,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } (void)tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); @@ -380,7 +380,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } (void)tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 03ff6d425f..d46726fd31 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2964,7 +2964,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -3015,7 +3015,7 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -3041,14 +3041,14 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t SSTbHbRsp hbRsp = {0}; hbRsp.pMetaRsp = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); if (hbRsp.pMetaRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } hbRsp.pIndexRsp = taosArrayInit(numOfStbs, sizeof(STableIndexRsp)); if (NULL == hbRsp.pIndexRsp) { taosArrayDestroy(hbRsp.pMetaRsp); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } @@ -3104,7 +3104,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t STableIndexRsp indexRsp = {0}; indexRsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); if (NULL == indexRsp.pIndex) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } @@ -3136,7 +3136,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { tFreeSSTbHbRsp(&hbRsp); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; TAOS_RETURN(code); } @@ -4066,20 +4066,20 @@ static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx **ppCtx) { if (!pCtx) return terrno; pCtx->pTsmaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (!pCtx->pTsmaMap) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _end; } pCtx->pDbMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (!pCtx->pDbMap) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _end; } pCtx->pResTbNames = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); pCtx->pVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (!pCtx->pVgMap) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _end; } *ppCtx = pCtx; @@ -4203,17 +4203,17 @@ static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupI reqs.info = *pVgInfo; reqs.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); if (reqs.req.pArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(reqs.req.pArray, &req) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &reqs, sizeof(reqs)) != 0) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else { if (taosArrayPush(pReqs->req.pArray, &req) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } return 0; @@ -4229,7 +4229,7 @@ static int32_t mndGetDbVgInfoForTsma(SMnode *pMnode, const char *dbname, SMDropT pInfo->dbInfo.dbVgInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); if (!pInfo->dbInfo.dbVgInfos) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _end; } mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos); @@ -4284,11 +4284,11 @@ static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWith SMDropTbTsmaInfos infos = {0}; infos.pTsmaInfos = taosArrayInit(2, sizeof(SMDropTbTsmaInfo)); if (!infos.pTsmaInfos) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _end; } if (taosHashPut(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid), &infos, sizeof(infos)) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _end; } } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0c57f8466d..cbe631912c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1993,7 +1993,7 @@ static int32_t mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeLis if (pInfo->pUpdateNodeList == NULL || pInfo->pDBMap == NULL) { mndDestroyVgroupChangeInfo(pInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t numOfNodes = taosArrayGetSize(pPrevNodeList); @@ -2164,7 +2164,7 @@ static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeLi SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (pHash == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } while (1) { @@ -2215,7 +2215,7 @@ static int32_t refreshNodeListFromExistedStreams(SMnode *pMnode, SArray *pNodeLi if (p == NULL) { mError("failed to put entry into node list, nodeId:%d, code: out of memory", pEntry->nodeId); if (code == 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } continue; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index d31d0dad65..43f9d8d055 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -349,7 +349,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pFailedChkpt == NULL || pOrphanTasks == NULL) { taosArrayDestroy(pFailedChkpt); taosArrayDestroy(pOrphanTasks); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } streamMutexLock(&execInfo.lock); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e3bef61bc0..1c6474d787 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -201,7 +201,7 @@ static int32_t mndGetOrCreateRebSub(SHashObj *pHash, const char *key, SMqRebInfo if (pRebInfo == NULL) { pRebInfo = tNewSMqRebSubscribe(key); if (pRebInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } code = taosHashPut(pHash, key, strlen(key) + 1, pRebInfo, sizeof(SMqRebInfo)); @@ -412,7 +412,7 @@ static int32_t processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows)); if(pOutput->pSub->offsetRows == NULL) { taosRUnLockLatch(&pSub->lock); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index d669994128..66c0fec273 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -546,10 +546,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; - if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY; + if (req.contLen <= 0) return terrno; req.pCont = rpcMallocCont(req.contLen); - if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (req.pCont == NULL) return terrno; memcpy(req.pCont, pRaw, req.contLen); (void)taosThreadMutexLock(&pMgmt->lock); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 43f8ffe61b..cc880aa4de 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -674,7 +674,7 @@ static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { void *ptr = taosArrayPush(pArray, pAction); if (ptr == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } return 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2d979c3327..41ddc2ab32 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -139,7 +139,7 @@ const static SIpV4Range defaultIpRange = {.ip = 16777343, .mask = 32}; static int32_t ipWhiteMgtInit() { ipWhiteMgt.pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); if (ipWhiteMgt.pIpWhiteTab == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } ipWhiteMgt.ver = 0; (void)taosThreadRwlockInit(&ipWhiteMgt.rw, NULL); @@ -558,11 +558,11 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); if (pIpWhiteTab == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } pUserNames = taosArrayInit(8, sizeof(void *)); if (pUserNames == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } while (1) { @@ -593,7 +593,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { taosMemoryFree(name); sdbRelease(pSdb, pUser); sdbCancelFetch(pSdb, pIter); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } sdbRelease(pSdb, pUser); @@ -614,7 +614,7 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { } if (taosArrayPush(pUserNames, &name) == NULL) { taosMemoryFree(name); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } } @@ -1011,7 +1011,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } int32_t dataPos = 0; @@ -1155,7 +1155,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t num = pUser->pIpWhiteList->num; int32_t tlen = sizeof(SIpWhiteList) + num * sizeof(SIpV4Range) + 4; if ((buf = taosMemoryCalloc(1, tlen)) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER); + TAOS_CHECK_GOTO(terrno, NULL, _OVER); } int32_t len = 0; TAOS_CHECK_GOTO(tSerializeIpWhiteList(buf, tlen, pUser->pIpWhiteList, &len), &lino, _OVER); @@ -1201,12 +1201,12 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { pRow = sdbAllocRow(sizeof(SUserObj)); if (pRow == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } pUser = sdbGetRowObj(pRow); if (pUser == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } int32_t dataPos = 0; @@ -1239,7 +1239,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pUser->topics = taosHashInit(numOfTopics, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pUser->readDbs == NULL || pUser->writeDbs == NULL || pUser->topics == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); goto _OVER; } @@ -1302,7 +1302,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { if (pUser->readTbs == NULL || pUser->writeTbs == NULL || pUser->alterTbs == NULL || pUser->readViews == NULL || pUser->writeViews == NULL || pUser->alterViews == NULL || pUser->useDbs == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); goto _OVER; } @@ -1312,7 +1312,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); @@ -1321,7 +1321,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) @@ -1335,7 +1335,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); @@ -1344,7 +1344,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) @@ -1359,7 +1359,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); @@ -1368,7 +1368,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) @@ -1382,7 +1382,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); @@ -1391,7 +1391,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) @@ -1405,7 +1405,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); @@ -1414,7 +1414,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) @@ -1428,7 +1428,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); @@ -1437,7 +1437,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); TAOS_MEMORY_REALLOC(value, valuelen * sizeof(char)); if (value == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) @@ -1452,7 +1452,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, keyLen * sizeof(char)); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } (void)memset(key, 0, keyLen); SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); @@ -1470,7 +1470,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { TAOS_MEMORY_REALLOC(key, len); if (key == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } SDB_GET_BINARY(pRaw, dataPos, key, len, _OVER); @@ -1535,7 +1535,7 @@ int32_t mndDupTableHash(SHashObj *pOld, SHashObj **ppNew) { *ppNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (*ppNew == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } char *tb = taosHashIterate(pOld, NULL); @@ -1560,7 +1560,7 @@ int32_t mndDupUseDbHash(SHashObj *pOld, SHashObj **ppNew) { *ppNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (*ppNew == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } int32_t *db = taosHashIterate(pOld, NULL); @@ -1661,7 +1661,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { TAOS_MEMORY_REALLOC(pOld->pIpWhiteList, sz); if (pOld->pIpWhiteList == NULL) { taosWUnLockLatch(&pOld->lock); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(pOld->pIpWhiteList, pNew->pIpWhiteList, sz); pOld->ipWhiteListVer = pNew->ipWhiteListVer; @@ -1715,7 +1715,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } else { SHashObj *pUniqueTab = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); if (pUniqueTab == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } int32_t dummpy = 0; for (int i = 0; i < pCreate->numIpRanges; i++) { @@ -1764,7 +1764,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate if (pTrans == NULL) { mError("user:%s, failed to create since %s", pCreate->user, terrstr()); taosMemoryFree(userObj.pIpWhiteList); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } mInfo("trans:%d, used to create user:%s", pTrans->id, pCreate->user); @@ -1905,7 +1905,7 @@ int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) { } pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } contLen = tSerializeSGetUserWhiteListRsp(pRsp, contLen, &wlRsp); @@ -1952,7 +1952,7 @@ int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq) { pRsp = rpcMallocCont(len); if (!pRsp) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } len = tSerializeSUpdateIpWhite(pRsp, len, &ipWhite); if (len < 0) { @@ -2579,7 +2579,7 @@ static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq) { } pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } contLen = tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); @@ -2655,7 +2655,7 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl TAOS_MEMORY_REALLOC(varstr, VARSTR_HEADER_SIZE + tlen); if (varstr == NULL) { sdbRelease(pSdb, pUser); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } varDataSetLen(varstr, tlen); (void)memcpy(varDataVal(varstr), buf, tlen); @@ -2820,12 +2820,12 @@ static int32_t mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, i if (bufSz < 5) bufSz = 5; TAOS_MEMORY_REALLOC(*sql, bufSz + 1); if (*sql == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } TAOS_MEMORY_REALLOC(*condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if ((*condition) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -2852,7 +2852,7 @@ static int32_t mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, i } else { TAOS_MEMORY_REALLOC(*condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if ((*condition) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } STR_WITH_MAXSIZE_TO_VARSTR((*condition), "", pShow->pMeta->pSchemas[cols].bytes); @@ -2952,7 +2952,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { sdbRelease(pSdb, pUser); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -2995,7 +2995,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { sdbRelease(pSdb, pUser); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -3039,7 +3039,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { sdbRelease(pSdb, pUser); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -3093,7 +3093,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock TAOS_MEMORY_REALLOC(condition, TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (condition == NULL) { sdbRelease(pSdb, pUser); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -3137,7 +3137,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); if (batchRsp.pArray == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } for (int32_t i = 0; i < numOfUses; ++i) { @@ -3169,7 +3169,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ } if (!(taosArrayPush(batchRsp.pArray, &rsp))) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&rsp); TAOS_CHECK_GOTO(code, &lino, _OVER); @@ -3191,7 +3191,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ } pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } rspLen = tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); if (rspLen < 0) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 2483c7f0d1..0cb4a45f29 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1445,7 +1445,7 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD int32_t contLen = sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pHead->contLen = htonl(contLen); @@ -1484,7 +1484,7 @@ int32_t mndAddChangeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV SMsgHead *pHead = taosMemoryMalloc(totallen); if (pHead == NULL) { taosMemoryFree(pReq); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pHead->contLen = htonl(totallen); @@ -3067,7 +3067,7 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro if (dbObj.cfg.pRetensions != NULL) { dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); if (dbObj.cfg.pRetensions == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } } diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 576ba736b5..03098d93e0 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -140,7 +140,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { SHashObj *hash = taosHashInit(64, taosGetDefaultHashFunction(hashType), true, HASH_ENTRY_LOCK); if (hash == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pSdb->maxId[sdbType] = 0; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index fd4819dadd..34a017a907 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -226,7 +226,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100); if (pRaw == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; mError("failed read sdb file since %s", tstrerror(code)); TAOS_RETURN(code); } @@ -278,7 +278,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { bufLen = pRaw->dataLen * 2; SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100); if (pNewRaw == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen); goto _OVER; } @@ -306,7 +306,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen)); if (plantContent == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } @@ -444,7 +444,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { newDataLen = ENCRYPTED_LEN(pRaw->dataLen); newData = taosMemoryMalloc(newDataLen); if (newData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); break; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 5a2d1c981c..b83554c6f9 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -165,7 +165,6 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); - terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } From 007b64a3c188278b757542c754ddd8fed84418cd Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 12 Sep 2024 15:48:07 +0800 Subject: [PATCH 46/61] fix: void function call issue --- source/libs/executor/src/groupcacheoperator.c | 2 +- source/libs/scheduler/src/schJob.c | 2 +- source/libs/scheduler/src/schTask.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 2540579413..9f92679ad5 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -1061,7 +1061,7 @@ static int32_t getCacheBlkFromDownstreamOperator(struct SOperatorInfo* pOperator pWaitCtx->newFetch = true; code = taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId)); if (code) { - qError("taosHashRemove session %" PRId64 " from waitSession failed, error: %s", pSessionId, tstrerror(code)); + qError("taosHashRemove session %" PRId64 " from waitSession failed, error: %s", *pSessionId, tstrerror(code)); return code; } QRY_ERR_RET(tsem_post(&pWaitCtx->waitSem)); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 57a21643ef..744a58b866 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -529,7 +529,7 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); code = tsem_post(&pJob->rspSem); if (code) { - ctgError("tsem_post failed for syncOp, error:%s", tstrerror(code)); + SCH_JOB_ELOG("tsem_post failed for syncOp, error:%s", tstrerror(code)); } } else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 8e9d46de31..d1908b7c02 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -1355,7 +1355,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { SCH_LOCK_TASK(pTask); if (pTask->delayTimer) { if (!taosTmrStopA(&pTask->delayTimer)) { - SCH_TASK_WLOG("stop delayTimer failed, status:%d", pTask->delayTimer); + SCH_TASK_WLOG("stop delayTimer failed, status:%d", pTask->status); } } schDropTaskOnExecNode(pJob, pTask); From 3d50df7191157fde34651dd9dbf570889dec619c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Sep 2024 16:05:07 +0800 Subject: [PATCH 47/61] refactor: remove void. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 4 + source/libs/stream/src/streamCheckStatus.c | 2 +- source/libs/stream/src/streamHb.c | 2 +- source/libs/stream/src/streamMeta.c | 202 +++++++++++++++----- source/libs/stream/src/streamStartHistory.c | 5 +- source/libs/stream/src/streamStartTask.c | 38 ++-- source/libs/stream/src/streamTask.c | 5 +- 7 files changed, 192 insertions(+), 66 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c00d9a93bb..bde7e23318 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -1235,6 +1235,10 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { pMeta->vgId, req.taskId); // ignore this code to avoid error code over write int32_t ret = streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); + if (ret) { + tqError("s-task:0x%x failed add check downstream failed, core:%s", req.taskId, tstrerror(ret)); + } + return code; } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 41124d8543..48ca2b32e4 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -316,7 +316,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { pInfo->pList = NULL; if (pInfo->checkRspTmr != NULL) { - (void)taosTmrStop(pInfo->checkRspTmr); + streamTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; } diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 4e35cb718c..703e6a3256 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -331,7 +331,7 @@ void destroyMetaHbInfo(SMetaHbInfo* pInfo) { tCleanupStreamHbMsg(&pInfo->hbMsg); if (pInfo->hbTmr != NULL) { - (void) taosTmrStop(pInfo->hbTmr); + streamTmrStop(pInfo->hbTmr); pInfo->hbTmr = NULL; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index bc902ccf29..abeb03bb1c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -62,7 +62,12 @@ static void streamMetaEnvInit() { } } -void streamMetaInit() { (void)taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } +void streamMetaInit() { + int32_t code = taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); + if (code) { + stError("failed to init stream Meta model, code:%s", tstrerror(code)); + } +} void streamMetaCleanup() { taosCloseRef(streamBackendId); @@ -114,13 +119,17 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId)); if (p == NULL) { - SArray* list = taosArrayInit(8, sizeof(void*)); - p = taosArrayPush(list, &rid); + SArray* pList = taosArrayInit(8, POINTER_BYTES); + if (pList == NULL) { + return terrno; + } + + p = taosArrayPush(pList, &rid); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - code = taosHashPut(gMetaRefMgt.pTable, &vgId, sizeof(vgId), &list, sizeof(void*)); + code = taosHashPut(gMetaRefMgt.pTable, &vgId, sizeof(vgId), &pList, sizeof(void*)); if (code) { stError("vgId:%d failed to put into metaRef table, rid:%" PRId64, (int32_t)vgId, *rid); return code; @@ -180,8 +189,13 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { code = tdbTbcMoveToFirst(pCur); if (code) { - (void)tdbTbcClose(pCur); - stError("vgId:%d failed to open stream meta file cursor, not perform compatible check", pMeta->vgId); + stError("vgId:%d failed to open stream meta file cursor, not perform compatible check, code:%s", pMeta->vgId, + tstrerror(code)); + code = tdbTbcClose(pCur); + if (code) { + stError("vgId:%d failed to close meta file cursor, code:%s", pMeta->vgId, tstrerror(code)); + } + return ret; } @@ -209,7 +223,10 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { tdbFree(pKey); tdbFree(pVal); - (void)tdbTbcClose(pCur); + code = tdbTbcClose(pCur); + if (code != 0) { + stError("vgId:%d failed to close meta file cursor, code:%s", pMeta->vgId, tstrerror(code)); + } return ret; } @@ -351,8 +368,8 @@ void streamMetaRemoveDB(void* arg, char* key) { int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId, int64_t stage, startComplete_fn_t fn, SStreamMeta** p) { - *p = NULL; int32_t code = 0; + QRY_PARAM_CHECK(p); SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -484,9 +501,26 @@ _err: taosMemoryFree(pMeta->path); if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap); if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); - if (pMeta->pTaskDb) (void)tdbTbClose(pMeta->pTaskDb); - if (pMeta->pCheckpointDb) (void)tdbTbClose(pMeta->pCheckpointDb); - if (pMeta->db) (void)tdbClose(pMeta->db); + if (pMeta->pTaskDb) { + int32_t ret = tdbTbClose(pMeta->pTaskDb); + if (ret) { + stError("vgId:%d tdb failed close task db, code:%s", pMeta->vgId, tstrerror(ret)); + } + pMeta->pTaskDb = NULL; + } + if (pMeta->pCheckpointDb) { + int32_t ret = tdbTbClose(pMeta->pCheckpointDb); + if (ret) { + stError("vgId:%d tdb failed close task checkpointDb, code:%s", pMeta->vgId, tstrerror(ret)); + } + } + if (pMeta->db) { + int32_t ret = tdbClose(pMeta->db); + if (ret) { + stError("vgId:%d tdb failed close meta db, code:%s", pMeta->vgId, tstrerror(ret)); + } + } + if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo); if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks); if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet); @@ -532,7 +566,7 @@ void streamMetaClear(SStreamMeta* pMeta) { // release the ref by timer if (p->info.delaySchedParam != 0 && p->info.fillHistory == 0) { // one more ref in timer stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", p->id.idStr, p->refCnt); - (void)taosTmrStop(p->schedInfo.pDelayTimer); + streamTmrStop(p->schedInfo.pDelayTimer); p->info.delaySchedParam = 0; streamMetaReleaseTask(pMeta, p); } @@ -567,7 +601,10 @@ void streamMetaClose(SStreamMeta* pMeta) { if (pMeta == NULL) { return; } - (void)taosRemoveRef(streamMetaId, pMeta->rid); + int32_t code = taosRemoveRef(streamMetaId, pMeta->rid); + if (code) { + stError("vgId:%d failed to remove ref:%d, code:%s", pMeta->vgId, pMeta->rid, tstrerror(code)); + } } void streamMetaCloseImpl(void* arg) { @@ -576,6 +613,7 @@ void streamMetaCloseImpl(void* arg) { return; } + int32_t code = 0; int32_t vgId = pMeta->vgId; stDebug("vgId:%d start to do-close stream meta", vgId); @@ -584,10 +622,22 @@ void streamMetaCloseImpl(void* arg) { streamMetaWUnLock(pMeta); // already log the error, ignore here - (void)tdbAbort(pMeta->db, pMeta->txn); - (void)tdbTbClose(pMeta->pTaskDb); - (void)tdbTbClose(pMeta->pCheckpointDb); - (void)tdbClose(pMeta->db); + code = tdbAbort(pMeta->db, pMeta->txn); + if (code) { + stError("vgId:%d failed to jump of trans for tdb, code:%s", vgId, tstrerror(code)); + } + code = tdbTbClose(pMeta->pTaskDb); + if (code) { + stError("vgId:%d failed to close taskDb, code:%s", vgId, tstrerror(code)); + } + code = tdbTbClose(pMeta->pCheckpointDb); + if (code) { + stError("vgId:%d failed to close checkpointDb, code:%s", vgId, tstrerror(code)); + } + code = tdbClose(pMeta->db); + if (code) { + stError("vgId:%d failed to close db, code:%s", vgId, tstrerror(code)); + } taosArrayDestroy(pMeta->pTaskList); taosArrayDestroy(pMeta->chkpSaved); @@ -611,7 +661,10 @@ void streamMetaCloseImpl(void* arg) { bkdMgtDestroy(pMeta->bkdChkptMgt); pMeta->role = NODE_ROLE_UNINIT; - (void)taosThreadRwlockDestroy(&pMeta->lock); + code = taosThreadRwlockDestroy(&pMeta->lock); + if (code) { + stError("vgId:%d destroy rwlock, code:%s", vgId, tstrerror(code)); + } taosMemoryFree(pMeta); stDebug("vgId:%d end to close stream meta", vgId); @@ -711,7 +764,7 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa } if (pTask->info.fillHistory == 0) { - (void)atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); + int32_t val = atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); } *pAdded = true; @@ -786,20 +839,26 @@ static void doRemoveIdFromList(SArray* pTaskList, int32_t num, SStreamTaskId* id } static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { + int32_t code = 0; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - (void)streamTaskSendCheckpointSourceRsp(pTask); + code = streamTaskSendCheckpointSourceRsp(pTask); + if (code) { + stError("s-task:%s vgId:%d failed to send checkpoint-source rsp, code:%s", pTask->id.idStr, pTask->pMeta->vgId, + tstrerror(code)); + } } - return 0; + return code; } int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { SStreamTask* pTask = NULL; int32_t vgId = pMeta->vgId; + int32_t code = 0; + STaskId id = {.streamId = streamId, .taskId = taskId}; // pre-delete operation streamMetaWLock(pMeta); - STaskId id = {.streamId = streamId, .taskId = taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { pTask = *ppTask; @@ -811,12 +870,16 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } // handle the dropping event - (void)streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); + code = streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); + if (code) { + stError("s-task:0x%" PRIx64 " failed to handle dropping event async, code:%s", id.taskId, tstrerror(code)); + } } else { stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", vgId, taskId); streamMetaWUnLock(pMeta); return 0; } + streamMetaWUnLock(pMeta); stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, vgId); @@ -850,12 +913,15 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t pTask = *ppTask; // it is an fill-history task, remove the related stream task's id that points to it if (pTask->info.fillHistory == 0) { - (void)atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); + int32_t ret = atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); } - int32_t code = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); + code = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); doRemoveIdFromList(pMeta->pTaskList, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); - (void)streamMetaRemoveTask(pMeta, &id); + code = streamMetaRemoveTask(pMeta, &id); + if (code) { + stError("vgId:%d failed to remove task:0x%" PRIx64 ", code:%s", pMeta->vgId, id.taskId, tstrerror(code)); + } int32_t size = (int32_t) taosHashGetSize(pMeta->pTasksMap); int32_t sizeInList = taosArrayGetSize(pMeta->pTaskList); @@ -871,7 +937,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt); - (void)taosTmrStop(pTask->schedInfo.pDelayTimer); + streamTmrStop(pTask->schedInfo.pDelayTimer); pTask->info.delaySchedParam = 0; streamMetaReleaseTask(pMeta, pTask); } @@ -936,8 +1002,11 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { code = tdbTbcMoveToFirst(pCur); if (code) { - (void)tdbTbcClose(pCur); - stError("failed to open stream meta file cursor, the latest checkpointId is 0, vgId:%d", pMeta->vgId); + stError("failed to move stream meta file cursor, the latest checkpointId is 0, vgId:%d", pMeta->vgId); + int32_t ret = tdbTbcClose(pCur); + if (ret != 0) { + stError("vgId:%d failed to close meta file cursor, code:%s", pMeta->vgId, tstrerror(ret)); + } return checkpointId; } @@ -960,7 +1029,11 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { tdbFree(pKey); tdbFree(pVal); - (void)tdbTbcClose(pCur); + int32_t ret = tdbTbcClose(pCur); + if (ret != 0) { + stError("vgId:%d failed to close meta file cursor, code:%s", pMeta->vgId, tstrerror(ret)); + } + return checkpointId; } @@ -981,6 +1054,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { } pRecycleList = taosArrayInit(4, sizeof(STaskId)); + if (pRecycleList == NULL) { + stError("vgId:%d failed prepare load all tasks, code:out of memory", vgId); + return; + } vgId = pMeta->vgId; stInfo("vgId:%d load stream tasks from meta files", vgId); @@ -996,7 +1073,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { if (code) { stError("vgId:%d failed to open stream meta cursor, code:%s, not load any stream tasks", vgId, tstrerror(terrno)); taosArrayDestroy(pRecycleList); - (void)tdbTbcClose(pCur); + int32_t ret = tdbTbcClose(pCur); + if (ret != 0) { + stError("vgId:%d failed to close meta file cursor, code:%s", pMeta->vgId, tstrerror(ret)); + } return; } @@ -1072,11 +1152,11 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { } if (pTask->info.fillHistory == 0) { - (void)atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); + int32_t val = atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); } if (streamTaskShouldPause(pTask)) { - (void)atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); + int32_t val = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); } } @@ -1090,7 +1170,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { if (taosArrayGetSize(pRecycleList) > 0) { for (int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) { STaskId* pId = taosArrayGet(pRecycleList, i); - (void)streamMetaRemoveTask(pMeta, pId); + code = streamMetaRemoveTask(pMeta, pId); + if (code) { + stError("s-task:0x%" PRIx64 " failed to remove task, code:%s", pId->taskId, tstrerror(code)); + } } } @@ -1099,8 +1182,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { pMeta->numOfStreamTasks, pMeta->numOfPausedTasks); taosArrayDestroy(pRecycleList); - - (void)streamMetaCommit(pMeta); + code = streamMetaCommit(pMeta); + if (code) { + stError("vgId:%d failed to commit, code:%s", pMeta->vgId, tstrerror(code)); + } } bool streamMetaTaskInTimer(SStreamMeta* pMeta) { @@ -1117,7 +1202,10 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->status.timerActive >= 1) { stDebug("s-task:%s in timer, blocking tasks in vgId:%d restart, set closing again", pTask->id.idStr, pMeta->vgId); - (void)streamTaskStop(pTask); + int32_t code = streamTaskStop(pTask); + if (code) { + stError("s-task:%s failed to stop task, code:%s", pTask->id.idStr, tstrerror(code)); + } inTimer = true; } } @@ -1150,7 +1238,10 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { SStreamTask* pTask = *(SStreamTask**)pIter; stDebug("vgId:%d s-task:%s set task closing flag", vgId, pTask->id.idStr); - (void)streamTaskStop(pTask); + int32_t code = streamTaskStop(pTask); + if (code) { + stError("vgId:%d failed to stop task:0x%x, code:%s", vgId, pTask->id.taskId, tstrerror(code)); + } } streamMetaWUnLock(pMeta); @@ -1168,7 +1259,6 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { SArray* pTaskList = NULL; int32_t code = streamMetaSendMsgBeforeCloseTasks(pMeta, &pTaskList); if (code != TSDB_CODE_SUCCESS) { -// return code; } streamMetaRUnLock(pMeta); @@ -1199,14 +1289,17 @@ void streamMetaStartHb(SStreamMeta* pMeta) { void streamMetaRLock(SStreamMeta* pMeta) { // stTrace("vgId:%d meta-rlock", pMeta->vgId); - (void)taosThreadRwlockRdlock(&pMeta->lock); + int32_t code = taosThreadRwlockRdlock(&pMeta->lock); + if (code) { + stError("vgId:%d meta-rlock failed, code:%s", pMeta->vgId, tstrerror(code)); + } } void streamMetaRUnLock(SStreamMeta* pMeta) { // stTrace("vgId:%d meta-runlock", pMeta->vgId); int32_t code = taosThreadRwlockUnlock(&pMeta->lock); if (code != TSDB_CODE_SUCCESS) { - stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code); + stError("vgId:%d meta-runlock failed, code:%s", pMeta->vgId, tstrerror(code)); } else { // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); } @@ -1214,13 +1307,18 @@ void streamMetaRUnLock(SStreamMeta* pMeta) { void streamMetaWLock(SStreamMeta* pMeta) { // stTrace("vgId:%d meta-wlock", pMeta->vgId); - (void)taosThreadRwlockWrlock(&pMeta->lock); - // stTrace("vgId:%d meta-wlock completed", pMeta->vgId); + int32_t code = taosThreadRwlockWrlock(&pMeta->lock); + if (code) { + stError("vgId:%d failed to apply wlock, code:%s", pMeta->vgId, tstrerror(code)); + } } void streamMetaWUnLock(SStreamMeta* pMeta) { // stTrace("vgId:%d meta-wunlock", pMeta->vgId); - (void)taosThreadRwlockUnlock(&pMeta->lock); + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code) { + stError("vgId:%d failed to apply wunlock, code:%s", pMeta->vgId, tstrerror(code)); + } } int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { @@ -1258,7 +1356,7 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { streamMetaReleaseTask(pMeta, pTask); } - (void)streamMetaSendHbHelper(pMeta); + code = streamMetaSendHbHelper(pMeta); pMeta->sendMsgBeforeClosing = false; return TSDB_CODE_SUCCESS; // always return true } @@ -1348,9 +1446,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta streamMetaRUnLock(pMeta); // add the failed task info, along with the related fill-history task info into tasks list. - (void)streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); + code = streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); if (hasFillhistoryTask) { - (void)streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); + code = streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); } } else { streamMetaRUnLock(pMeta); @@ -1365,12 +1463,18 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { int32_t startTs = pTask->execInfo.checkTs; - (void)streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); + int32_t code = streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); + if (code) { + stError("s-task:%s failed to add self task failed to start", pTask->id.idStr, tstrerror(code)); + } // automatically set the related fill-history task to be failed. if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { STaskId* pId = &pTask->hTaskInfo.id; - (void)streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); + code = streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); + if (code) { + stError("s-task:0x%" PRIx64 " failed to add self task failed to start", pId->taskId, tstrerror(code)); + } } } diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 941b8f5145..ca2da9ef8e 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -459,7 +459,10 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { int32_t code = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId, &pInfo); if (code) { stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); - (void)streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); + int32_t ret = streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); + if (ret) { + stError("s-task:%s add task check downstream result failed, code:%s", idStr, tstrerror(ret)); + } return code; } diff --git a/source/libs/stream/src/streamStartTask.c b/source/libs/stream/src/streamStartTask.c index c2e8a523e5..168a8adc2d 100644 --- a/source/libs/stream/src/streamStartTask.c +++ b/source/libs/stream/src/streamStartTask.c @@ -62,9 +62,12 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); - if (pTask == NULL) { - stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); - (void)streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if ((pTask == NULL) || (code != 0)) { + stError("vgId:%d failed to acquire task:0x%x during start task, it may be dropped", pMeta->vgId, pTaskId->taskId); + int32_t ret = streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (ret) { + stError("s-task:0x%x add check downstream failed, core:%s", pTaskId->taskId, tstrerror(ret)); + } continue; } @@ -85,9 +88,13 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); - if (pTask == NULL) { + if ((pTask == NULL )|| (code != 0)) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); - (void)streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); + int32_t ret = streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (ret) { + stError("s-task:0x%x failed add check downstream failed, core:%s", pTaskId->taskId, tstrerror(ret)); + } + continue; } @@ -105,11 +112,11 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", pTask->id.idStr); - (void)streamLaunchFillHistoryTask(pTask); // todo: how about retry launch fill-history task? + code = streamLaunchFillHistoryTask(pTask); // todo: how about retry launch fill-history task? } - (void)streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, - true); + code = streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, + true); streamMetaReleaseTask(pMeta, pTask); continue; } @@ -333,9 +340,13 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId); code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask); - if (pTask == NULL) { + if ((pTask == NULL) || (code != 0)) { stError("vgId:%d failed to acquire task:0x%x when starting task", vgId, taskId); - (void)streamMetaAddFailedTask(pMeta, streamId, taskId); + int32_t ret = streamMetaAddFailedTask(pMeta, streamId, taskId); + if (ret) { + stError("s-task:0x%x add check downstream failed, core:%s", taskId, tstrerror(ret)); + } + return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } @@ -431,7 +442,10 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { continue; } - (void)streamTaskStop(pTask); + int32_t ret = streamTaskStop(pTask); + if (ret) { + stError("s-task:0x%x failed to stop task, code:%s", pTaskId->taskId, tstrerror(ret)); + } streamMetaReleaseTask(pMeta, pTask); } @@ -441,7 +455,7 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { stDebug("vgId:%d stop all %d task(s) completed, elapsed time:%.2f Sec.", pMeta->vgId, num, el); streamMetaRUnLock(pMeta); - return 0; + return code; } int32_t streamTaskCheckIfReqConsenChkptId(SStreamTask* pTask, int64_t ts) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 5d6cf39e40..79c87f942c 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -687,13 +687,14 @@ int32_t streamTaskStop(SStreamTask* pTask) { int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP); if (code) { - stError("failed to handle STOP event, s-task:%s", id); + stError("failed to handle STOP event, s-task:%s, code:%s", id, tstrerror(code)); + return code; } if (pTask->info.taskLevel != TASK_LEVEL__SINK && pTask->exec.pExecutor != NULL) { code = qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); if (code != TSDB_CODE_SUCCESS) { - stError("s-task:%s failed to kill task related query handle", id); + stError("s-task:%s failed to kill task related query handle, code:%s", id, tstrerror(code)); } } From 90d97849845e030dc33e91ac4e92eb545ebd84bd Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 12 Sep 2024 16:12:31 +0800 Subject: [PATCH 48/61] osEnv --- include/os/os.h | 2 ++ include/os/osEnv.h | 2 +- source/common/src/cos.c | 1 - source/dnode/mgmt/node_mgmt/src/dmEnv.c | 6 ++++- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 4 +++- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/tfs/src/tfsDisk.c | 4 ++-- source/os/src/osEnv.c | 10 +++++---- source/os/src/osFile.c | 27 ++++++++++++++--------- source/os/src/osString.c | 23 +++++++++++++------ source/os/src/osSysinfo.c | 8 +++---- source/util/src/tlog.c | 4 ++-- 12 files changed, 59 insertions(+), 34 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index 9e5e9221e4..165563f634 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -129,6 +129,8 @@ extern int64_t tsRandErrDivisor; extern int64_t tsRandErrScope; extern threadlocal bool tsEnableRandErr; +#define TAOS_UNUSED(expr) (void)(expr) + #ifdef __cplusplus } #endif diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 5f7cdf44dc..f235d3f235 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -55,7 +55,7 @@ extern SDiskSpace tsLogSpace; extern SDiskSpace tsTempSpace; int32_t osDefaultInit(); -void osUpdate(); +int32_t osUpdate(); void osCleanup(); bool osLogSpaceAvailable(); diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 82b1fad2ce..88f97a498d 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -1871,7 +1871,6 @@ void s3EvictCache(const char *path, long object_size) { taosDirName(dir_name); if (taosGetDiskSize((char *)dir_name, &disk_size) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); vError("failed to get disk:%s size since %s", path, terrstr()); return; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 620aed709f..f48ca35330 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -89,9 +89,13 @@ static bool dmDataSpaceAvailable() { } static int32_t dmCheckDiskSpace() { - osUpdate(); // availability int32_t code = 0; + code = osUpdate(); + if(code != 0) { + code = 0; // ignore the error, just log it + dError("failed to update os info since %s", tstrerror(code)); + } if (!dmDataSpaceAvailable()) { code = TSDB_CODE_NO_DISKSPACE; return code; diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 38bd9f6b92..f67901d6d5 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -159,7 +159,9 @@ int32_t dmRunDnode(SDnode *pDnode) { } if (count == 10) { - osUpdate(); + if(osUpdate() != 0) { + dError("failed to update os info"); + } count = 0; } else { count++; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 656af563e4..b568b03c2c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -232,7 +232,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche } int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); if (len < 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + TAOS_CHECK_GOTO(len, NULL, _exception); } memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 07fa8d79b9..35a118baad 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -33,7 +33,7 @@ int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path, pDisk->id = id; pDisk->disable = disable; if (taosGetDiskSize(pDisk->path, &pDisk->size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); // TODO: refactor this line + code = terrno; TAOS_CHECK_GOTO(code, &lino, _exit); } _exit: @@ -57,7 +57,7 @@ STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { int32_t tfsUpdateDiskSize(STfsDisk *pDisk) { if (taosGetDiskSize(pDisk->path, &pDisk->size) < 0) { - int32_t code = TAOS_SYSTEM_ERROR(errno); // TODO: refactor this line + int32_t code = terrno; fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, tstrerror(code)); TAOS_RETURN(code); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 8334d609b1..c0273e4a6f 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -94,16 +94,18 @@ int32_t osDefaultInit() { return code; } -void osUpdate() { +int32_t osUpdate() { + int code = 0; if (tsLogDir[0] != 0) { - (void)taosGetDiskSize(tsLogDir, &tsLogSpace.size); + code = taosGetDiskSize(tsLogDir, &tsLogSpace.size); } if (tsDataDir[0] != 0) { - (void)taosGetDiskSize(tsDataDir, &tsDataSpace.size); + code = taosGetDiskSize(tsDataDir, &tsDataSpace.size); } if (tsTempDir[0] != 0) { - (void)taosGetDiskSize(tsTempDir, &tsTempSpace.size); + code = taosGetDiskSize(tsTempDir, &tsTempSpace.size); } + return code; } void osCleanup() {} diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 94ead5c129..164626299b 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -174,9 +174,12 @@ int64_t taosCopyFile(const char *from, const char *to) { } code = taosFsyncFile(pFileTo); + if (code != 0) { + goto _err; + } - (void)taosCloseFile(&pFileFrom); - (void)taosCloseFile(&pFileTo); + TAOS_UNUSED(taosCloseFile(&pFileFrom)); + TAOS_UNUSED(taosCloseFile(&pFileTo)); if (code != 0) { terrno = code; @@ -187,10 +190,10 @@ int64_t taosCopyFile(const char *from, const char *to) { _err: - if (pFileFrom != NULL) (void)taosCloseFile(&pFileFrom); - if (pFileTo != NULL) (void)taosCloseFile(&pFileTo); + if (pFileFrom != NULL) TAOS_UNUSED(taosCloseFile(&pFileFrom)); + if (pFileTo != NULL) TAOS_UNUSED(taosCloseFile(&pFileTo)); /* coverity[+retval] */ - (void)taosRemoveFile(to); + TAOS_UNUSED(taosRemoveFile(to)); terrno = code; return -1; @@ -1120,8 +1123,8 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { (void)taosThreadRwlockWrlock(&((*ppFile)->rwlock)); #endif if ((*ppFile)->fp != NULL) { - (void)fflush((*ppFile)->fp); - (void)fclose((*ppFile)->fp); + TAOS_UNUSED(fflush((*ppFile)->fp)); + TAOS_UNUSED(fclose((*ppFile)->fp)); (*ppFile)->fp = NULL; } #ifdef WINDOWS @@ -1471,7 +1474,11 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { while (!feof(pSrcFile->fp)) { len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp); if (len > 0) { - (void)gzwrite(dstFp, data, len); + if(gzwrite(dstFp, data, len) == 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + ret = terrno; + goto cmp_end; + } } } @@ -1481,11 +1488,11 @@ cmp_end: (void)close(fd); } if (pSrcFile) { - (void)taosCloseFile(&pSrcFile); + TAOS_UNUSED(taosCloseFile(&pSrcFile)); } if (dstFp) { - (void)gzclose(dstFp); + TAOS_UNUSED(gzclose(dstFp)); } taosMemoryFree(data); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index e2e0c04515..ffc64f3493 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -94,6 +94,8 @@ int32_t taosStr2int64(const char *str, int64_t *val) { int64_t ret = strtoll(str, &endptr, 10); if (errno == ERANGE && (ret == LLONG_MAX || ret == LLONG_MIN)) { return TAOS_SYSTEM_ERROR(errno); + } else if (errno == EINVAL && ret == 0) { + return TSDB_CODE_INVALID_PARA; } else { *val = ret; return 0; @@ -106,7 +108,7 @@ int32_t taosStr2int16(const char *str, int16_t *val) { if (code) { return code; } else if (tmp > INT16_MAX || tmp < INT16_MIN) { - return TSDB_CODE_INVALID_PARA; + return TAOS_SYSTEM_ERROR(ERANGE); } else { *val = (int16_t)tmp; return 0; @@ -119,7 +121,7 @@ int32_t taosStr2int32(const char *str, int32_t *val) { if (code) { return code; } else if (tmp > INT32_MAX || tmp < INT32_MIN) { - return TSDB_CODE_INVALID_PARA; + return TAOS_SYSTEM_ERROR(ERANGE); } else { *val = (int32_t)tmp; return 0; @@ -132,7 +134,7 @@ int32_t taosStr2int8(const char *str, int8_t *val) { if (code) { return code; } else if (tmp > INT8_MAX || tmp < INT8_MIN) { - return TSDB_CODE_INVALID_PARA; + return TAOS_SYSTEM_ERROR(ERANGE); } else { *val = (int8_t)tmp; return 0; @@ -336,17 +338,20 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 #endif } +// if success, return the number of bytes written to mbs ( >= 0) +// otherwise return error code ( < 0) int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); - return -1; + terrno = TSDB_CODE_APP_ERROR; + return terrno; #else int32_t idx = -1; int32_t code = 0; iconv_t conv = taosAcquireConv(&idx, C2M); if ((iconv_t)-1 == conv || (iconv_t)0 == conv) { - return false; + return TSDB_CODE_APP_ERROR; } size_t ucs4_input_len = ucs4_max_len; @@ -364,10 +369,13 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { #endif } +// if success, return the number of bytes written to mbs ( >= 0) +// otherwise return error code ( < 0) int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); - return -1; + terrno = TSDB_CODE_APP_ERROR; + return terrno; #else size_t ucs4_input_len = ucs4_max_len; @@ -384,7 +392,8 @@ int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t c bool taosValidateEncodec(const char *encodec) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); - return true; + terrno = TSDB_CODE_APP_ERROR; + return false; #else iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); if (cd == (iconv_t)(-1)) { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 137bb8f0db..93fa9fa222 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -779,15 +779,15 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { return 0; } else { // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - // terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; } #elif defined(_TD_DARWIN_64) struct statvfs info; if (statvfs(dataDir, &info)) { // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - // terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } else { diskSize->total = info.f_blocks * info.f_frsize; diskSize->avail = info.f_bavail * info.f_frsize; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index bde3d48a6d..bd22a4e42b 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -215,7 +215,7 @@ int32_t taosInitSlowLog() { int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; - osUpdate(); + TAOS_CHECK_RETURN(osUpdate()); TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); if (tsc){ @@ -908,7 +908,7 @@ static void *taosAsyncOutputLog(void *param) { updateCron++; taosMsleep(writeInterval); if (count > 1000) { - osUpdate(); + TAOS_UNUSED(osUpdate()); count = 0; } From b35a435704bd9377fbd4fa58c7feb3485c4d350c Mon Sep 17 00:00:00 2001 From: Bo Ding Date: Thu, 12 Sep 2024 16:12:50 +0800 Subject: [PATCH 49/61] docs(taosx): update pi document --- docs/zh/06-advanced/05-data-in/03-pi.md | 151 +++++++++++++++++++++--- 1 file changed, 134 insertions(+), 17 deletions(-) diff --git a/docs/zh/06-advanced/05-data-in/03-pi.md b/docs/zh/06-advanced/05-data-in/03-pi.md index 2cffb4ce0d..6b65b1337f 100644 --- a/docs/zh/06-advanced/05-data-in/03-pi.md +++ b/docs/zh/06-advanced/05-data-in/03-pi.md @@ -7,21 +7,29 @@ sidebar_label: "PI" ## 功能概述 -PI 系统是一套用于数据收集、查找、分析、传递和可视化的软件产品,可以作为管理实时数据和事件的企业级系统的基础架构。 +PI 系统是一套用于数据收集、查找、分析、传递和可视化的软件产品,可以作为管理实时数据和事件的企业级系统的基础架构。taosX 可以 PI 连接器插件从 PI 系统中提取实时数据或历史数据。 -taosX 可以通过 PI 连接器插件从 PI 系统中提取实时数据。 +从数据的实时性角度来看 PI 数据源任务分为两类:**实时任务**和**回填任务**。在任务类型下拉列表两类任务分别对应名称: **PI** 和 **PI backfill**。 + +从数据模型角度来看, PI 数据源任务分为**单列模型**任务和**多列模型**任务: +1. **单列模型**任务将一个 PI Point 映射为 TDengine 一张表 +2. **多列模型**任务将一个 PI AF 元素映射为一张表 -有两类 PI 任务: 实时同步任务和历史数据迁移任务,在任务下拉列表的名称分别对应 “PI” 和 “PI backfill”。 +从连接的数据源类型来讲,PI 数据源任务又分为 **Archive Server** 数据源和 **AF Server** 数据源。对于 **Archive Server** 数据源,只能使用**单列模型**。对于 **AF Server** 数据源,既可以选择**单列模型**,又可以选择**多列模型**。 + +用户通过一个 CSV 文件配置从 PI 到 TDengine 的数据映射规则,这个 CSV 文件称为**模型配置文件**: +1. 对于使用 AF Server 单列模型的任务,taosX 会自动识别元素的哪些属性是引用 PI Point 的数据,并把一个 PI Point 属性对应到一张表。 +2. 对于使用 AF Server 的多列模型任务,一个元素对应一张表。taosX 默认会把 PI Point 属性映射到 TDengine Metric 列,把其它属性映射到 TDengine TAG 列。 ## 创建任务 -### 1. 新增数据源 +### 新增数据源 在数据写入页面中,点击 **+新增数据源** 按钮,进入新增数据源页面。 ![kafka-01.png](./kafka-01.png) -### 2. 配置基本信息 +### 基本配置 在 **名称** 中输入任务名称,如:“test”; @@ -33,7 +41,7 @@ taosX 可以通过 PI 连接器插件从 PI 系统中提取实时数据。 ![basic.png](./pic/pi-01-agent.png) -### 3. 配置连接信息 +### 连接配置 PI 连接器支持两种连接方式: @@ -46,21 +54,130 @@ PI 连接器支持两种连接方式: 点击 **连通性检查** 按钮,检查数据源是否可用。 -### 4. 配置数据模型 +### 数据模型配置 -这一部分,我们用一个 csv 文件配置 PI 的数据模型到 TDengine 的数据模型的映射规则。这里所说的映射规则包含 3 方面内容: +这一部分有两个 Tab,分别对应单列模型的配置和多列模型的配置。如果你是第一次配置,无论选择单列模型还是多列模型,请务必点击“下载默认配置”按钮。这个操作会触发生成默认的**模型配置文件**,同时也会把**模型配置文件**下载到本地,你可以查看或编辑。编辑后还可以再上传,来覆盖默认的配置。 -1. 定义源数据范围,即哪些点或哪些模板需要同步到 TDengine。 -2. 定义过滤规则,即符合什么条件的数据才需要同步到 TDengine。 -3. 定义转换规则,即对原始数据做什么样的变换后再写入 TDengine。 +如果你想同步所有点位或所有模板的元素,那么用默认的配置就好了。如果你想要过滤特定命名模式的点位或元素模板,那么需要先填写过滤条件再点击“下载默认配置”。 -如果您不知道具体怎么操作,可以点击“下载默认配置”按钮,下载得到的 csv 文件有详细的使用说明。 +#### 多列模型配置文件 -### 5. 其他配置 +下面是一个多列模型配置文件的示例。这个配置文件包含两个超级表的配置:一个是 metertemplate 表,它的数据来自模板 MeterTemplate 的元素;另一个是 farm 表,它的数据来自 Farm 模板的元素。 -其余的配置,比较重要的是: +```csv +SuperTable,metertemplate +SubTable,${element_name}_${element_id} +Template,MeterTemplate +Filter, +ts,KEY,TIMESTAMP,$ts +voltage,COLUMN,DOUBLE,$voltage +voltage_status,COLUMN,INT,$voltage_status +current,COLUMN,DOUBLE,$current +current_status,COLUMN,INT,$current_status +element_id,TAG,VARCHAR(100),$element_id +element_name,TAG,VARCHAR(100),$element_name +path,TAG,VARCHAR(100),$path +categories,TAG,VARCHAR(100),$categories -1. 对于 PI 任务,配置“重启补偿时间”,如果任务意外中断,重启时配置这个参数非常有用,它会让 taosX 自动 backfill 一段时间的数据。 -2. 对于 PI backfill 任务,需要配置 backfill 的开始和结束时间。 +SuperTable,farm +SubTable,${element_name}_${element_id} +Template,Farm +Filter, +ts,KEY,TIMESTAMP,$ts +wind_speed,COLUMN,FLOAT,$wind_speed +wind_speed_status,COLUMN,INT,$wind_speed_status +power_production,COLUMN,FLOAT,$power_production +power_production_status,COLUMN,INT,$power_production_status +lost_power,COLUMN,FLOAT,$lost_power +lost_power_status,COLUMN,INT,$lost_power_status +farm_lifetime_production__weekly_,COLUMN,FLOAT,$farm_lifetime_production__weekly_ +farm_lifetime_production__weekly__status,COLUMN,INT,$farm_lifetime_production__weekly__status +farm_lifetime_production__hourly_,COLUMN,FLOAT,$farm_lifetime_production__hourly_ +farm_lifetime_production__hourly__status,COLUMN,INT,$farm_lifetime_production__hourly__status +element_id,TAG,VARCHAR(100),$element_id +element_name,TAG,VARCHAR(100),$element_name +path,TAG,VARCHAR(100),$path +categories,TAG,VARCHAR(100),$categories +``` + +多列模型配置文件由一个或多个超级表的定义组成。每个超级表的配置都包括: + +1. 超级表和模板的对应关系 +2. 属性和 TDengine Metric 列的对应关系 +3. 属性和 TDengine TAG 列的对应关系 +4. 源数据过滤条件 +5. 对于每一列,无论是 Metrics 列还是 TAG 列,都可以配置一个映射规则,详见[零代码第三方数据接入](../)“数据提取、过滤和转换”部分 + +#### 单列模型配置文件 + +下面是一个单列模型配置文件的示例。 + +```csv +SuperTable,volt_float32 +SubTable,${point_name} +Filter, +ts,KEY,TIMESTAMP,$ts +value,COLUMN,FLOAT,$value +status,COLUMN,INT,$status +path,TAG,VARCHAR(200),$path +point_name,TAG,VARCHAR(100),$point_name +ptclassname,TAG,VARCHAR(100),$ptclassname +sourcetag,TAG,VARCHAR(100),$sourcetag +tag,TAG,VARCHAR(100),$tag +descriptor,TAG,VARCHAR(100),$descriptor +exdesc,TAG,VARCHAR(100),$exdesc +engunits,TAG,VARCHAR(100),$engunits +pointsource,TAG,VARCHAR(100),$pointsource +step,TAG,VARCHAR(100),$step +future,TAG,VARCHAR(100),$future +element_paths,TAG,VARCHAR(512),`$element_paths.replace("\\", ".")` + +SuperTable,milliampere_float32 +SubTable,${point_name} +Filter, +ts,KEY,TIMESTAMP,$ts +value,COLUMN,FLOAT,$value +status,COLUMN,INT,$status +path,TAG,VARCHAR(200),$path +point_name,TAG,VARCHAR(100),$point_name +ptclassname,TAG,VARCHAR(100),$ptclassname +sourcetag,TAG,VARCHAR(100),$sourcetag +tag,TAG,VARCHAR(100),$tag +descriptor,TAG,VARCHAR(100),$descriptor +exdesc,TAG,VARCHAR(100),$exdesc +engunits,TAG,VARCHAR(100),$engunits +pointsource,TAG,VARCHAR(100),$pointsource +step,TAG,VARCHAR(100),$step +future,TAG,VARCHAR(100),$future +element_paths,TAG,VARCHAR(512),`$element_paths.replace("\\", ".")` + +Meter_1000004_Voltage,POINT,volt_float32 +Meter_1000004_Current,POINT,milliampere_float32 +Meter_1000001_Voltage,POINT,volt_float32 +Meter_1000001_Current,POINT,milliampere_float32 +Meter_1000474_Voltage,POINT,volt_float32 +Meter_1000474_Current,POINT,milliampere_float32 +``` + +单列模型配置文件分为两个部分,第一部分通多列模型配置文件,由若干个超级表的定义组成。第二部分是点位列表,这一部分配置了点位和超级表的映射关系。默认生成的配置把 UOM 相同且数据类型相同的点映射到同一个超级表。 + +### Backfill 配置 + +1. 对于 PI 任务,可配置“重启补偿时间”,如果任务意外中断,重启时配置这个参数非常有用,它会让 taosX 自动回填一段时间的数据。 +2. 对于 PI backfill 任务,必须配置 backfill 的开始和结束时间。 + +### 高级选项 + +对于不同的任务类型高级选项部分有所不同。通用的高级选项有: +1. 连接器日志级别 +2. 连接器查询和发送数据的批次大小 +3. 单次读取最大延迟 + +对于**多列模型的实时任务**,还有以下开关选项: + +1. 是否同步新增的元素。如果打开,则 PI 连接器会监听模板下新增的元素,无需重启任务,即可自动同步新增元素的数据。 +2. 是否同步静态属性的变化。如果打开,PI 连接器会同步所有静态属性(非 PI Point 属性)的变化。也就是说如果在 PI AF Server 一个元素的静态属性值做了修改,那么 TDengine 表相应 TAG 的值也会修改。 +3. 是否同步删除元素的操作。如果打开,则 PI 连接器会监听配置的模板下删除元素的事件,并同步删除 TDengine 对应子表。 +4. 是否同步删除历史数据。如果打开,则对于某个元素的时序数据,如果在 PI 中某个时间的数据被删除了,TDengine 对应时间对应列的数据会被置空。 +5. 是否同步修改历史数据。如果打开,则对于某个元素的时序数据,如果在 PI 中历史数据被修改了,TDengine 对应时间的数据也会更新。 -高级配置部分可以配置连接器日志的级别、批次大小、和批次延迟。用于 Debug 和 性能优化。 \ No newline at end of file From 5e8f6557b88bb8fe5e1f8fd496592c8d41ab0d6a Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 12 Sep 2024 16:16:45 +0800 Subject: [PATCH 50/61] fix: (last) heap-buffer-overflow in tsdbCacheDel --- source/dnode/vnode/src/tsdb/tsdbCache.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 46e5f31530..07e45a3dc3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1874,9 +1874,9 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE TAOS_CHECK_EXIT(code); } else { if (!remainCols) { - remainCols = taosArrayInit(numCols * 2, sizeof(SLastKey)); + remainCols = taosArrayInit(numCols * 2, sizeof(SIdxKey)); } - if (!taosArrayPush(remainCols, &lastKey)) { + if (!taosArrayPush(remainCols, &(SIdxKey){i, lastKey})) { TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } } @@ -1904,7 +1904,9 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE code = terrno; goto _exit; } - ((SLastKey *)key)[0] = *(SLastKey *)taosArrayGet(remainCols, i); + SIdxKey* idxKey = taosArrayGet(remainCols, i); + + ((SLastKey *)key)[0] = idxKey->key; keys_list[i] = key; keys_list_sizes[i] = klen; @@ -1918,10 +1920,11 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE for (int i = 0; i < numKeys; ++i) { SLastCol *pLastCol = NULL; (void)tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); - SLastKey *pLastKey = (SLastKey *)keys_list[i]; + SIdxKey* idxKey = taosArrayGet(remainCols, i); + SLastKey *pLastKey = &idxKey->key; if (NULL != pLastCol && (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey)) { SLastCol noCacheCol = {.rowKey.ts = TSKEY_MIN, - .colVal = COL_VAL_NONE(pLastKey->cid, pTSchema->columns[i].type), + .colVal = COL_VAL_NONE(pLastKey->cid, pTSchema->columns[idxKey->idx].type), .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; if ((code = tsdbCachePutToRocksdb(pTsdb, pLastKey, &noCacheCol)) != TSDB_CODE_SUCCESS) { @@ -1969,7 +1972,7 @@ _exit: int32_t tsdbOpenCache(STsdb *pTsdb) { int32_t code = 0, lino = 0; - size_t cfgCapacity = pTsdb->pVnode->config.cacheLastSize * 1024 * 1024; + size_t cfgCapacity = (size_t)pTsdb->pVnode->config.cacheLastSize * 1024 * 1024; SLRUCache *pCache = taosLRUCacheInit(cfgCapacity, 0, .5); if (pCache == NULL) { From 4785d977f530e4b2a5aed9f1aa003c17c21f4ce6 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 12 Sep 2024 16:36:28 +0800 Subject: [PATCH 51/61] docs: add taosAdapter connection pool configuration description --- .../01-components/03-taosadapter.md | 196 ++++++++++-------- .../01-components/03-taosadapter.md | 196 ++++++++++-------- 2 files changed, 213 insertions(+), 179 deletions(-) diff --git a/docs/en/14-reference/01-components/03-taosadapter.md b/docs/en/14-reference/01-components/03-taosadapter.md index 50b4019fd7..00a8aa65c3 100644 --- a/docs/en/14-reference/01-components/03-taosadapter.md +++ b/docs/en/14-reference/01-components/03-taosadapter.md @@ -54,95 +54,102 @@ Command-line arguments take precedence over environment variables over configura ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" + --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --instanceId int instance ID. Env "TAOS_ADAPTER_INSTANCE_ID" (default 32) + --log.compress whether to compress old log. Env "TAOS_ADAPTER_LOG_COMPRESS" + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.level string log level (trace debug info warning error). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") + --log.reservedDiskSize string reserved disk size for log dir (KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_RESERVED_DISK_SIZE" (default "1GB") + --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration deprecated: log rotation time always 24 hours. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (trace debug info warning error). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" (default true) + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + --pool.maxWait int max count of waiting for connection. Env "TAOS_ADAPTER_POOL_MAX_WAIT" + --pool.waitTimeout int wait for connection timeout seconds. Env "TAOS_ADAPTER_POOL_WAIT_TIMEOUT" (default 60) + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") + --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp4") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --uploadKeeper.enable Whether to enable sending metrics to keeper. Env "TAOS_ADAPTER_UPLOAD_KEEPER_ENABLE" (default true) + --uploadKeeper.interval duration send to Keeper interval. Env "TAOS_ADAPTER_UPLOAD_KEEPER_INTERVAL" (default 15s) + --uploadKeeper.retryInterval duration retry interval. Env "TAOS_ADAPTER_UPLOAD_KEEPER_RETRY_INTERVAL" (default 5s) + --uploadKeeper.retryTimes uint retry times. Env "TAOS_ADAPTER_UPLOAD_KEEPER_RETRY_TIMES" (default 3) + --uploadKeeper.timeout duration send to Keeper timeout. Env "TAOS_ADAPTER_UPLOAD_KEEPER_TIMEOUT" (default 5s) + --uploadKeeper.url string Keeper url. Env "TAOS_ADAPTER_UPLOAD_KEEPER_URL" (default "http://127.0.0.1:6043/adapter_report") + -V, --version Print the version and exit ``` Note: @@ -163,6 +170,17 @@ For details on the CORS protocol, please refer to: [https://www.w3.org/wiki/CORS See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/blob/3.0/example/config/taosadapter.toml) for sample configuration files. +### Connection Pool Parameters + +When using the RESTful interface, TDengine connections are managed through a connection pool. The pool can be configured with the following parameters: + +- **`pool.maxConnect`**: The maximum number of connections allowed in the pool. The default value is 2 times the number of CPU cores. It is recommended to keep the default setting. +- **`pool.maxIdle`**: The maximum number of idle connections allowed in the pool. By default, this is the same as `pool.maxConnect`. It is recommended to keep the default setting. +- **`pool.idleTimeout`**: The idle timeout duration for connections. By default, connections never time out. It is recommended to keep the default setting. +- **`pool.waitTimeout`**: The timeout for acquiring a connection from the pool. The default value is 60 seconds. If a connection cannot be acquired within the timeout, an HTTP 503 status code will be returned. This parameter is available starting from version 3.3.3.0. +- **`pool.maxWait`**: The maximum number of requests waiting to acquire a connection from the pool. The default value is 0, meaning there is no limit. If the number of queued requests exceeds this value, new requests will return an HTTP 503 status code. This parameter is available starting from version 3.3.3.0. + + ## Feature List - RESTful interface diff --git a/docs/zh/14-reference/01-components/03-taosadapter.md b/docs/zh/14-reference/01-components/03-taosadapter.md index ed90f54bca..443b769dde 100644 --- a/docs/zh/14-reference/01-components/03-taosadapter.md +++ b/docs/zh/14-reference/01-components/03-taosadapter.md @@ -54,96 +54,102 @@ taosAdapter 支持通过命令行参数、环境变量和配置文件来进行 ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") - --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" + --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --instanceId int instance ID. Env "TAOS_ADAPTER_INSTANCE_ID" (default 32) + --log.compress whether to compress old log. Env "TAOS_ADAPTER_LOG_COMPRESS" + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.level string log level (trace debug info warning error). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") + --log.reservedDiskSize string reserved disk size for log dir (KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_RESERVED_DISK_SIZE" (default "1GB") + --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration deprecated: log rotation time always 24 hours. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (trace debug info warning error). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" (default true) + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + --pool.maxWait int max count of waiting for connection. Env "TAOS_ADAPTER_POOL_MAX_WAIT" + --pool.waitTimeout int wait for connection timeout seconds. Env "TAOS_ADAPTER_POOL_WAIT_TIMEOUT" (default 60) + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") + --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp4") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --uploadKeeper.enable Whether to enable sending metrics to keeper. Env "TAOS_ADAPTER_UPLOAD_KEEPER_ENABLE" (default true) + --uploadKeeper.interval duration send to Keeper interval. Env "TAOS_ADAPTER_UPLOAD_KEEPER_INTERVAL" (default 15s) + --uploadKeeper.retryInterval duration retry interval. Env "TAOS_ADAPTER_UPLOAD_KEEPER_RETRY_INTERVAL" (default 5s) + --uploadKeeper.retryTimes uint retry times. Env "TAOS_ADAPTER_UPLOAD_KEEPER_RETRY_TIMES" (default 3) + --uploadKeeper.timeout duration send to Keeper timeout. Env "TAOS_ADAPTER_UPLOAD_KEEPER_TIMEOUT" (default 5s) + --uploadKeeper.url string Keeper url. Env "TAOS_ADAPTER_UPLOAD_KEEPER_URL" (default "http://127.0.0.1:6043/adapter_report") + -V, --version Print the version and exit ``` 备注: @@ -164,6 +170,16 @@ AllowWebSockets 示例配置文件参见 [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/blob/3.0/example/config/taosadapter.toml)。 +### 连接池参数说明 + +在使用 RESTful 接口请求时,系统将通过连接池管理 TDengine 连接。连接池可通过以下参数进行配置: + +- **`pool.maxConnect`**:连接池允许的最大连接数,默认值为 2 倍 CPU 核心数。建议保持默认设置。 +- **`pool.maxIdle`**:连接池中允许的最大空闲连接数,默认与 `pool.maxConnect` 相同。建议保持默认设置。 +- **`pool.idleTimeout`**:连接空闲超时时间,默认永不超时。建议保持默认设置。 +- **`pool.waitTimeout`**:从连接池获取连接的超时时间,默认设置为 60 秒。如果在超时时间内未能获取连接,将返回 HTTP 状态码 503。该参数从版本 3.3.3.0 开始提供。 +- **`pool.maxWait`**:连接池中等待获取连接的请求数上限,默认值为 0,表示不限制。当排队请求数超过此值时,新的请求将返回 HTTP 状态码 503。该参数从版本 3.3.3.0 开始提供。 + ## 功能列表 - RESTful 接口 From a7da6a7d5118f1f46642d60a667007d67f71b447 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 12 Sep 2024 17:46:29 +0800 Subject: [PATCH 52/61] compare return --- source/libs/index/inc/indexComm.h | 2 +- source/libs/index/src/indexCache.c | 6 ++--- source/libs/index/src/indexComm.c | 40 ++++++++++++++++++------------ source/libs/index/src/indexTfile.c | 6 ++--- source/os/src/osDir.c | 4 +-- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 09fd2f3555..227a97e298 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -30,7 +30,7 @@ char* idxPackJsonData(SIndexTerm* itm); char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip); -typedef enum { MATCH, CONTINUE, BREAK } TExeCond; +typedef enum { MATCH, CONTINUE, BREAK, ERROR } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 1b31c1e50b..97d49ca6e0 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -153,9 +153,8 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - terrno = TSDB_CODE_SUCCESS; TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); - if (terrno != TSDB_CODE_SUCCESS) { + if (cond == ERROR) { code = terrno; goto _return; } @@ -335,10 +334,9 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR goto _return; } memcpy(p, c->colVal, strlen(c->colVal)); - terrno = TSDB_CODE_SUCCESS; cond = cmpFn(p + skip, term->colVal, dType); taosMemoryFree(p); - if (terrno != TSDB_CODE_SUCCESS) { + if (cond == ERROR) { code = terrno; goto _return; } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 8b2e6002aa..d97e27a317 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -118,6 +118,14 @@ static FORCE_INLINE TExeCond tCompareContains(void* a, void* b, int8_t type) { } return tCompare(func, QUERY_TERM, a, b, type); } + +#define CHECKCOMERROR(expr) \ + do { \ + if ((expr) != 0) { \ + return ERROR; \ + } \ + } while (0) + static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); if (func == NULL) { @@ -133,51 +141,51 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d #if 1 if (dtype == TSDB_DATA_TYPE_TIMESTAMP) { int64_t va; - taosStr2int64(a, &va); + CHECKCOMERROR(taosStr2int64(a, &va)); int64_t vb; - taosStr2int64(b, &vb); + CHECKCOMERROR(taosStr2int64(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) { uint8_t va; - taosStr2int8(a, &va); + CHECKCOMERROR(taosStr2int8(a, &va)); uint8_t vb; - taosStr2int8(b, &vb); + CHECKCOMERROR(taosStr2int8(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_TINYINT) { int8_t va; - taosStr2int8(a, &va); + CHECKCOMERROR(taosStr2int8(a, &va)); int8_t vb; - taosStr2int8(b, &vb); + CHECKCOMERROR(taosStr2int8(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_SMALLINT) { int16_t va; - taosStr2int16(a, &va); + CHECKCOMERROR(taosStr2int16(a, &va)); int16_t vb; - taosStr2int16(b, &vb); + CHECKCOMERROR(taosStr2int16(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_USMALLINT) { uint16_t va; - taosStr2int16(a, &va); + CHECKCOMERROR(taosStr2int16(a, &va)); uint16_t vb; - taosStr2int16(b, &vb); + CHECKCOMERROR(taosStr2int16(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_INT) { int32_t va; - taosStr2int32(a, &va); + CHECKCOMERROR(taosStr2int32(a, &va)); int32_t vb; - taosStr2int32(b, &vb); + CHECKCOMERROR(taosStr2int32(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_UINT) { uint32_t va; - taosStr2int32(a, &va); + CHECKCOMERROR(taosStr2int32(a, &va)); uint32_t vb; - taosStr2int32(b, &vb); + CHECKCOMERROR(taosStr2int32(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_BIGINT) { int64_t va; - taosStr2int64(a, &va); + CHECKCOMERROR(taosStr2int64(a, &va)); int64_t vb; - taosStr2int64(b, &vb); + CHECKCOMERROR(taosStr2int64(b, &vb)); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_UBIGINT) { uint64_t va, vb; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 7037242b65..2a47e3393c 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -367,9 +367,8 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, FstSlice* s = &rt->data; char* ch = (char*)fstSliceData(s, NULL); - terrno = TSDB_CODE_SUCCESS; TExeCond cond = cmpFn(ch, p, tem->colType); - if (TSDB_CODE_SUCCESS != terrno) { + if (ERROR == cond) { swsResultDestroy(rt); code = terrno; goto _return; @@ -520,10 +519,9 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt goto _return; } memcpy(tBuf, ch, sz); - terrno = TSDB_CODE_SUCCESS; cond = cmpFn(tBuf + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType)); taosMemoryFree(tBuf); - if (TSDB_CODE_SUCCESS != terrno) { + if (ERROR == cond) { swsResultDestroy(rt); code = terrno; goto _return; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index dcce5543b2..45fdd79837 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -104,7 +104,7 @@ void taosRemoveDir(const char *dirname) { if (taosDirEntryIsDir(de)) { taosRemoveDir(filename); } else { - (void)taosRemoveFile(filename); + TAOS_UNUSED(taosRemoveFile(filename)); // printf("file:%s is removed\n", filename); } } @@ -315,7 +315,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (fileSec <= 100) continue; int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { - (void)taosRemoveFile(filename); + TAOS_UNUSED(taosRemoveFile(filename)); uInfo("file:%s is removed, days:%d keepDays:%d, sed:%"PRId64, filename, days, keepDays, fileSec); } else { // printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); From a167815a3a290b059695f1b228d46165641170bb Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Tue, 10 Sep 2024 09:00:31 +0800 Subject: [PATCH 53/61] fix(query)[TD-31955]. Fix memory leak in exceptional scenarios --- source/util/src/tpagedbuf.c | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 32b9b6e18d..331afe06e5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -411,7 +411,7 @@ _error: return TSDB_CODE_OUT_OF_MEMORY; } -static char* doExtractPage(SDiskbasedBuf* pBuf, bool* newPage) { +static char* doExtractPage(SDiskbasedBuf* pBuf) { char* availablePage = NULL; if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { availablePage = evictBufPage(pBuf); @@ -425,7 +425,6 @@ static char* doExtractPage(SDiskbasedBuf* pBuf, bool* newPage) { if (availablePage == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; } - *newPage = true; } return availablePage; @@ -434,8 +433,7 @@ static char* doExtractPage(SDiskbasedBuf* pBuf, bool* newPage) { void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { pBuf->statis.getPages += 1; - bool newPage = false; - char* availablePage = doExtractPage(pBuf, &newPage); + char* availablePage = doExtractPage(pBuf); if (availablePage == NULL) { return NULL; } @@ -451,9 +449,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { code = lruListPushFront(pBuf->lruList, pi); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(pi); - if (newPage) { - taosMemoryFree(availablePage); - } + taosMemoryFree(availablePage); terrno = code; return NULL; } @@ -464,9 +460,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { // register page id info pi = registerNewPageInfo(pBuf, *pageId); if (pi == NULL) { - if (newPage) { - taosMemoryFree(availablePage); - } + taosMemoryFree(availablePage); return NULL; } @@ -480,7 +474,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { if (TSDB_CODE_SUCCESS == code) { pBuf->totalBufSize += pBuf->pageSize; } else { - if (newPage) taosMemoryFree(availablePage); + taosMemoryFree(availablePage); (void)taosArrayPop(pBuf->pIdList); (void)tSimpleHashRemove(pBuf->all, pageId, sizeof(int32_t)); taosMemoryFree(pi); @@ -538,8 +532,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { return (void*)(GET_PAYLOAD_DATA(*pi)); } else { // not in memory - bool newPage = false; - (*pi)->pData = doExtractPage(pBuf, &newPage); + (*pi)->pData = doExtractPage(pBuf); // failed to evict buffer page, return with error code. if ((*pi)->pData == NULL) { @@ -551,7 +544,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { int32_t code = lruListPushFront(pBuf->lruList, *pi); if (TSDB_CODE_SUCCESS != code) { - if (newPage) taosMemoryFree((*pi)->pData); + taosMemoryFree((*pi)->pData); terrno = code; return NULL; } @@ -561,9 +554,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { if (HAS_DATA_IN_DISK(*pi)) { int32_t code = loadPageFromDisk(pBuf, *pi); if (code != 0) { - if (newPage) { - taosMemoryFree((*pi)->pData); - } + taosMemoryFree((*pi)->pData); terrno = code; return NULL; From 55efe1b92b2656c123b58a07bb0969f6f303d61a Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 10:34:32 +0000 Subject: [PATCH 54/61] fix/restore-2-replica --- source/dnode/mnode/impl/inc/mndVgroup.h | 4 +-- source/dnode/mnode/impl/src/mndVgroup.c | 33 ++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 444c16083c..682a51a687 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -55,8 +55,8 @@ void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *p void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid); bool mndVgroupInDnode(SVgObj *pVgroup, int32_t dnodeId); -int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup, - SDnodeObj *pDnode); +int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup, SDnodeObj *pDnode, + SDnodeObj *pAnotherDnode); int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 2483c7f0d1..dd1af7ef90 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2847,8 +2847,8 @@ int32_t mndBuildRaftAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pO TAOS_RETURN(code); } -int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup, - SDnodeObj *pDnode) { +int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup, SDnodeObj *pDnode, + SDnodeObj *pAnotherDnode) { int32_t code = 0; SVgObj newVgroup = {0}; memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); @@ -2864,7 +2864,33 @@ int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj } } TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, &newVgroup.vnodeGid[selected])); - } else if (newVgroup.replica == 2 || newVgroup.replica == 3) { + } else if (newVgroup.replica == 2) { + for (int i = 0; i < newVgroup.replica; i++) { + if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) { + newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_LEARNER; + } else { + newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER; + } + } + TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pAnotherDnode)); + + for (int i = 0; i < newVgroup.replica; i++) { + if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) { + newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_LEARNER; + } else { + newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER; + } + } + TAOS_CHECK_RETURN(mndRestoreAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, pDnode)); + + for (int i = 0; i < newVgroup.replica; i++) { + newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) { + } + } + TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode)); + TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode)); + } else if (newVgroup.replica == 3) { for (int i = 0; i < newVgroup.replica; i++) { if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) { newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_LEARNER; @@ -2881,7 +2907,6 @@ int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj } TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode)); } - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); if (pVgRaw == NULL) { code = TSDB_CODE_MND_RETURN_VALUE_NULL; From 6202f13feefd666bfdc4d5341ca6755fc3f089f2 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 11:37:48 +0000 Subject: [PATCH 55/61] fix/restore-2-replica --- source/dnode/mnode/impl/src/mndVgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index dd1af7ef90..c9ce57c204 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2889,7 +2889,7 @@ int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj } } TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode)); - TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode)); + TAOS_CHECK_RETURN(mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pAnotherDnode)); } else if (newVgroup.replica == 3) { for (int i = 0; i < newVgroup.replica; i++) { if (newVgroup.vnodeGid[i].dnodeId == pDnode->id) { From 7b4116b28e0bd0efc1a035db376d610d7bdf7c46 Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Wed, 11 Sep 2024 18:08:39 +0800 Subject: [PATCH 56/61] fix(query)[TD-32027]. Fix null pointer access in exceptional cases --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 34 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 8f66df1ffb..29871c1089 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4035,26 +4035,30 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra return code; } -static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowIndex, SRowKey* pKey, SRowMerger* pMerger, +static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t* rowIndex, SRowKey* pKey, SRowMerger* pMerger, SVersionRange* pVerRange, int32_t step) { - while (rowIndex < pBlockData->nRow && rowIndex >= 0) { + int32_t code = 0; + while ((*rowIndex) < pBlockData->nRow && (*rowIndex) >= 0) { SRowKey cur; - tColRowGetKey(pBlockData, rowIndex, &cur); + tColRowGetKey(pBlockData, (*rowIndex), &cur); if (pkCompEx(&cur, pKey) != 0) { break; } - if (pBlockData->aVersion[rowIndex] > pVerRange->maxVer || pBlockData->aVersion[rowIndex] < pVerRange->minVer) { - rowIndex += step; + if (pBlockData->aVersion[(*rowIndex)] > pVerRange->maxVer || + pBlockData->aVersion[(*rowIndex)] < pVerRange->minVer) { + (*rowIndex) += step; continue; } - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - rowIndex += step; + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, (*rowIndex)); + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + (*rowIndex) += step; } - - return rowIndex; + return code; } typedef enum { @@ -4076,7 +4080,10 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) { - pDumpInfo->rowIndex = doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pVerRange, step); + code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pVerRange, step); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { *state = CHECK_FILEBLOCK_CONT; } @@ -4096,7 +4103,10 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { - pDumpInfo->rowIndex = doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pRange, step); + code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pRange, step); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } // all rows are consumed, let's try next file block From 82d83b2e6b95a233b8e0eafd3005a699f32983ae Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 12 Sep 2024 19:56:49 +0800 Subject: [PATCH 57/61] osSysinfo --- include/os/os.h | 6 ++ include/os/osSysinfo.h | 10 +-- include/util/taoserror.h | 1 + source/dnode/mgmt/node_util/src/dmUtil.c | 34 +++++++-- source/os/src/osDir.c | 4 +- source/os/src/osFile.c | 12 ++-- source/os/src/osLocale.c | 9 ++- source/os/src/osSysinfo.c | 90 +++++++++++++++--------- 8 files changed, 112 insertions(+), 54 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index 165563f634..e3808065dd 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -130,6 +130,12 @@ extern int64_t tsRandErrScope; extern threadlocal bool tsEnableRandErr; #define TAOS_UNUSED(expr) (void)(expr) +#define TAOS_SKIP_ERROR(expr) \ + { \ + int32_t _code = terrno; \ + (void)(expr); \ + terrno = _code; \ + } #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 5a76be1d1e..a6d8a6502f 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -39,15 +39,17 @@ int64_t taosGetOsUptime(); int32_t taosGetEmail(char *email, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen); int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); -void taosGetCpuCores(float *numOfCores, bool physical); -void taosGetCpuUsage(double *cpu_system, double *cpu_engine); +int32_t taosGetCpuCores(float *numOfCores, bool physical); +int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine); int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma, char* avx512); int32_t taosGetTotalMemory(int64_t *totalKB); int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); -void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); -void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); +int32_t taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); +void taosSetDefaultProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); +int32_t taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); +void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4e909b0a1c..eb329192a2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -156,6 +156,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_OUT_OF_BUFFER TAOS_DEF_ERROR_CODE(0, 0x0137) #define TSDB_CODE_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0138) #define TSDB_CODE_SOCKET_ERROR TAOS_DEF_ERROR_CODE(0, 0x0139) +#define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/dnode/mgmt/node_util/src/dmUtil.c b/source/dnode/mgmt/node_util/src/dmUtil.c index 68f4c9d0ff..f73f041f4a 100644 --- a/source/dnode/mgmt/node_util/src/dmUtil.c +++ b/source/dnode/mgmt/node_util/src/dmUtil.c @@ -55,14 +55,36 @@ void *dmSetMgmtHandle(SArray *pArray, tmsg_t msgType, void *nodeMsgFp, bool need } void dmGetMonitorSystemInfo(SMonSysInfo *pInfo) { - taosGetCpuUsage(&pInfo->cpu_system, &pInfo->cpu_engine); - taosGetCpuCores(&pInfo->cpu_cores, false); - (void)taosGetProcMemory(&pInfo->mem_engine); - (void)taosGetSysMemory(&pInfo->mem_system); + int32_t code = 0; + code = taosGetCpuUsage(&pInfo->cpu_system, &pInfo->cpu_engine); + if (code != 0) { + dError("failed to get cpu usage since %s", tstrerror(code)); + } + code = taosGetCpuCores(&pInfo->cpu_cores, false); + if (code != 0) { + dError("failed to get cpu cores since %s", tstrerror(code)); + } + code = taosGetProcMemory(&pInfo->mem_engine); + if (code != 0) { + dError("failed to get proc memory since %s", tstrerror(code)); + } + code = taosGetSysMemory(&pInfo->mem_system); + if (code != 0) { + dError("failed to get sys memory since %s", tstrerror(code)); + } pInfo->mem_total = tsTotalMemoryKB; pInfo->disk_engine = 0; pInfo->disk_used = tsDataSpace.size.used; pInfo->disk_total = tsDataSpace.size.total; - taosGetCardInfoDelta(&pInfo->net_in, &pInfo->net_out); - taosGetProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); + code = taosGetCardInfoDelta(&pInfo->net_in, &pInfo->net_out); + if (code != 0) { + dError("failed to get card info since %s", tstrerror(code)); + taosSetDefaultCardInfoDelta(&pInfo->net_in, &pInfo->net_out); + } + code = taosGetProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); + if (code != 0) { + dError("failed to get proc io delta since %s", tstrerror(code)); + taosSetDefaultProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); + } + return; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 45fdd79837..e06c1a6aac 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -323,8 +323,8 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { } } - (void)taosCloseDir(&pDir); - (void)rmdir(dirname); + TAOS_UNUSED(taosCloseDir(&pDir)); + TAOS_UNUSED(rmdir(dirname)); } int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 164626299b..0e5b6b71a1 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -190,10 +190,10 @@ int64_t taosCopyFile(const char *from, const char *to) { _err: - if (pFileFrom != NULL) TAOS_UNUSED(taosCloseFile(&pFileFrom)); - if (pFileTo != NULL) TAOS_UNUSED(taosCloseFile(&pFileTo)); + if (pFileFrom != NULL) TAOS_SKIP_ERROR(taosCloseFile(&pFileFrom)); + if (pFileTo != NULL) TAOS_SKIP_ERROR(taosCloseFile(&pFileTo)); /* coverity[+retval] */ - TAOS_UNUSED(taosRemoveFile(to)); + TAOS_SKIP_ERROR(taosRemoveFile(to)); terrno = code; return -1; @@ -1485,14 +1485,14 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { cmp_end: if (fd >= 0) { - (void)close(fd); + TAOS_SKIP_ERROR(close(fd)); } if (pSrcFile) { - TAOS_UNUSED(taosCloseFile(&pSrcFile)); + TAOS_SKIP_ERROR(taosCloseFile(&pSrcFile)); } if (dstFp) { - TAOS_UNUSED(gzclose(dstFp)); + TAOS_SKIP_ERROR(gzclose(dstFp)); } taosMemoryFree(data); diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 2f835a7a27..d7fbb05a5d 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -135,9 +135,14 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { str++; char *revisedCharset = taosCharsetReplace(str); - tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); - taosMemoryFree(revisedCharset); + if (NULL == revisedCharset) { + (void)strcpy(outCharset, "UTF-8"); + } else { + tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); + + taosMemoryFree(revisedCharset); + } // printf("charset not configured, set to system default:%s", outCharset); } else { strcpy(outCharset, "UTF-8"); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 93fa9fa222..f9c4f081ff 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -179,10 +179,8 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { char line[1024]; ssize_t bytes = taosGetsFile(pFile, sizeof(line), line); if (bytes < 0) { - code = terrno; - (void)taosCloseFile(&pFile); - terrno = code; - return code; + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); + return terrno; } char cpu[10] = {0}; @@ -230,9 +228,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { char line[1024] = {0}; ssize_t bytes = taosGetsFile(pFile, sizeof(line), line); if (bytes < 0) { - code = terrno; - (void)taosCloseFile(&pFile); - terrno = code; + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return code; } @@ -250,7 +246,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); #endif return 0; @@ -427,7 +423,7 @@ int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t if (++cnt >= 3) break; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return code; #endif } @@ -497,13 +493,13 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { if (strncmp(line, "processor", 9) == 0) coreCount += 1; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); if (code != 0 && (done & 1) == 0) { TdFilePtr pFile1 = taosOpenFile("/proc/device-tree/model", TD_FILE_READ | TD_FILE_STREAM); if (pFile1 != NULL) { ssize_t bytes = taosGetsFile(pFile1, maxLen, cpuModel); - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); if (bytes > 0) { code = 0; done |= 1; @@ -535,9 +531,9 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { // Returns the container's CPU quota if successful, otherwise returns the physical CPU cores static int32_t taosCntrGetCpuCores(float *numOfCores) { #ifdef WINDOWS - return -1; + return TSDB_CODE_UNSUPPORT_OS; #elif defined(_TD_DARWIN_64) - return -1; + return TSDB_CODE_UNSUPPORT_OS; #else TdFilePtr pFile = NULL; if (!(pFile = taosOpenFile(tsCpuQuotaFile, TD_FILE_READ | TD_FILE_STREAM))) { @@ -545,11 +541,11 @@ static int32_t taosCntrGetCpuCores(float *numOfCores) { } char qline[32] = {0}; if (taosGetsFile(pFile, sizeof(qline), qline) <= 0) { - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); goto _sys; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); float quota = taosStr2Float(qline, NULL); if (quota < 0) { goto _sys; @@ -561,11 +557,11 @@ static int32_t taosCntrGetCpuCores(float *numOfCores) { char pline[32] = {0}; if (taosGetsFile(pFile, sizeof(pline), pline) <= 0) { - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); goto _sys; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); float period = taosStr2Float(pline, NULL); float quotaCores = quota / period; @@ -575,10 +571,16 @@ static int32_t taosCntrGetCpuCores(float *numOfCores) { } else { *numOfCores = sysCores; } + if(*numOfCores <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } goto _end; _sys: *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + if(*numOfCores <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } _end: return 0; @@ -586,26 +588,35 @@ _end: #endif } -void taosGetCpuCores(float *numOfCores, bool physical) { +int32_t taosGetCpuCores(float *numOfCores, bool physical) { #ifdef WINDOWS SYSTEM_INFO info; GetSystemInfo(&info); *numOfCores = info.dwNumberOfProcessors; - return; + return 0; #elif defined(_TD_DARWIN_64) *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + if(*numOfCores <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } return; #else if (physical) { *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + if(*numOfCores <= 0) { + return TAOS_SYSTEM_ERROR(errno); + } } else { - (void)taosCntrGetCpuCores(numOfCores); + int code= taosCntrGetCpuCores(numOfCores); + if(code != 0) { + return code; + } } - return; + return 0; #endif } -void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { +int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine) { static int64_t lastSysUsed = -1; static int64_t lastSysTotal = -1; static int64_t lastProcTotal = -1; @@ -639,6 +650,7 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { lastSysTotal = curSysTotal; lastProcTotal = curProcTotal; } + return 0; } #define __cpuid_fix(level, a, b, c, d) \ @@ -860,7 +872,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int if (readIndex >= 4) break; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); if (readIndex < 4) { return -1; @@ -870,7 +882,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int #endif } -void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { +int32_t taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { static int64_t last_rchars = -1; static int64_t last_wchars = -1; static int64_t last_read_bytes = -1; @@ -879,7 +891,8 @@ void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, i static int64_t cur_wchars = 0; static int64_t cur_read_bytes = 0; static int64_t cur_write_bytes = 0; - if (taosGetProcIO(&cur_rchars, &cur_wchars, &cur_read_bytes, &cur_write_bytes) == 0) { + int32_t code = taosGetProcIO(&cur_rchars, &cur_wchars, &cur_read_bytes, &cur_write_bytes); + if (code == 0) { if(last_rchars >=0 && last_wchars >=0 && last_read_bytes >=0 && last_write_bytes >= 0){ *rchars = cur_rchars - last_rchars; *wchars = cur_wchars - last_wchars; @@ -897,11 +910,15 @@ void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, i last_read_bytes = cur_read_bytes; last_write_bytes = cur_write_bytes; } else { - *rchars = 0; - *wchars = 0; - *read_bytes = 0; - *write_bytes = 0; + return code; } + return 0; +} +void taosSetDefaultProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { + *rchars = 0; + *wchars = 0; + *read_bytes = 0; + *write_bytes = 0; } int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { @@ -953,18 +970,19 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { *transmit_bytes += o_tbytes; } - (void)taosCloseFile(&pFile); + TAOS_SKIP_ERROR(taosCloseFile(&pFile)); return 0; #endif } -void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { +int32_t taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { static int64_t last_receive_bytes = -1; static int64_t last_transmit_bytes = -1; int64_t cur_receive_bytes = 0; int64_t cur_transmit_bytes = 0; - if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) { + int32_t code = taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes); + if (code == 0) { if(last_receive_bytes >= 0 && last_transmit_bytes >= 0){ *receive_bytes = cur_receive_bytes - last_receive_bytes; *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; @@ -977,9 +995,13 @@ void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { last_receive_bytes = cur_receive_bytes; last_transmit_bytes = cur_transmit_bytes; } else { - *receive_bytes = 0; - *transmit_bytes = 0; + return code; } + return 0; +} +void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { + *receive_bytes = 0; + *transmit_bytes = 0; } void taosKillSystem() { From d41111d5f88dff0e6eb17ce29aca067ee99c6344 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 12 Sep 2024 20:12:58 +0800 Subject: [PATCH 58/61] fix: build on win --- source/libs/index/inc/indexComm.h | 2 +- source/libs/index/src/indexCache.c | 4 ++-- source/libs/index/src/indexTfile.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 227a97e298..173443dfc5 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -30,7 +30,7 @@ char* idxPackJsonData(SIndexTerm* itm); char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip); -typedef enum { MATCH, CONTINUE, BREAK, ERROR } TExeCond; +typedef enum { MATCH, CONTINUE, BREAK, FAILED } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 97d49ca6e0..828a3e2e9e 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -154,7 +154,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); - if (cond == ERROR) { + if (cond == FAILED) { code = terrno; goto _return; } @@ -336,7 +336,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR memcpy(p, c->colVal, strlen(c->colVal)); cond = cmpFn(p + skip, term->colVal, dType); taosMemoryFree(p); - if (cond == ERROR) { + if (cond == FAILED) { code = terrno; goto _return; } diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 2a47e3393c..eb00cc7990 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -368,7 +368,7 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, char* ch = (char*)fstSliceData(s, NULL); TExeCond cond = cmpFn(ch, p, tem->colType); - if (ERROR == cond) { + if (FAILED == cond) { swsResultDestroy(rt); code = terrno; goto _return; @@ -521,7 +521,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt memcpy(tBuf, ch, sz); cond = cmpFn(tBuf + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType)); taosMemoryFree(tBuf); - if (ERROR == cond) { + if (FAILED == cond) { swsResultDestroy(rt); code = terrno; goto _return; From 2643d7ad6381f5cfc25a720b6de6ff986722b117 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 12 Sep 2024 21:57:58 +0800 Subject: [PATCH 59/61] build error --- source/libs/index/src/indexComm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index d97e27a317..0fab194044 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -122,7 +122,7 @@ static FORCE_INLINE TExeCond tCompareContains(void* a, void* b, int8_t type) { #define CHECKCOMERROR(expr) \ do { \ if ((expr) != 0) { \ - return ERROR; \ + return FAILED; \ } \ } while (0) From 466e0b5f8e37f5ecd3148d6e76e43f8d15f4886b Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 13 Sep 2024 07:19:11 +0800 Subject: [PATCH 60/61] build on mac --- source/os/src/osSysinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f9c4f081ff..36b1d7b60d 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -599,7 +599,7 @@ int32_t taosGetCpuCores(float *numOfCores, bool physical) { if(*numOfCores <= 0) { return TAOS_SYSTEM_ERROR(errno); } - return; + return 0; #else if (physical) { *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); From d255e2a7a514bdf684e874936b5dc8b78f5a7847 Mon Sep 17 00:00:00 2001 From: charles Date: Fri, 13 Sep 2024 07:57:23 +0800 Subject: [PATCH 61/61] add test case for ts-5389 by charles --- tests/army/query/last/test_last.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/army/query/last/test_last.py b/tests/army/query/last/test_last.py index 287cc97974..5c13155cc6 100644 --- a/tests/army/query/last/test_last.py +++ b/tests/army/query/last/test_last.py @@ -174,6 +174,19 @@ class TDTestCase(TBase): tdSql.checkData(0, 2, None) tdLog.info("Finish test_last_with_primarykey_str_rt") + def test_ts5389(self): + """add test case to cover the crash issue of ts-5389 + """ + tdSql.execute("create database db_ts5389;") + tdSql.execute("use db_ts5389;") + tdSql.execute("create stable trackers(ts timestamp, reg_firmware_rev double) tags(site nchar(8), tracker nchar(16), zone nchar(2));") + tdSql.execute("create table tr1 using trackers tags ('MI-01', 'N29-26', '12');") + tdSql.execute("create table tr2 using trackers tags ('MI-01', 'N29-6', '11');") + tdSql.execute("insert into tr1 values(now,null);") + tdSql.execute("insert into tr2 values(now,null);") + tdSql.query("select distinct site,zone,tracker,last(reg_firmware_rev) from trackers where ts > now() -1h and site='MI-01' partition by site;") + tdSql.checkRows(1) + def run(self): self.prepare_data() # regular table @@ -182,9 +195,12 @@ class TDTestCase(TBase): # child tables self.test_last_with_primarykey_int_ct() self.test_last_with_primarykey_str_ct() + # ts-5389 + self.test_ts5389() def stop(self): tdSql.execute("drop database db_td30816;") + tdSql.execute("drop database db_ts5389;") tdSql.close() tdLog.success("%s successfully executed" % __file__)