From a4962b58a6fb06c594ef3949cb657c3ecabe2af7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 5 Sep 2024 17:52:21 +0800 Subject: [PATCH 01/20] add trace --- source/util/src/tref.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 0eac7b4427..d3fafa4167 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -19,10 +19,11 @@ #include "tlog.h" #include "tutil.h" -#define TSDB_REF_OBJECTS 50 -#define TSDB_REF_STATE_EMPTY 0 -#define TSDB_REF_STATE_ACTIVE 1 -#define TSDB_REF_STATE_DELETED 2 +#define TSDB_REF_OBJECTS 50 +#define TSDB_REF_STATE_EMPTY 0 +#define TSDB_REF_STATE_ACTIVE 1 +#define TSDB_REF_STATE_DELETED 2 +#define TSDB_REF_ITER_THRESHOLD 100 typedef struct SRefNode { struct SRefNode *prev; // previous node @@ -188,6 +189,7 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid) { int32_t hash; SRefNode *pNode; SRefSet *pSet; + int32_t iter = 0; void *p = NULL; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { @@ -220,10 +222,14 @@ void *taosAcquireRef(int32_t rsetId, int64_t rid) { if (pNode->rid == rid) { break; } - + iter++; pNode = pNode->next; } + if (iter >= TSDB_REF_ITER_THRESHOLD) { + uWarn("rsetId:%d rid:%" PRId64 " iter:%d", rsetId, rid, iter); + } + if (pNode) { if (pNode->removed == 0) { pNode->count++; @@ -277,6 +283,7 @@ void *taosIterateRef(int32_t rsetId, int64_t rid) { do { newP = NULL; int32_t hash = 0; + int32_t iter = 0; if (rid > 0) { hash = rid % pSet->max; taosLockList(pSet->lockedBy + hash); @@ -285,6 +292,11 @@ void *taosIterateRef(int32_t rsetId, int64_t rid) { while (pNode) { if (pNode->rid == rid) break; pNode = pNode->next; + iter++; + } + + if (iter >= TSDB_REF_ITER_THRESHOLD) { + uWarn("rsetId:%d rid:%" PRId64 " iter:%d", rsetId, rid, iter); } if (pNode == NULL) { @@ -376,6 +388,7 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { int32_t hash; SRefSet *pSet; SRefNode *pNode; + int32_t iter = 0; int32_t released = 0; int32_t code = 0; @@ -403,6 +416,11 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { if (pNode->rid == rid) break; pNode = pNode->next; + iter++; + } + + if (iter >= TSDB_REF_ITER_THRESHOLD) { + uWarn("rsetId:%d rid:%" PRId64 " iter:%d", rsetId, rid, iter); } if (pNode) { From 0b21b7dff175950abba00dcb4d551afac2dd09af Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 14:25:50 +0800 Subject: [PATCH 02/20] fix indirect leak --- source/libs/transport/src/trans.c | 12 ++- source/libs/transport/src/transCli.c | 109 ++++++++------------ source/libs/transport/src/transSvr.c | 143 ++++++++++----------------- 3 files changed, 98 insertions(+), 166 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1c783f44fe..f017909341 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -103,7 +103,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->timeToGetConn = 10 * 1000; } pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; - + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -124,14 +124,20 @@ _end: } void rpcClose(void* arg) { tInfo("start to close rpc"); + if (arg == NULL) { + return; + } (void)transRemoveExHandle(transGetInstMgt(), (int64_t)arg); (void)transReleaseExHandle(transGetInstMgt(), (int64_t)arg); tInfo("end to close rpc"); return; } void rpcCloseImpl(void* arg) { + if (arg == NULL) return; SRpcInfo* pRpc = (SRpcInfo*)arg; - (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + if (pRpc->tcphandle != NULL) { + (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + } taosMemoryFree(pRpc); } @@ -168,7 +174,7 @@ int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64 return transSendRequest(shandle, pEpSet, pMsg, NULL); } int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { - if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0|| pRid == NULL) { + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { return transSendRequest(shandle, pEpSet, pMsg, pCtx); } else { return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 78320c450c..5052d79ef3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -99,12 +99,11 @@ typedef struct SCliMsg { } SCliMsg; typedef struct SCliThrd { - TdThread thread; // tid - int64_t pid; // pid - uv_loop_t* loop; - SAsyncPool* asyncPool; - uv_prepare_t* prepare; - void* pool; // conn pool + TdThread thread; // tid + int64_t pid; // pid + uv_loop_t* loop; + SAsyncPool* asyncPool; + void* pool; // conn pool // timer handles SArray* timerList; // msg queue @@ -167,7 +166,6 @@ static void cliSendCb(uv_write_t* req, int status); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); static void cliIdleCb(uv_idle_t* handle); -static void cliPrepareCb(uv_prepare_t* handle); static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd); static void cliSendBatchCb(uv_write_t* req, int status); @@ -231,7 +229,9 @@ static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static int32_t createThrdObj(void* trans, SCliThrd** pThrd); static void destroyThrdObj(SCliThrd* pThrd); -static void cliWalkCb(uv_handle_t* handle, void* arg); + +int32_t cliSendQuit(SCliThrd* thrd); +static void cliWalkCb(uv_handle_t* handle, void* arg); #define CLI_RELEASE_UV(loop) \ do { \ @@ -2119,33 +2119,6 @@ static void cliAsyncCb(uv_async_t* handle) { if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); } -static void cliPrepareCb(uv_prepare_t* handle) { - SCliThrd* thrd = handle->data; - tTrace("prepare work start"); - - SAsyncPool* pool = thrd->asyncPool; - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - (void)taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); - - int count = 0; - while (!QUEUE_IS_EMPTY(&wq)) { - queue* h = QUEUE_HEAD(&wq); - QUEUE_REMOVE(h); - - SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - (*cliAsyncHandle[pMsg->type])(pMsg, thrd); - count++; - } - } - tTrace("prepare work end"); - if (thrd->stopMsg != NULL) cliHandleQuit(thrd->stopMsg, thrd); -} void cliDestroyConnMsgs(SCliConn* conn, bool destroy) { transCtxCleanup(&conn->ctx); @@ -2260,6 +2233,12 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, _err: if (cli) { + for (int i = 0; i < cli->numOfThreads; i++) { + if (cli->pThreadObj[i]) { + (void)cliSendQuit(cli->pThreadObj[i]); + destroyThrdObj(cli->pThreadObj[i]); + } + } taosMemoryFree(cli->pThreadObj); taosMemoryFree(cli); } @@ -2339,37 +2318,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(code, NULL, _end); } - pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); - if (pThrd->prepare == NULL) { - tError("failed to create prepre since:%s", tstrerror(code)); - TAOS_CHECK_GOTO(code, NULL, _end); - } - - code = uv_prepare_init(pThrd->loop, pThrd->prepare); - if (code != 0) { - tError("failed to create prepre since:%s", uv_err_name(code)); - TAOS_CHECK_GOTO(TSDB_CODE_THIRDPARTY_ERROR, NULL, _end); - } - pThrd->prepare->data = pThrd; - - int32_t timerSize = 64; - pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); - if (pThrd->timerList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - - for (int i = 0; i < timerSize; i++) { - uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); - if (timer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - (void)uv_timer_init(pThrd->loop, timer); - if (taosArrayPush(pThrd->timerList, &timer) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - } - pThrd->pool = createConnPool(4); if (pThrd->pool == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -2402,6 +2350,23 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); } + int32_t timerSize = 64; + pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); + if (pThrd->timerList == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + for (int i = 0; i < timerSize; i++) { + uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + if (timer == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + (void)uv_timer_init(pThrd->loop, timer); + if (taosArrayPush(pThrd->timerList, &timer) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + } pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = trans; pThrd->quit = false; @@ -2411,17 +2376,21 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { _end: if (pThrd) { + (void)taosThreadMutexDestroy(&pThrd->msgMtx); + (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); - taosMemoryFree(pThrd->prepare); - (void)taosThreadMutexDestroy(&pThrd->msgMtx); transAsyncPoolDestroy(pThrd->asyncPool); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); - taosMemoryFree(pThrd->prepare); + + destroyConnPool(pThrd); + transDQDestroy(pThrd->delayQueue, NULL); + transDQDestroy(pThrd->timeoutQueue, NULL); + transDQDestroy(pThrd->waitConnQueue, NULL); taosHashCleanup(pThrd->fqdn2ipCache); taosHashCleanup(pThrd->failFastCache); taosHashCleanup(pThrd->batchCache); @@ -2450,8 +2419,8 @@ static void destroyThrdObj(SCliThrd* pThrd) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } + uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); - taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); taosHashCleanup(pThrd->failFastCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 11aa468b19..0727e49d8b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -82,14 +82,13 @@ typedef struct { int64_t ver; } SIpWhiteListTab; typedef struct SWorkThrd { - TdThread thread; - uv_connect_t connect_req; - uv_pipe_t* pipe; - uv_os_fd_t fd; - uv_loop_t* loop; - SAsyncPool* asyncPool; - uv_prepare_t* prepare; - queue msg; + TdThread thread; + uv_connect_t connect_req; + uv_pipe_t* pipe; + uv_os_fd_t fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + queue msg; queue conn; void* pTransInst; @@ -98,6 +97,7 @@ typedef struct SWorkThrd { SIpWhiteListTab* pWhiteList; int64_t whiteListVer; int8_t enableIpWhiteList; + int8_t inited; } SWorkThrd; typedef struct SServerObj { @@ -139,7 +139,6 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); static void uvShutDownCb(uv_shutdown_t* req, int status); -static void uvPrepareCb(uv_prepare_t* handle); static bool uvRecvReleaseReq(SSvrConn* conn, STransMsgHead* pHead); @@ -180,6 +179,11 @@ static void uvDestroyConn(uv_handle_t* handle); static void* transWorkerThread(void* arg); static void* transAcceptThread(void* arg); +static void destroyWorkThrd(SWorkThrd* pThrd); +static void destroyWorkThrdObj(SWorkThrd* pThrd); + +static void sendQuitToWorkThrd(SWorkThrd* pThrd); + // add handle loop static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); @@ -849,52 +853,6 @@ static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { } return false; } -static void uvPrepareCb(uv_prepare_t* handle) { - // prepare callback - SWorkThrd* pThrd = handle->data; - SAsyncPool* pool = pThrd->asyncPool; - - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - (void)taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); - - while (!QUEUE_IS_EMPTY(&wq)) { - queue* head = QUEUE_HEAD(&wq); - QUEUE_REMOVE(head); - - SSvrMsg* msg = QUEUE_DATA(head, SSvrMsg, q); - if (msg == NULL) { - tError("unexcept occurred, continue"); - continue; - } - // release handle to rpc init - if (msg->type == Quit || msg->type == Update) { - (*transAsyncHandle[msg->type])(msg, pThrd); - continue; - } else { - STransMsg transMsg = msg->msg; - - SExHandle* exh1 = transMsg.info.handle; - int64_t refId = transMsg.info.refId; - SExHandle* exh2 = transAcquireExHandle(transGetSvrRefMgt(), refId); - if (exh2 == NULL || exh1 != exh2) { - tTrace("handle except msg %p, ignore it", exh1); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); - destroySmsg(msg); - continue; - } - msg->pConn = exh1->handle; - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); - (*transAsyncHandle[msg->type])(msg, pThrd); - } - } - } -} static void uvWorkDoTask(uv_work_t* req) { // doing time-consumeing task @@ -1101,25 +1059,6 @@ static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { QUEUE_INIT(&pThrd->msg); - pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); - if (pThrd->prepare == NULL) { - tError("failed to init prepare"); - return TSDB_CODE_OUT_OF_MEMORY; - } - - code = uv_prepare_init(pThrd->loop, pThrd->prepare); - if (code != 0) { - tError("failed to init prepare since %s", uv_err_name(code)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - - code = uv_prepare_start(pThrd->prepare, uvPrepareCb); - if (code != 0) { - tError("failed to start prepare since %s", uv_err_name(code)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - pThrd->prepare->data = pThrd; - // conn set QUEUE_INIT(&pThrd->conn); @@ -1421,11 +1360,11 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - if (false == taosValidIpAndPort(srv->ip, srv->port)) { - code = TAOS_SYSTEM_ERROR(errno); - tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - goto End; - } + // if (false == taosValidIpAndPort(srv->ip, srv->port)) { + // code = TAOS_SYSTEM_ERROR(errno); + // tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + // goto End; + // } char pipeName[PATH_MAX]; #if defined(WINDOWS) || defined(DARWIN) @@ -1485,12 +1424,14 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, code = TSDB_CODE_OUT_OF_MEMORY; goto End; } + srv->pThreadObj[i] = thrd; thrd->pTransInst = shandle; thrd->quit = false; thrd->pTransInst = shandle; thrd->pWhiteList = uvWhiteListCreate(); if (thrd->pWhiteList == NULL) { + destroyWorkThrdObj(thrd); code = TSDB_CODE_OUT_OF_MEMORY; goto End; } @@ -1501,8 +1442,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - srv->pThreadObj[i] = thrd; - uv_os_sock_t fds[2]; if ((code = uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE)) != 0) { tError("failed to create pipe, errmsg: %s", uv_err_name(code)); @@ -1539,6 +1478,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, tError("failed to create worker-thread:%d", i); goto End; } + thrd->inited = 1; } #endif @@ -1560,6 +1500,12 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->inited = true; return srv; End: + for (int i = 0; i < srv->numOfThreads; i++) { + if (srv->pThreadObj[i] != NULL) { + SWorkThrd* thrd = srv->pThreadObj[i]; + destroyWorkThrd(thrd); + } + } transCloseServer(srv); terrno = code; return NULL; @@ -1663,20 +1609,27 @@ void uvHandleUpdate(SSvrMsg* msg, SWorkThrd* thrd) { taosMemoryFree(msg); } +void destroyWorkThrdObj(SWorkThrd* pThrd) { + if (pThrd == NULL) { + return; + } + transAsyncPoolDestroy(pThrd->asyncPool); + uvWhiteListDestroy(pThrd->pWhiteList); + uv_loop_close(pThrd->loop); + taosMemoryFree(pThrd->loop); + taosMemoryFree(pThrd); +} void destroyWorkThrd(SWorkThrd* pThrd) { if (pThrd == NULL) { return; } - (void)taosThreadJoin(pThrd->thread, NULL); - SRV_RELEASE_UV(pThrd->loop); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); - transAsyncPoolDestroy(pThrd->asyncPool); - - uvWhiteListDestroy(pThrd->pWhiteList); - - taosMemoryFree(pThrd->prepare); - taosMemoryFree(pThrd->loop); - taosMemoryFree(pThrd); + if (pThrd->inited) { + sendQuitToWorkThrd(pThrd); + (void)taosThreadJoin(pThrd->thread, NULL); + SRV_RELEASE_UV(pThrd->loop); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); + } + destroyWorkThrdObj(pThrd); } void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); @@ -1693,13 +1646,15 @@ void transCloseServer(void* arg) { tDebug("send quit msg to accept thread"); (void)uv_async_send(srv->pAcceptAsync); (void)taosThreadJoin(srv->thread, NULL); + SRV_RELEASE_UV(srv->loop); + (void)uv_loop_close(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); } } else { + SRV_RELEASE_UV(srv->loop); (void)uv_loop_close(srv->loop); } @@ -1708,7 +1663,9 @@ void transCloseServer(void* arg) { taosMemoryFree(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - taosMemoryFree(srv->pipe[i]); + if (srv->pipe[i] != NULL) { + taosMemoryFree(srv->pipe[i]); + } } taosMemoryFree(srv->pipe); From b6b0e266eb3033821a1655cadaafa26581124229 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 14:28:34 +0800 Subject: [PATCH 03/20] fix indirect leak --- source/libs/transport/src/transSvr.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 0727e49d8b..7e06767cb5 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1360,11 +1360,11 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - // if (false == taosValidIpAndPort(srv->ip, srv->port)) { - // code = TAOS_SYSTEM_ERROR(errno); - // tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - // goto End; - // } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + code = TAOS_SYSTEM_ERROR(errno); + tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + goto End; + } char pipeName[PATH_MAX]; #if defined(WINDOWS) || defined(DARWIN) From 0973fb6eed8a96bc0ec30dfa6e5b59fcfc35b6b3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 16:14:19 +0800 Subject: [PATCH 04/20] fix indirect leak --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5052d79ef3..85c8f7c21a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2387,7 +2387,7 @@ _end: } taosArrayDestroy(pThrd->timerList); - destroyConnPool(pThrd); + (void)destroyConnPool(pThrd); transDQDestroy(pThrd->delayQueue, NULL); transDQDestroy(pThrd->timeoutQueue, NULL); transDQDestroy(pThrd->waitConnQueue, NULL); @@ -2419,7 +2419,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } - uv_loop_close(pThrd->loop); + (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7e06767cb5..38c7edeeaf 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1615,7 +1615,7 @@ void destroyWorkThrdObj(SWorkThrd* pThrd) { } transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); - uv_loop_close(pThrd->loop); + (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } From 3b283b63507a64fddd30fa7a219e15e559f733ee Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 16:36:19 +0800 Subject: [PATCH 05/20] fix indirect leak --- source/libs/transport/src/transCli.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 85c8f7c21a..c2ce00c1cf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2383,6 +2383,7 @@ _end: transAsyncPoolDestroy(pThrd->asyncPool); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + (void)uv_timer_stop(timer); taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); @@ -2417,8 +2418,10 @@ static void destroyThrdObj(SCliThrd* pThrd) { tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + (void)uv_timer_stop(timer); taosMemoryFree(timer); } + (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); From 46a79c0be333809745ea9e69580bd5d71af0b934 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 17:50:35 +0800 Subject: [PATCH 06/20] fix indirect leak --- source/libs/transport/src/transCli.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c2ce00c1cf..803e50ce94 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -619,7 +619,11 @@ void* createConnPool(int size) { return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); } void* destroyConnPool(SCliThrd* pThrd) { - void* pool = pThrd->pool; + void* pool = pThrd->pool; + if (pool == NULL) { + return NULL; + } + SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conns)) { From 2fa083c82feb626f4ec261d50f73b8ed0f17192a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 10 Sep 2024 19:03:27 +0800 Subject: [PATCH 07/20] opt transport --- source/libs/transport/src/transCli.c | 1 - source/libs/transport/src/transSvr.c | 16 +++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 803e50ce94..a0c7315fe7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2426,7 +2426,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(timer); } - (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 38c7edeeaf..3a597fe6ad 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -188,11 +188,13 @@ static void sendQuitToWorkThrd(SWorkThrd* pThrd); static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); -#define SRV_RELEASE_UV(loop) \ - do { \ - (void)uv_walk(loop, uvWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ +#define SRV_RELEASE_UV(loop) \ + do { \ + if (loop && uv_loop_alive(loop)) { \ + (void)uv_walk(loop, uvWalkCb, NULL); \ + (void)uv_run(loop, UV_RUN_DEFAULT); \ + (void)uv_loop_close(loop); \ + } \ } while (0); #define ASYNC_ERR_JRET(thrd) \ @@ -1615,7 +1617,6 @@ void destroyWorkThrdObj(SWorkThrd* pThrd) { } transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); - (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } @@ -1648,14 +1649,11 @@ void transCloseServer(void* arg) { (void)taosThreadJoin(srv->thread, NULL); SRV_RELEASE_UV(srv->loop); - (void)uv_loop_close(srv->loop); - for (int i = 0; i < srv->numOfThreads; i++) { destroyWorkThrd(srv->pThreadObj[i]); } } else { SRV_RELEASE_UV(srv->loop); - (void)uv_loop_close(srv->loop); } taosMemoryFree(srv->pThreadObj); From fede52d5b7743ec553d10eaac72af5d694f6568d Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 10 Sep 2024 17:48:47 +0800 Subject: [PATCH 08/20] enh: (last) rm rwritebatch --- source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/tsdb/tsdbCache.c | 31 +++++++++++-------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 59d70889e5..c85c376c47 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -342,7 +342,6 @@ typedef struct { rocksdb_writeoptions_t *writeoptions; rocksdb_readoptions_t *readoptions; rocksdb_writebatch_t *writebatch; - rocksdb_writebatch_t *rwritebatch; STSchema *pTSchema; } SRocksCache; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3a3e20612e..f4a423bfa7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -214,10 +214,8 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { } rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); - rocksdb_writebatch_t *rwritebatch = rocksdb_writebatch_create(); pTsdb->rCache.writebatch = writebatch; - pTsdb->rCache.rwritebatch = rwritebatch; pTsdb->rCache.my_comparator = cmp; pTsdb->rCache.options = options; pTsdb->rCache.writeoptions = writeoptions; @@ -248,7 +246,6 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { rocksdb_close(pTsdb->rCache.db); rocksdb_flushoptions_destroy(pTsdb->rCache.flushoptions); rocksdb_writebatch_destroy(pTsdb->rCache.writebatch); - rocksdb_writebatch_destroy(pTsdb->rCache.rwritebatch); rocksdb_readoptions_destroy(pTsdb->rCache.readoptions); rocksdb_writeoptions_destroy(pTsdb->rCache.writeoptions); rocksdb_options_destroy(pTsdb->rCache.options); @@ -258,8 +255,8 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { taosMemoryFree(pTsdb->rCache.pTSchema); } -static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) { - rocksdb_writebatch_t *wb = read ? pTsdb->rCache.rwritebatch : pTsdb->rCache.writebatch; +static void rocksMayWrite(STsdb *pTsdb, bool force) { + rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; int count = rocksdb_writebatch_count(wb); if ((force && count > 0) || count >= ROCKS_BATCH_SIZE) { @@ -516,8 +513,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); - rocksMayWrite(pTsdb, true, false); - rocksMayWrite(pTsdb, true, true); + rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -667,8 +663,7 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); - rocksMayWrite(pTsdb, true, false); - rocksMayWrite(pTsdb, true, true); + rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); if (NULL != err) { @@ -862,7 +857,7 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra taosMemoryFree(pTSchema); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -903,7 +898,7 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { taosMemoryFree(pTSchema); - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -934,7 +929,7 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -973,7 +968,7 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1243,7 +1238,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosMemoryFreeClear(pToFree); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1582,7 +1577,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } // store result back to rocks cache - wb = pTsdb->rCache.rwritebatch; + wb = pTsdb->rCache.writebatch; char *value = NULL; size_t vlen = 0; code = tsdbCacheSerialize(pLastCol, &value, &vlen); @@ -1597,7 +1592,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } if (wb) { - rocksMayWrite(pTsdb, false, true); + rocksMayWrite(pTsdb, false); } _exit: @@ -1797,6 +1792,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache if (remainCols && TARRAY_SIZE(remainCols) > 0) { (void)taosThreadMutexLock(&pTsdb->lruMutex); + rocksMayWrite(pTsdb, false); + for (int i = 0; i < TARRAY_SIZE(remainCols);) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; LRUHandle *h = taosLRUCacheLookup(pCache, &idxKey->key, ROCKS_KEY_LEN); @@ -1944,7 +1941,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE taosMemoryFreeClear(pLastCol); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); _exit: (void)taosThreadMutexUnlock(&pTsdb->lruMutex); From cffa2a496ced1d33c425f36ea5a7577b101cbca9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 09:37:09 +0800 Subject: [PATCH 09/20] fix mem leak --- source/libs/transport/src/transSvr.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3a597fe6ad..48042b84c1 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -188,13 +188,11 @@ static void sendQuitToWorkThrd(SWorkThrd* pThrd); static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); -#define SRV_RELEASE_UV(loop) \ - do { \ - if (loop && uv_loop_alive(loop)) { \ - (void)uv_walk(loop, uvWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ - } \ +#define SRV_RELEASE_UV(loop) \ + do { \ + (void)uv_walk(loop, uvWalkCb, NULL); \ + (void)uv_run(loop, UV_RUN_DEFAULT); \ + (void)uv_loop_close(loop); \ } while (0); #define ASYNC_ERR_JRET(thrd) \ From abbcc4afa087dec2984e5c5b90987ddc81d4b472 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 11 Sep 2024 03:24:59 +0000 Subject: [PATCH 10/20] fix/TS-5404-use-temp-transid-when-timeout --- source/dnode/mnode/impl/src/mndTrans.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 63cab4168a..18ceddb7d0 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -64,6 +64,8 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); +int32_t tsMaxTransId = 0; + int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_TRANS, @@ -602,7 +604,9 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, tstrncpy(pTrans->opername, opername, TSDB_TRANS_OPER_LEN); } - pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); + int32_t sdbMaxId = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); + int32_t oldId = atomic_load_32(&tsMaxTransId); + pTrans->id = TMAX(sdbMaxId, oldId); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->conflict = conflict; @@ -1027,6 +1031,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); + atomic_store_32(&tsMaxTransId, pTrans->id); TAOS_RETURN(code); } mInfo("trans:%d, prepare finished", pTrans->id); From 44a955218cfcedb9b9445e058fcb3a87e4177c95 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 13:48:23 +0800 Subject: [PATCH 11/20] fix: (last) call rocksdb_write before rocksdb_multi_get --- source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/tsdb/tsdbCache.c | 111 +++++++++--------------- 2 files changed, 40 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c85c376c47..d4a33a5478 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -362,7 +362,6 @@ struct STsdb { SMemTable *imem; STsdbFS fs; // old SLRUCache *lruCache; - SCacheFlushState flushState; TdThreadMutex lruMutex; SLRUCache *biCache; TdThreadMutex biMutex; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f4a423bfa7..53e20d459f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -267,7 +267,6 @@ static void rocksMayWrite(STsdb *pTsdb, bool force) { tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, count, err); rocksdb_free(err); - // pTsdb->flushState.flush_count = 0; } rocksdb_writebatch_clear(wb); @@ -456,47 +455,23 @@ static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size TAOS_RETURN(TSDB_CODE_SUCCESS); } -static void tsdbCachePutBatch(SLastCol *pLastCol, const void *key, size_t klen, SCacheFlushState *state) { - int32_t code = 0; - STsdb *pTsdb = state->pTsdb; - SRocksCache *rCache = &pTsdb->rCache; - rocksdb_writebatch_t *wb = rCache->writebatch; - char *rocks_value = NULL; - size_t vlen = 0; - - code = tsdbCacheSerialize(pLastCol, &rocks_value, &vlen); - if (code) { - tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); - return; - } - - rocksdb_writebatch_put(wb, (char *)key, klen, rocks_value, vlen); - - taosMemoryFree(rocks_value); - - if (++state->flush_count >= ROCKS_BATCH_SIZE) { - char *err = NULL; - - rocksdb_write(rCache->db, rCache->writeoptions, wb, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - state->flush_count, err); - rocksdb_free(err); - } - - rocksdb_writebatch_clear(wb); - - state->flush_count = 0; - } -} +static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol); int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + STsdb *pTsdb = (STsdb *)ud; + + int32_t code = tsdbCachePutToRocksdb(pTsdb, (SLastKey *)key, pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, flush dirty lru failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; + } pLastCol->dirty = 0; + + rocksMayWrite(pTsdb, false); } return 0; @@ -511,7 +486,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { (void)taosThreadMutexLock(&pTsdb->lruMutex); - taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); + taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); @@ -602,7 +577,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + tsdbCacheFlushDirty(key, klen, pLastCol, ud); } for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { @@ -639,11 +614,11 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i SLastKey *pLastKey = &(SLastKey){.lflag = lflag, .uid = uid, .cid = cid}; LRUStatus status = taosLRUCacheInsert(pCache, pLastKey, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + TAOS_LRU_PRIORITY_LOW, pTsdb); if (status != TAOS_LRU_STATUS_OK) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); code = TSDB_CODE_FAILED; + pLastCol = NULL; } _exit: @@ -661,7 +636,7 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->lruCache; rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; - taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); + taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); @@ -733,6 +708,10 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, char **values_list = NULL; size_t *values_list_sizes = NULL; + + // was written by caller + // rocksMayWrite(pTsdb, true); // flush writebatch cache + TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, 2, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), NULL, _exit); @@ -857,7 +836,7 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra taosMemoryFree(pTSchema); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -898,7 +877,7 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { taosMemoryFree(pTSchema); - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -929,7 +908,7 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -968,7 +947,7 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1078,11 +1057,11 @@ static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLa TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(pLRULastCol, &charge)); LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, pLastKey, ROCKS_KEY_LEN, pLRULastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLRULastCol); code = TSDB_CODE_FAILED; + pLRULastCol = NULL; } _exit: @@ -1172,6 +1151,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray keys_list_sizes[i] = ROCKS_KEY_LEN; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + code = tsdbCacheGetValuesFromRocks(pTsdb, num_keys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes); if (code) { @@ -1238,7 +1219,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosMemoryFreeClear(pToFree); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1568,32 +1549,22 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); - taosMemoryFree(pLastCol); + pLastCol = NULL; TAOS_CHECK_EXIT(TSDB_CODE_FAILED); } // store result back to rocks cache - wb = pTsdb->rCache.writebatch; - char *value = NULL; - size_t vlen = 0; - code = tsdbCacheSerialize(pLastCol, &value, &vlen); + code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, pLastCol); if (code) { - tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); - } else { - SLastKey *key = &idxKey->key; - size_t klen = ROCKS_KEY_LEN; - rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); - taosMemoryFree(value); + tsdbError("vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + TAOS_CHECK_EXIT(code); } } - if (wb) { - rocksMayWrite(pTsdb, false); - } + rocksMayWrite(pTsdb, false); _exit: taosArrayDestroy(lastrowTmpIndexArray); @@ -1633,6 +1604,8 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA keys_list_sizes[i] = ROCKS_KEY_LEN; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + code = tsdbCacheGetValuesFromRocks(pTsdb, num_keys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes); if (code) { @@ -1681,11 +1654,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA } LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); - taosMemoryFreeClear(pLastCol); taosMemoryFreeClear(pToFree); TAOS_CHECK_EXIT(TSDB_CODE_FAILED); } @@ -1792,7 +1763,6 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache if (remainCols && TARRAY_SIZE(remainCols) > 0) { (void)taosThreadMutexLock(&pTsdb->lruMutex); - rocksMayWrite(pTsdb, false); for (int i = 0; i < TARRAY_SIZE(remainCols);) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; @@ -1907,6 +1877,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE keys_list_sizes[i] = klen; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, numKeys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), NULL, _exit); @@ -1941,7 +1913,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE taosMemoryFreeClear(pLastCol); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); _exit: (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1983,9 +1955,6 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { (void)taosThreadMutexInit(&pTsdb->lruMutex, NULL); - pTsdb->flushState.pTsdb = pTsdb; - pTsdb->flushState.flush_count = 0; - _err: if (code) { tsdbError("tsdb/cache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); From 11b3faee8e5d3fe20d3d7fd488ed9ca056da89f2 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 11 Sep 2024 08:56:39 +0800 Subject: [PATCH 12/20] remove (void) from parser/planner/nodes/sma --- include/common/tname.h | 2 +- source/client/src/clientMain.c | 2 +- source/client/src/clientRawBlockWrite.c | 16 +-- source/client/src/clientSml.c | 7 +- source/common/src/tname.c | 3 +- source/dnode/mnode/impl/src/mndSma.c | 12 +- source/libs/nodes/src/nodesUtilFuncs.c | 15 ++- source/libs/parser/src/parAuthenticator.c | 5 +- source/libs/parser/src/parTranslater.c | 132 ++++++++++++---------- 9 files changed, 110 insertions(+), 84 deletions(-) diff --git a/include/common/tname.h b/include/common/tname.h index f0a68f28c4..c7c34310b9 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -37,7 +37,7 @@ typedef struct SName { char tname[TSDB_TABLE_NAME_LEN]; } SName; -SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName); +void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName); int32_t tNameExtractFullName(const SName* name, char* dst); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1c1fff9b7b..3d72c14fdd 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1526,7 +1526,7 @@ int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); SName tableName; - (void)toName(pTscObj->acctId, db, table, &tableName); + toName(pTscObj->acctId, db, table, &tableName); SVgroupInfo vgInfo; code = catalogGetTableHashVgroup(pCtg, &conn, &tableName, &vgInfo); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index d5785cce6b..0f8f0ec4e7 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -946,7 +946,8 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; - RAW_RETURN_CHECK(tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name)); + toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName); + RAW_RETURN_CHECK(tNameExtractFullName(&tableName, pReq.name)); SCmdMsgInfo pCmdMsg = {0}; pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; @@ -1021,7 +1022,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; SName pName = {0}; - (void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, + toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); // ignore the return value, always return pName STableMeta* pTableMeta = NULL; code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); @@ -1045,7 +1046,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; - if (tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name) != 0) { + toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName); + if (tNameExtractFullName(&tableName, pReq.name) != 0) { code = TSDB_CODE_INVALID_PARA; goto end; } @@ -1150,7 +1152,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -1163,7 +1165,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SName sName = {0}; tb_uid_t oldSuid = pCreateReq->ctb.suid; // pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); - (void)toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { code = TSDB_CODE_SUCCESS; @@ -1307,7 +1309,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); + toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo)); STableMeta* pTableMeta = NULL; @@ -1451,7 +1453,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { SVgroupInfo pInfo = {0}; SName pName = {0}; - (void)toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); + toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo)); pArray = taosArrayInit(1, sizeof(void*)); RAW_NULL_CHECK(pArray); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 05678e1cbf..ad2655bc94 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -116,7 +116,7 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char return TSDB_CODE_SML_INVALID_DATA; } } else { - (void)toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore + toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); //ignore } pAuth.type = type; @@ -1113,7 +1113,10 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.commentLen = -1; pReq.igExists = true; - (void)tNameExtractFullName(pName, pReq.name); + code = tNameExtractFullName(pName, pReq.name); + if (TSDB_CODE_SUCCESS != code) { + goto end; + } pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 8d0f324509..4bb89464d0 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -87,12 +87,11 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in #endif -SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { +void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; snprintf(pName->dbname, sizeof(pName->dbname), "%s", pDbName); snprintf(pName->tname, sizeof(pName->tname), "%s", pTableName); - return pName; } int32_t tNameExtractFullName(const SName* name, char* dst) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 284d65cd9c..f91f7d49cc 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -368,7 +368,11 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, SEncoder encoder = {0}; int32_t contLen; SName name = {0}; - (void)tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + return NULL; + } SVDropTSmaReq req = {0}; req.indexUid = pSma->uid; @@ -1669,8 +1673,7 @@ static int32_t mndSetUpdateDbTsmaVersionPrepareLogs(SMnode *pMnode, STrans *pTra TAOS_RETURN(code); } - (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - TAOS_RETURN(code); + TAOS_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY)); } static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { @@ -1686,8 +1689,7 @@ static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTran TAOS_RETURN(code); } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - TAOS_RETURN(code); + TAOS_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); } static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1d6c19346a..8e2b739934 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -235,7 +235,10 @@ void nodesDestroyAllocatorSet() { int64_t refId = 0; while (NULL != pAllocator) { refId = pAllocator->self; - (void)taosRemoveRef(g_allocatorReqRefPool, refId); + int32_t code = taosRemoveRef(g_allocatorReqRefPool, refId); + if (TSDB_CODE_SUCCESS != code) { + nodesError("failed to remove ref at: %s:%d, rsetId:%d, refId:%"PRId64, __func__, __LINE__, g_allocatorReqRefPool, refId); + } pAllocator = taosIterateRef(g_allocatorReqRefPool, refId); } taosCloseRef(g_allocatorReqRefPool); @@ -328,7 +331,10 @@ void nodesDestroyAllocator(int64_t allocatorId) { return; } - (void)taosRemoveRef(g_allocatorReqRefPool, allocatorId); + int32_t code = taosRemoveRef(g_allocatorReqRefPool, allocatorId); + if (TSDB_CODE_SUCCESS != code) { + nodesError("failed to remove ref at: %s:%d, rsetId:%d, refId:%"PRId64, __func__, __LINE__, g_allocatorReqRefPool, allocatorId); + } } static int32_t makeNode(ENodeType type, int32_t size, SNode** ppNode) { @@ -1090,7 +1096,10 @@ void nodesDestroyNode(SNode* pNode) { pStmt->destroyParseFileCxt(&pStmt->pParFileCxt); } - (void)taosCloseFile(&pStmt->fp); + int32_t code = taosCloseFile(&pStmt->fp); + if (TSDB_CODE_SUCCESS != code) { + nodesError("failed to close file: %s:%d", __func__, __LINE__); + } break; } case QUERY_NODE_CREATE_DATABASE_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 60cc1024bf..7fbbd90d40 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -46,7 +46,7 @@ static int32_t setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const c int32_t code = tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName)); if (TSDB_CODE_SUCCESS != code) return code; } else { - (void)toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); + toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); } pAuth->type = type; pAuth->isView = isView; @@ -171,8 +171,9 @@ static EDealRes authSelectImpl(SNode* pNode, void* pContext) { #ifdef TD_ENTERPRISE SName name; STableMeta* pTableMeta = NULL; + toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name); int32_t code = getTargetMetaImpl( - pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name), &pTableMeta, true); + pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS == code && TSDB_VIEW_TABLE == pTableMeta->tableType) { isView = true; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a0b8c13e58..ebe6c82193 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -526,7 +526,8 @@ static int32_t getTargetMeta(STranslateContext* pCxt, const SName* pName, STable static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { SName name; - return getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta, false); + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + return getTargetMeta(pCxt, &name, pMeta, false); } static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCfg** pCfg) { @@ -557,7 +558,7 @@ static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, STableMeta** pMeta) { SParseContext* pParCxt = pCxt->pParseCxt; SName name; - (void)toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); int32_t code = TSDB_CODE_SUCCESS; if (pParCxt->async) { code = getTableMetaFromCache(pCxt->pMetaCache, &name, pMeta); @@ -635,7 +636,8 @@ static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pNam static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SVgroupInfo* pInfo) { SName name; - return getTableHashVgroupImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pInfo); + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + return getTableHashVgroupImpl(pCxt, &name, pInfo); } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, @@ -4016,7 +4018,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo SName tsmaTargetTbName = {0}; SVgroupInfo vgInfo = {0}; bool exists = false; - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); @@ -4678,9 +4680,8 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare // The SRealTableNode created through ROLLUP already has STableMeta. if (NULL == pRealTable->pMeta) { SName name; - code = getTargetMeta( - pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &(pRealTable->pMeta), true); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); + code = getTargetMeta( pCxt, &name, &(pRealTable->pMeta), true); if (TSDB_CODE_SUCCESS != code) { (void)generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); return code; @@ -6232,7 +6233,7 @@ static void findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTbnames for (int j = 0; j < nTbls; ++j) { SName snameTb; char* tbName = taosArrayGetP(aTbnames, j); - (void)toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); + toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); SVgroupInfo vgInfo = {0}; bool bExists; int32_t code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); @@ -6261,7 +6262,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa int32_t code = 0; SRealTableNode* pRealTable = pInfo->pRealTable; char* tbName = taosArrayGetP(pInfo->aTbnames, 0); - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, tbName, &snameTb); STableMeta* pMeta = NULL; TAOS_CHECK_RETURN(catalogGetCachedTableMeta(pCxt->pParseCxt->pCatalog, &snameTb, &pMeta)); @@ -6282,7 +6283,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) { STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i); SName tsmaTargetTbName = {0}; - (void)toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); int32_t len = snprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); @@ -8822,7 +8823,8 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm } if (TSDB_CODE_SUCCESS == code) { pReq->numOfFuncs = taosArrayGetSize(pReq->pFuncs); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + code = tNameExtractFullName(&tableName, pReq->name); } if (TSDB_CODE_SUCCESS == code) code = collectUseTable(&tableName, pCxt->pTables); @@ -8866,19 +8868,20 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt SDropTableClause* pClause = (SDropTableClause*)nodesListGetNode(pStmt->pTables, 0); SName tableName; if (pStmt->withTsma) return TSDB_CODE_SUCCESS; - return doTranslateDropSuperTable( - pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), pClause->ignoreNotExists); + toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName); + return doTranslateDropSuperTable( pCxt, &tableName, pClause->ignoreNotExists); } static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { SName tableName; - return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), - pStmt->ignoreNotExists); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); } static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { SName tableName; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pAlterReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName); + int32_t code = tNameExtractFullName(&tableName, pAlterReq->name); if (TSDB_CODE_SUCCESS != code) return code; pAlterReq->alterType = pStmt->alterType; @@ -9394,10 +9397,12 @@ static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, pReq->name); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stb); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = tNameExtractFullName(&name, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { pReq->igExists = pStmt->ignoreExists; @@ -9545,10 +9550,12 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SCreateTagIndexReq* pReq) { SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->idxName); + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, pReq->idxName); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = tNameExtractFullName(&name, pReq->stbName); } if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); @@ -9584,8 +9591,8 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS int32_t code = 0; SName name; STableMeta* pMeta = NULL; - - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + code = getTargetMeta(pCxt, &name, &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -9627,7 +9634,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { SMDropSmaReq dropSmaReq = {0}; SName name; - int32_t code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), dropSmaReq.name); + toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name); + int32_t code = tNameExtractFullName(&name, dropSmaReq.name); if (TSDB_CODE_SUCCESS != code) return code; dropSmaReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq); @@ -9702,11 +9710,11 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS SName name; if ('\0' != pStmt->subSTbName[0]) { pReq->subType = TOPIC_SUB_TYPE__TABLE; - (void)toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); (void)tNameGetFullDbName(&name, pReq->subDbName); if (TSDB_CODE_SUCCESS == code) { - (void)tNameExtractFullName(&name, pReq->subStbName); - if (pStmt->pQuery != NULL) { + code = tNameExtractFullName(&name, pReq->subStbName); + if (TSDB_CODE_SUCCESS == code && pStmt->pQuery != NULL) { code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } } @@ -9829,8 +9837,8 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt .mgmtEps = pParCxt->mgmtEpSet}; SName name; STableMeta* pMeta = NULL; - int32_t code = - getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false); + toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); + int32_t code = getTargetMeta(pCxt, &name, &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -9952,7 +9960,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { int32_t origCode = code; SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); SViewMeta* pMeta = NULL; code = getViewMetaFromMetaCache(pCxt, &name, &pMeta); if (TSDB_CODE_SUCCESS != code) { @@ -10072,9 +10080,8 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt SName name; STableMeta* pMeta = NULL; int8_t tableType = 0; - int32_t code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &pMeta, true); + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name); + int32_t code = getTargetMeta(pCxt, &name, &pMeta, true); if (NULL != pMeta) { tableType = pMeta->tableType; taosMemoryFree(pMeta); @@ -11103,10 +11110,11 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* if ('\0' != pStmt->targetTabName[0]) { strcpy(name.dbname, pStmt->targetDbName); strcpy(name.tname, pStmt->targetTabName); - (void)tNameExtractFullName(&name, pReq->targetStbFullName); + code = tNameExtractFullName(&name, pReq->targetStbFullName); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildCreateStreamQuery(pCxt, pStmt, pReq); } - - code = buildCreateStreamQuery(pCxt, pStmt, pReq); if (TSDB_CODE_SUCCESS == code) { pReq->sql = taosStrdup(pCxt->pParseCxt->pSql); if (NULL == pReq->sql) { @@ -11312,7 +11320,7 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; SName name; char dbFName[TSDB_DB_FNAME_LEN]; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); (void)tNameGetFullDbName(&name, dbFName); int32_t code = validateCreateView(pCxt, pStmt); @@ -11363,7 +11371,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) return TSDB_CODE_OUT_OF_MEMORY; } dropReq.igNotExists = pStmt->ignoreNotExists; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -11399,7 +11407,8 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { int64_t s = taosReadFile(tfile, *buf, *len); if (s != *len) { - (void)taosCloseFile(&tfile); + int32_t code = taosCloseFile(&tfile); + qError("failed to close file: %s in %s:%d, err: %s", pName, __func__, __LINE__, tstrerror(code)); taosMemoryFreeClear(*buf); return TSDB_CODE_APP_ERROR; } @@ -11473,8 +11482,8 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, int32_t code = createRealTableForGrantTable(pStmt, &pTable); if (TSDB_CODE_SUCCESS == code) { SName name; - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), - &(pTable->pMeta), false); + toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name); + code = getTargetMeta(pCxt, &name, &(pTable->pMeta), false); if (code) { nodesDestroyNode((SNode*)pTable); return code; @@ -11516,8 +11525,8 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { if (0 != pStmt->tabName[0]) { SName name; STableMeta* pTableMeta = NULL; - code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -11552,8 +11561,8 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { if (0 != pStmt->tabName[0]) { SName name; STableMeta* pTableMeta = NULL; - code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -11667,7 +11676,7 @@ static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTabl int32_t code = getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pDbCfg); if (TSDB_CODE_SUCCESS == code) { SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = getTableCfg(pCxt, &name, (STableCfg**)&pStmt->pTableCfg); } return code; @@ -11678,7 +11687,7 @@ static int32_t translateShowCreateView(STranslateContext* pCxt, SShowCreateViewS return TSDB_CODE_OPS_NOT_SUPPORT; #else SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); return getViewMetaFromMetaCache(pCxt, &name, (SViewMeta**)&pStmt->pViewMeta); #endif } @@ -11949,10 +11958,11 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm SName name; SDbCfgInfo pDbInfo = {0}; int32_t code = TSDB_CODE_SUCCESS; - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), pReq->name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); + code = tNameExtractFullName(&name, pReq->name); if (TSDB_CODE_SUCCESS == code) { memset(&name, 0, sizeof(SName)); - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, useTbName); code = tNameExtractFullName(useTbName, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { @@ -12025,7 +12035,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm if (TSDB_CODE_SUCCESS == code) { memset(useTbName, 0, sizeof(SName)); memcpy(pStmt->originalTbName, pRecursiveTsma->tb, TSDB_TABLE_NAME_LEN); - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName), pReq->stb); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pRecursiveTsma->tb, useTbName); + code = tNameExtractFullName(useTbName, pReq->stb); } if (TSDB_CODE_SUCCESS == code) { numOfCols = pRecursiveTsma->pUsedCols->size; @@ -12146,7 +12157,7 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData if (TSDB_CODE_SUCCESS == code) { SName name = {0}; - (void)toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); + toName(pParseCxt->acctId, pStmt->dbName, pStmt->originalTbName, &name); code = collectUseTable(&name, cxt.pTargetTables); } @@ -12165,13 +12176,14 @@ static int32_t translateDropTSMA(STranslateContext* pCxt, SDropTSMAStmt* pStmt) SMDropSmaReq dropReq = {0}; SName name; STableTSMAInfo* pTsma = NULL; - code = tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name), dropReq.name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tsmaName, &name); + code = tNameExtractFullName(&name, dropReq.name); if (TSDB_CODE_SUCCESS == code) { dropReq.igNotExists = pStmt->ignoreNotExists; code = getTsma(pCxt, &name, &pTsma); } if (code == TSDB_CODE_SUCCESS) { - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pTsma->tb, &name); code = collectUseTable(&name, pCxt->pTargetTables); } if (TSDB_CODE_SUCCESS == code) @@ -13071,10 +13083,8 @@ static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) { int32_t code = 0; SName name; STableMeta* pTableMeta = NULL; - code = getTargetMeta(pCxt, - toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, - ((SValueNode*)pShow->pTbName)->literal, &name), - &pTableMeta, true); + toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, &name); + code = getTargetMeta(pCxt, &name, &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); goto _exit; @@ -13433,7 +13443,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = checkCreateTable(pCxt, pStmt, false); SVgroupInfo info = {0}; SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); if (TSDB_CODE_SUCCESS == code) { code = getTableHashVgroupImpl(pCxt, &name, &info); } @@ -13681,7 +13691,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla } if (TSDB_CODE_SUCCESS == code) { SName name; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = collectUseTable(&name, pCxt->pTargetTables); } @@ -14447,7 +14457,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { FOREACH(pNode, pStmt->pTables) { SDropTableClause* pClause = (SDropTableClause*)pNode; SName name; - (void)toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); + toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); int32_t code = buildDropTableVgroupHashmap(pCxt, pClause, &name, &tableType, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); @@ -14523,7 +14533,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTsmas = NULL; int32_t code = TSDB_CODE_SUCCESS; if (pCxt->pMetaCache) { - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (code != TSDB_CODE_SUCCESS) return code; if (pTsmas && pTsmas->size > 0) return TSDB_CODE_TSMA_MUST_BE_DROPPED; @@ -14707,7 +14717,7 @@ static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt SArray* pTsmas = NULL; SName tbName; int32_t code = 0; - (void)toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tbName); if (pCxt->pMetaCache) code = getTableTsmasFromCache(pCxt->pMetaCache, &tbName, &pTsmas); if (TSDB_CODE_SUCCESS != code) return code; if (pTsmas && pTsmas->size > 0) { From 6c610e477755b9f4eafdc6bf6a2318224e28067c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 14:45:03 +0800 Subject: [PATCH 13/20] fix mem leak --- source/libs/transport/src/transCli.c | 1 - source/libs/transport/src/transSvr.c | 14 +++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a0c7315fe7..ce9adc836e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2422,7 +2422,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); - (void)uv_timer_stop(timer); taosMemoryFree(timer); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 48042b84c1..1db5d0a6d7 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1398,10 +1398,21 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, thrd->quit = false; thrd->pTransInst = shandle; thrd->pWhiteList = uvWhiteListCreate(); + if (thrd->pWhiteList == NULL) { + destroyWorkThrdObj(thrd); + code = TSDB_CODE_OUT_OF_MEMORY; + goto End; + } - srv->pThreadObj[i] = thrd; srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); + if (srv->pipe[i] == NULL) { + destroyWorkThrdObj(thrd); + code = TSDB_CODE_OUT_OF_MEMORY; + goto End; + } + thrd->pipe = &(srv->pipe[i][1]); // init read + srv->pThreadObj[i] = thrd; if ((code = addHandleToWorkloop(thrd, pipeName)) != 0) { goto End; @@ -1415,6 +1426,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, tError("failed to create worker-thread:%d", i); goto End; } + thrd->inited = 1; } #else From 0b6a934586e7d07b51b04d6054be096efc10faf7 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 11 Sep 2024 09:15:20 +0000 Subject: [PATCH 14/20] fix/TS-5404-use-temp-transid-when-timeout-lock --- source/dnode/mnode/impl/src/mndTrans.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 18ceddb7d0..724c9637e6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -64,7 +64,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); -int32_t tsMaxTransId = 0; +static int32_t tsMaxTransId = 0; int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { @@ -605,8 +605,9 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, } int32_t sdbMaxId = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); - int32_t oldId = atomic_load_32(&tsMaxTransId); - pTrans->id = TMAX(sdbMaxId, oldId); + sdbReadLock(pMnode->pSdb, SDB_TRANS); + pTrans->id = TMAX(sdbMaxId, tsMaxTransId + 1); + sdbUnLock(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->conflict = conflict; @@ -1031,7 +1032,9 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); - atomic_store_32(&tsMaxTransId, pTrans->id); + sdbReadLock(pMnode->pSdb, SDB_TRANS); + tsMaxTransId = TMAX(pTrans->id, tsMaxTransId); + sdbUnLock(pMnode->pSdb, SDB_TRANS); TAOS_RETURN(code); } mInfo("trans:%d, prepare finished", pTrans->id); From 4d103c09d490f67fd5b6fb836d91449a8731433c Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 19:22:04 +0800 Subject: [PATCH 15/20] fix: ignore tsdbCacheFlushDirty errcode --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 53e20d459f..74bd837775 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -577,7 +577,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCacheFlushDirty(key, klen, pLastCol, ud); + (void)tsdbCacheFlushDirty(key, klen, pLastCol, ud); } for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { From f9d38ef7bae2dc05f3187b95d5e194d05387d512 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Thu, 12 Sep 2024 09:36:37 +0800 Subject: [PATCH 16/20] fix:[TD-32041] Fix rand function wrong behaviour. --- source/libs/scalar/src/sclfunc.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4e8107a5b2..3b9cad68ad 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2888,13 +2888,16 @@ int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut } int32_t randFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (!IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0]))) { - int32_t seed; - GET_TYPED_DATA(seed, int32_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData); - taosSeedRand(seed); - } + int32_t seed; int32_t numOfRows = inputNum == 1 ? pInput[0].numOfRows : TMAX(pInput[0].numOfRows, pInput[1].numOfRows); for (int32_t i = 0; i < numOfRows; ++i) { + // for constant seed, only set seed once + if ((pInput[0].numOfRows == 1 && i == 0) || (pInput[0].numOfRows != 1)) { + if (!IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) && !colDataIsNull_s(pInput[0].columnData, i)) { + GET_TYPED_DATA(seed, int32_t, GET_PARAM_TYPE(&pInput[0]), colDataGetData(pInput[0].columnData, i)); + taosSeedRand(seed); + } + } double random_value = (double)(taosRand() % RAND_MAX) / RAND_MAX; colDataSetDouble(pOutput->columnData, i, &random_value); } From 28b3c92dddd0f9d3f91ef3b77e8d051e5abb2494 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 02:36:07 +0000 Subject: [PATCH 17/20] fix/TS-5404-use-write-lock --- source/dnode/mnode/impl/src/mndTrans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 724c9637e6..43f8ffe61b 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1032,7 +1032,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); - sdbReadLock(pMnode->pSdb, SDB_TRANS); + sdbWriteLock(pMnode->pSdb, SDB_TRANS); tsMaxTransId = TMAX(pTrans->id, tsMaxTransId); sdbUnLock(pMnode->pSdb, SDB_TRANS); TAOS_RETURN(code); From 41dfc643b768f9ca1e8538a3b5cd6512f8737dda Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Thu, 12 Sep 2024 13:45:17 +0800 Subject: [PATCH 18/20] enh:modify error code pass --- source/client/src/clientHb.c | 16 +++--- source/client/src/clientImpl.c | 29 +++++------ source/client/src/clientMain.c | 2 +- source/client/src/clientMsgHandler.c | 30 +++++------ source/client/src/clientSml.c | 74 ++++++++++++++-------------- source/client/src/clientSmlJson.c | 8 +-- source/client/src/clientSmlLine.c | 16 +++--- source/client/src/clientSmlTelnet.c | 2 +- source/client/src/clientStmt.c | 33 ++++++------- source/client/src/clientStmt2.c | 25 +++++----- source/client/src/clientTmq.c | 22 ++++----- 11 files changed, 126 insertions(+), 131 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 47b80c03d1..8297ac362e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -180,7 +180,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == vgInfo->vgHash) { tscError("hash init[%d] failed", rsp->vgNum); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } @@ -188,7 +188,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { tscError("hash push failed, errno:%d", errno); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -689,7 +689,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { (void)releaseRequest(*rid); if (NULL == taosArrayPush(hbBasic->queryDesc, &desc)) { taosArrayDestroy(desc.subDesc); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pIter = taosHashIterate(pObj->pRequests, pIter); @@ -777,7 +777,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient pKv->value = qUserAuth; pKv->valueLen += sizeof(SUserAuthVersion); } else { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } goto _return; } @@ -785,7 +785,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient // key/user not exist, add user SUserAuthVersion *user = taosMemoryMalloc(sizeof(SUserAuthVersion)); if (!user) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } tstrncpy(user->user, pTscObj->user, TSDB_USER_LEN); @@ -1436,7 +1436,7 @@ int32_t appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key, SAppHbMgr **pAppHbMg TSC_ERR_RET(hbMgrInit()); *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (*pAppHbMgr == NULL) { - TSC_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + TSC_ERR_JRET(terrno); } // init stat (*pAppHbMgr)->startTime = taosGetTimestampMs(); @@ -1456,14 +1456,14 @@ int32_t appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key, SAppHbMgr **pAppHbMg (*pAppHbMgr)->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); if ((*pAppHbMgr)->activeInfo == NULL) { - TSC_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + TSC_ERR_JRET(terrno); } // taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq); TSC_ERR_JRET(taosThreadMutexLock(&clientHbMgr.lock)); if (taosArrayPush(clientHbMgr.appHbMgrs, &(*pAppHbMgr)) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; (void)taosThreadMutexUnlock(&clientHbMgr.lock); goto _return; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d77b8dcbb7..484d9bb7e5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -228,7 +228,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen); @@ -247,7 +247,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*pRequest)->allocatorRefId = -1; @@ -258,7 +258,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -595,7 +595,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr if (NULL == taosArrayPush(nodeList, &load)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } @@ -619,7 +619,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } if (NULL == taosArrayAddBatch(nodeList, pData, mnodeNum)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tscDebug("0x%" PRIx64 " %s policy, use mnode list, num:%d", pRequest->requestId, policy, mnodeNum); @@ -646,7 +646,7 @@ int32_t buildQnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } if (NULL == taosArrayAddBatch(nodeList, pData, qNodeNum)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tscDebug("0x%" PRIx64 " qnode policy, use qnode list, num:%d", pRequest->requestId, qNodeNum); goto _return; @@ -665,7 +665,7 @@ int32_t buildQnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } if (NULL == taosArrayAddBatch(nodeList, pData, mnodeNum)) { taosArrayDestroy(nodeList); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tscDebug("0x%" PRIx64 " qnode policy, use mnode list, num:%d", pRequest->requestId, mnodeNum); @@ -699,7 +699,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } if (NULL == taosArrayPush(pDbVgList, &pRes->pRes)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -791,7 +791,7 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* } if (NULL == taosArrayPush(pDbVgList, &pVgList)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -903,8 +903,7 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, pArray = taosArrayInit(tbNum, sizeof(STbSVersion)); if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t i = 0; i < tbNum; ++i) { @@ -915,7 +914,7 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, } STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion, .tver = tbInfo->tversion}; if (NULL == taosArrayPush(pArray, &tbSver)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } } @@ -1662,7 +1661,7 @@ static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInf void* pReq = taosMemoryMalloc(contLen); if (NULL == pReq) { taosMemoryFree(*pMsgSendInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (-1 == tSerializeSConnectReq(pReq, contLen, &connectReq)) { @@ -2050,7 +2049,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { taosReleaseConv(idx, conv, C2M); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pResultInfo->convertBuf[i] = p; @@ -2441,7 +2440,6 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR if (pResultInfo->decompBuf == NULL) { pResultInfo->decompBuf = taosMemoryMalloc(payloadLen); if (pResultInfo->decompBuf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to prepare the decompress buffer, size:%d", payloadLen); return terrno; } @@ -2450,7 +2448,6 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR if (pResultInfo->decompBufSize < payloadLen) { char* p = taosMemoryRealloc(pResultInfo->decompBuf, payloadLen); if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to prepare the decompress buffer, size:%d", payloadLen); return terrno; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1c1fff9b7b..4cd8afe880 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -216,7 +216,7 @@ int32_t fetchWhiteListCallbackFn(void *param, SDataBuf *pMsg, int32_t code) { taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pInfo); tFreeSGetUserWhiteListRsp(&wlRsp); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int i = 0; i < wlRsp.numWhiteLists; ++i) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 54020b77ef..f8d443aaad 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -511,19 +511,19 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(SHOW_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); - TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, terrno); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD1_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD2_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD3_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); int32_t numOfCfg = taosArrayGetSize(pVars); code = blockDataEnsureCapacity(pBlock, numOfCfg); @@ -531,26 +531,26 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { SVariablesInfo* pInfo = taosArrayGet(pVars, i); - TSDB_CHECK_NULL(pInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pInfo, code, line, END, terrno); char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - TSDB_CHECK_NULL(pColInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); code = colDataSetVal(pColInfo, i, name, false); TSDB_CHECK_CODE(code, line, END); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - TSDB_CHECK_NULL(pColInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); code = colDataSetVal(pColInfo, i, value, false); TSDB_CHECK_CODE(code, line, END); char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(scope, pInfo->scope, TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - TSDB_CHECK_NULL(pColInfo, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); code = colDataSetVal(pColInfo, i, scope, false); TSDB_CHECK_CODE(code, line, END); } @@ -662,29 +662,29 @@ static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) { pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData)); - TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pBlock->pDataBlock, code, line, END, terrno); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = COMPACT_DB_RESULT_FIELD1_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = COMPACT_DB_RESULT_FIELD3_LEN; - TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); code = blockDataEnsureCapacity(pBlock, 1); TSDB_CHECK_CODE(code, line, END); SColumnInfoData* pResultCol = taosArrayGet(pBlock->pDataBlock, 0); - TSDB_CHECK_NULL(pResultCol, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pResultCol, code, line, END, terrno); SColumnInfoData* pIdCol = taosArrayGet(pBlock->pDataBlock, 1); - TSDB_CHECK_NULL(pIdCol, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pIdCol, code, line, END, terrno); SColumnInfoData* pReasonCol = taosArrayGet(pBlock->pDataBlock, 2); - TSDB_CHECK_NULL(pReasonCol, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pReasonCol, code, line, END, terrno); char result[COMPACT_DB_RESULT_FIELD1_LEN] = {0}; char reason[COMPACT_DB_RESULT_FIELD3_LEN] = {0}; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 05678e1cbf..fa6a00b197 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -178,7 +178,7 @@ int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, if (tag->cols == NULL) { uError("SML:smlBuildTableInfo failed to allocate memory"); taosMemoryFree(tag); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *tInfo = tag; @@ -210,7 +210,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); if (measure == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); @@ -251,11 +251,11 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (i < pTableMeta->tableInfo.numOfColumns) { if(taosArrayPush((*sMeta)->cols, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else { if(taosArrayPush((*sMeta)->tags, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } @@ -328,7 +328,7 @@ END: int32_t smlJoinMeasureTag(SSmlLineInfo *elements) { elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); if (elements->measureTag == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(elements->measureTag, elements->measure, elements->measureLen); (void)memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); @@ -455,14 +455,14 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); if (elements->colArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } if (taosArrayPush(elements->colArray, kvTs) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(elements->colArray, kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } info->preLine = *elements; @@ -937,7 +937,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (hashTmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } int32_t i = 0; @@ -997,7 +997,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO field.bytes = getBytes(kv->type, kv->length); (void)memcpy(field.name, kv->key, kv->keyLen); if (taosArrayPush(results, &field) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); @@ -1046,7 +1046,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); if (pReq.pColumns == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (int32_t i = 0; i < pReq.numOfColumns; ++i) { @@ -1059,7 +1059,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1106,7 +1106,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); if (taosArrayPush(pReq.pTags, &field) == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1124,7 +1124,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -1185,7 +1185,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); if (measure == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } (void)memcpy(measure, superTable, superTableLen); @@ -1205,12 +1205,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); if (pColumns == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); if (pTags == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosArrayDestroy(pColumns); goto end; } @@ -1252,7 +1252,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (hashTmp == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (uint16_t i = pTableMeta->tableInfo.numOfColumns; @@ -1278,14 +1278,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); if (pColumns == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); if (pTags == NULL){ taosArrayDestroy(pColumns); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { @@ -1297,14 +1297,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (taosArrayPush(pColumns, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } else { if (taosArrayPush(pTags, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1374,14 +1374,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); if (pColumns == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); if (pTags == NULL){ taosArrayDestroy(pColumns); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { @@ -1393,14 +1393,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (taosArrayPush(pColumns, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } else { if (taosArrayPush(pTags, &field) == NULL){ taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } } @@ -1497,7 +1497,7 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { if (taosArrayPush(metaArray, kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; @@ -1547,7 +1547,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { if(taosArrayPush(metaArray, kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; @@ -1646,7 +1646,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { uError("create SSmlHandle hash obj failed"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAILED; } taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); @@ -1665,7 +1665,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAILED; } @@ -1681,7 +1681,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (!kvHash) { uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1703,7 +1703,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { if (taosArrayPush(colsArray, &kvHash) == NULL) { taosHashCleanup(kvHash); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; } @@ -1799,12 +1799,12 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); if (info->pRequest->dbList == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); if (data == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -1817,7 +1817,7 @@ static int32_t smlInsertData(SSmlHandle *info) { int measureLen = tableData->sTableNameLen; char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); if (measure == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); @@ -1828,11 +1828,11 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); if (info->pRequest->tableList == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } if (taosArrayPush(info->pRequest->tableList, &pName) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 10ff234552..ece1ddf61f 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -526,7 +526,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ return ret; } if (taosArrayPush(preLineKV, &kv) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { @@ -873,7 +873,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ cJSON_Delete(valueJson); elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ret = smlParseValueFromJSONObj(valueJson, &kv); if (ret != TSDB_CODE_SUCCESS) { @@ -901,7 +901,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ cJSON_Delete(tagsJson); uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ret = smlParseTagsFromJSON(info, tagsJson, elements); if (unlikely(ret)) { @@ -965,7 +965,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { payloadNum = payloadNum << 1; void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); if (tmp == NULL) { - ret = TSDB_CODE_OUT_OF_MEMORY; + ret = terrno; return ret; } info->lines = (SSmlLineInfo *)tmp; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 14a334e54c..e620ca9b0c 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -150,7 +150,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } void *data = taosMemoryMalloc(pVal->length); if(data == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); pVal->value = data; @@ -271,7 +271,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(keyLen); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); @@ -280,7 +280,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(valueLen); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, value, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); @@ -294,7 +294,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; if(taosArrayPush(preLineKV, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { @@ -422,7 +422,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (keyEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.keyLen); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, key, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); @@ -433,7 +433,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (valueEscaped) { char *tmp = (char *)taosMemoryMalloc(kv.length); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); @@ -459,11 +459,11 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); if (currElement->colArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index a25ef4ec95..e8601e33bc 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -132,7 +132,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ .keyEscaped = false, .valueEscaped = false}; if (taosArrayPush(preLineKV, &kv) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { return TSDB_CODE_SML_INVALID_DATA; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 52b56abb91..175628cc7d 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -15,7 +15,7 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void** } else if (pTblBuf->buffIdx < taosArrayGetSize(pTblBuf->pBufList)) { pTblBuf->pCurBuff = taosArrayGetP(pTblBuf->pBufList, pTblBuf->buffIdx++); if (NULL == pTblBuf->pCurBuff) { - return TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); + return TAOS_GET_TERRNO(terrno); } *pBuf = pTblBuf->pCurBuff; pTblBuf->buffOffset = pTblBuf->buffUnit; @@ -186,7 +186,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { pRes->fields = taosMemoryMalloc(size); pRes->userFields = taosMemoryMalloc(size); if (NULL == pRes->fields || NULL == pRes->userFields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); (void)memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); @@ -204,7 +204,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); } @@ -212,7 +212,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); } @@ -306,7 +306,7 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { }; if (taosHashPut(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid), &cache, sizeof(cache))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (pStmt->sql.autoCreateTbl) { @@ -369,7 +369,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { if (NULL == pStmt->sql.pBindInfo) { pStmt->sql.pBindInfo = taosMemoryMalloc(pTableCtx->boundColsInfo.numOfBound * sizeof(*pStmt->sql.pBindInfo)); if (NULL == pStmt->sql.pBindInfo) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -612,7 +612,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } pStmt->exec.pCurrBlock = pNewBlock; @@ -702,7 +702,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } pStmt->exec.pCurrBlock = pNewBlock; @@ -722,7 +722,6 @@ int32_t stmtResetStmt(STscStmt* pStmt) { pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_OUT_OF_MEMORY; STMT_ERR_RET(terrno); } @@ -739,7 +738,7 @@ int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) { SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i); *p = taosArrayInit(20, POINTER_BYTES); if (*p == NULL) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } } @@ -812,15 +811,15 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { pTblBuf->buffSize = pTblBuf->buffUnit * 1000; pTblBuf->pBufList = taosArrayInit(100, POINTER_BYTES); if (NULL == pTblBuf->pBufList) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } void* buff = taosMemoryMalloc(pTblBuf->buffSize); if (NULL == buff) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pTblBuf->pBufList, &buff) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pTblBuf->pCurBuff = buff; @@ -948,11 +947,11 @@ int32_t stmtInitStbInterlaceTableInfo(STscStmt* pStmt) { for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -1184,11 +1183,11 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray** for (int32_t i = 0; i < 100; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index e7e08c0982..01a7c4be4c 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -643,7 +643,6 @@ static int32_t stmtResetStmt(STscStmt2* pStmt) { pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_OUT_OF_MEMORY; STMT_ERR_RET(terrno); } @@ -660,7 +659,7 @@ static int32_t stmtAsyncOutput(STscStmt2* pStmt, void* param) { SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i); *p = taosArrayInit(20, POINTER_BYTES); if (*p == NULL) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } } @@ -733,15 +732,15 @@ static int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { pTblBuf->buffSize = pTblBuf->buffUnit * 1000; pTblBuf->pBufList = taosArrayInit(100, POINTER_BYTES); if (NULL == pTblBuf->pBufList) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } void* buff = taosMemoryMalloc(pTblBuf->buffSize); if (NULL == buff) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pTblBuf->pBufList, &buff) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pTblBuf->pCurBuff = buff; @@ -796,7 +795,7 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { } pStmt->sql.siInfo.pTableCols = taosArrayInit(STMT_TABLE_COLS_NUM, POINTER_BYTES); if (NULL == pStmt->sql.siInfo.pTableCols) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = terrno; (void)stmtClose(pStmt); return NULL; } @@ -889,11 +888,11 @@ static int32_t stmtInitStbInterlaceTableInfo(STscStmt2* pStmt) { for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -1121,11 +1120,11 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt2* pStmt, SArray** for (int32_t i = 0; i < 100; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } } @@ -1160,7 +1159,7 @@ static int32_t stmtCacheBlock(STscStmt2* pStmt) { }; if (taosHashPut(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid), &cache, sizeof(cache))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (pStmt->sql.autoCreateTbl) { @@ -1554,7 +1553,7 @@ static int32_t createParseContext(const SRequestObj* pRequest, SParseContext** p *pCxt = taosMemoryCalloc(1, sizeof(SParseContext)); if (*pCxt == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } **pCxt = (SParseContext){.requestId = pRequest->requestId, @@ -1671,7 +1670,7 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { } else { SSqlCallbackWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); if (pWrapper == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } else { pWrapper->pRequest = pRequest; pRequest->pWrapper = pWrapper; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 783815d97f..cd443f88e5 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1094,7 +1094,7 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { if (*topics == NULL) { *topics = tmq_list_new(); if (*topics == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } taosRLockLatch(&tmq->lock); @@ -1353,7 +1353,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAIL; } @@ -1394,7 +1394,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } if (taosArrayPush(req.topicNames, &topicFName) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFree(topicFName); goto FAIL; } @@ -1404,7 +1404,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req); buf = taosMemoryMalloc(tlen); if (buf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAIL; } @@ -2017,7 +2017,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFreeClear(msg); return code; } @@ -2713,7 +2713,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { tscError("failed to allocate memory for sync commit"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(&pInfo->sem, 0, 0) != 0) { tscError("failed to init sem for sync commit"); @@ -2785,7 +2785,7 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { tscError("consumer:0x%" PRIx64 " failed to prepare seek operation", tmq->consumerId); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(&pInfo->sem, 0, 0) != 0) { @@ -2922,7 +2922,7 @@ FAIL: int32_t syncAskEp(tmq_t* pTmq) { SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); - if (pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pInfo == NULL) return terrno; if (tsem2_init(&pInfo->sem, 0, 0) != 0) { taosMemoryFree(pInfo); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -3427,7 +3427,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a pCommon->pList = taosArrayInit(4, sizeof(tmq_topic_assignment)); if (pCommon->pList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } if (tsem2_init(&pCommon->rsp, 0, 0) != 0) { @@ -3445,7 +3445,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); if (pParam == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -3649,7 +3649,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ if (pParam == NULL) { taosMemoryFree(msg); taosMemoryFree(sendInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(&pParam->sem, 0, 0) != 0) { taosMemoryFree(msg); From 2c53c08726de3fb3171437a027ca2fcd54831533 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 12 Sep 2024 11:31:10 +0800 Subject: [PATCH 19/20] fix(stream):fix issue when ts less than 0 --- source/libs/stream/src/streamUpdate.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 40de393570..9a1a30ac7b 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -155,7 +155,7 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b } pInfo->pTsBuckets = NULL; pInfo->pTsSBFs = NULL; - pInfo->minTS = -1; + pInfo->minTS = INT64_MIN; pInfo->interval = adjustInterval(interval, precision); pInfo->watermark = adjustWatermark(pInfo->interval, interval, watermark); pInfo->numSBFs = 0; @@ -181,7 +181,7 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b QUERY_CHECK_CODE(code, lino, _end); } - TSKEY dumy = 0; + TSKEY dumy = INT64_MIN; for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) { void* tmp = taosArrayPush(pInfo->pTsBuckets, &dumy); if (!tmp) { @@ -231,11 +231,7 @@ _end: static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - if (ts < 0) { - code = TSDB_CODE_FAILED; - QUERY_CHECK_CODE(code, lino, _end); - } - if (pInfo->minTS < 0) { + if (pInfo->minTS == INT64_MIN) { pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval); } int64_t index = (int64_t)((ts - pInfo->minTS) / pInfo->interval); @@ -349,7 +345,7 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index); - if (ts < maxTs - pInfo->watermark) { + if (ts < maxTs - pInfo->watermark && maxTs != INT64_MIN) { // this window has been closed. if (pInfo->pCloseWinSBF) { code = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen, &res); From b35a435704bd9377fbd4fa58c7feb3485c4d350c Mon Sep 17 00:00:00 2001 From: Bo Ding Date: Thu, 12 Sep 2024 16:12:50 +0800 Subject: [PATCH 20/20] docs(taosx): update pi document --- docs/zh/06-advanced/05-data-in/03-pi.md | 151 +++++++++++++++++++++--- 1 file changed, 134 insertions(+), 17 deletions(-) diff --git a/docs/zh/06-advanced/05-data-in/03-pi.md b/docs/zh/06-advanced/05-data-in/03-pi.md index 2cffb4ce0d..6b65b1337f 100644 --- a/docs/zh/06-advanced/05-data-in/03-pi.md +++ b/docs/zh/06-advanced/05-data-in/03-pi.md @@ -7,21 +7,29 @@ sidebar_label: "PI" ## 功能概述 -PI 系统是一套用于数据收集、查找、分析、传递和可视化的软件产品,可以作为管理实时数据和事件的企业级系统的基础架构。 +PI 系统是一套用于数据收集、查找、分析、传递和可视化的软件产品,可以作为管理实时数据和事件的企业级系统的基础架构。taosX 可以 PI 连接器插件从 PI 系统中提取实时数据或历史数据。 -taosX 可以通过 PI 连接器插件从 PI 系统中提取实时数据。 +从数据的实时性角度来看 PI 数据源任务分为两类:**实时任务**和**回填任务**。在任务类型下拉列表两类任务分别对应名称: **PI** 和 **PI backfill**。 + +从数据模型角度来看, PI 数据源任务分为**单列模型**任务和**多列模型**任务: +1. **单列模型**任务将一个 PI Point 映射为 TDengine 一张表 +2. **多列模型**任务将一个 PI AF 元素映射为一张表 -有两类 PI 任务: 实时同步任务和历史数据迁移任务,在任务下拉列表的名称分别对应 “PI” 和 “PI backfill”。 +从连接的数据源类型来讲,PI 数据源任务又分为 **Archive Server** 数据源和 **AF Server** 数据源。对于 **Archive Server** 数据源,只能使用**单列模型**。对于 **AF Server** 数据源,既可以选择**单列模型**,又可以选择**多列模型**。 + +用户通过一个 CSV 文件配置从 PI 到 TDengine 的数据映射规则,这个 CSV 文件称为**模型配置文件**: +1. 对于使用 AF Server 单列模型的任务,taosX 会自动识别元素的哪些属性是引用 PI Point 的数据,并把一个 PI Point 属性对应到一张表。 +2. 对于使用 AF Server 的多列模型任务,一个元素对应一张表。taosX 默认会把 PI Point 属性映射到 TDengine Metric 列,把其它属性映射到 TDengine TAG 列。 ## 创建任务 -### 1. 新增数据源 +### 新增数据源 在数据写入页面中,点击 **+新增数据源** 按钮,进入新增数据源页面。 ![kafka-01.png](./kafka-01.png) -### 2. 配置基本信息 +### 基本配置 在 **名称** 中输入任务名称,如:“test”; @@ -33,7 +41,7 @@ taosX 可以通过 PI 连接器插件从 PI 系统中提取实时数据。 ![basic.png](./pic/pi-01-agent.png) -### 3. 配置连接信息 +### 连接配置 PI 连接器支持两种连接方式: @@ -46,21 +54,130 @@ PI 连接器支持两种连接方式: 点击 **连通性检查** 按钮,检查数据源是否可用。 -### 4. 配置数据模型 +### 数据模型配置 -这一部分,我们用一个 csv 文件配置 PI 的数据模型到 TDengine 的数据模型的映射规则。这里所说的映射规则包含 3 方面内容: +这一部分有两个 Tab,分别对应单列模型的配置和多列模型的配置。如果你是第一次配置,无论选择单列模型还是多列模型,请务必点击“下载默认配置”按钮。这个操作会触发生成默认的**模型配置文件**,同时也会把**模型配置文件**下载到本地,你可以查看或编辑。编辑后还可以再上传,来覆盖默认的配置。 -1. 定义源数据范围,即哪些点或哪些模板需要同步到 TDengine。 -2. 定义过滤规则,即符合什么条件的数据才需要同步到 TDengine。 -3. 定义转换规则,即对原始数据做什么样的变换后再写入 TDengine。 +如果你想同步所有点位或所有模板的元素,那么用默认的配置就好了。如果你想要过滤特定命名模式的点位或元素模板,那么需要先填写过滤条件再点击“下载默认配置”。 -如果您不知道具体怎么操作,可以点击“下载默认配置”按钮,下载得到的 csv 文件有详细的使用说明。 +#### 多列模型配置文件 -### 5. 其他配置 +下面是一个多列模型配置文件的示例。这个配置文件包含两个超级表的配置:一个是 metertemplate 表,它的数据来自模板 MeterTemplate 的元素;另一个是 farm 表,它的数据来自 Farm 模板的元素。 -其余的配置,比较重要的是: +```csv +SuperTable,metertemplate +SubTable,${element_name}_${element_id} +Template,MeterTemplate +Filter, +ts,KEY,TIMESTAMP,$ts +voltage,COLUMN,DOUBLE,$voltage +voltage_status,COLUMN,INT,$voltage_status +current,COLUMN,DOUBLE,$current +current_status,COLUMN,INT,$current_status +element_id,TAG,VARCHAR(100),$element_id +element_name,TAG,VARCHAR(100),$element_name +path,TAG,VARCHAR(100),$path +categories,TAG,VARCHAR(100),$categories -1. 对于 PI 任务,配置“重启补偿时间”,如果任务意外中断,重启时配置这个参数非常有用,它会让 taosX 自动 backfill 一段时间的数据。 -2. 对于 PI backfill 任务,需要配置 backfill 的开始和结束时间。 +SuperTable,farm +SubTable,${element_name}_${element_id} +Template,Farm +Filter, +ts,KEY,TIMESTAMP,$ts +wind_speed,COLUMN,FLOAT,$wind_speed +wind_speed_status,COLUMN,INT,$wind_speed_status +power_production,COLUMN,FLOAT,$power_production +power_production_status,COLUMN,INT,$power_production_status +lost_power,COLUMN,FLOAT,$lost_power +lost_power_status,COLUMN,INT,$lost_power_status +farm_lifetime_production__weekly_,COLUMN,FLOAT,$farm_lifetime_production__weekly_ +farm_lifetime_production__weekly__status,COLUMN,INT,$farm_lifetime_production__weekly__status +farm_lifetime_production__hourly_,COLUMN,FLOAT,$farm_lifetime_production__hourly_ +farm_lifetime_production__hourly__status,COLUMN,INT,$farm_lifetime_production__hourly__status +element_id,TAG,VARCHAR(100),$element_id +element_name,TAG,VARCHAR(100),$element_name +path,TAG,VARCHAR(100),$path +categories,TAG,VARCHAR(100),$categories +``` + +多列模型配置文件由一个或多个超级表的定义组成。每个超级表的配置都包括: + +1. 超级表和模板的对应关系 +2. 属性和 TDengine Metric 列的对应关系 +3. 属性和 TDengine TAG 列的对应关系 +4. 源数据过滤条件 +5. 对于每一列,无论是 Metrics 列还是 TAG 列,都可以配置一个映射规则,详见[零代码第三方数据接入](../)“数据提取、过滤和转换”部分 + +#### 单列模型配置文件 + +下面是一个单列模型配置文件的示例。 + +```csv +SuperTable,volt_float32 +SubTable,${point_name} +Filter, +ts,KEY,TIMESTAMP,$ts +value,COLUMN,FLOAT,$value +status,COLUMN,INT,$status +path,TAG,VARCHAR(200),$path +point_name,TAG,VARCHAR(100),$point_name +ptclassname,TAG,VARCHAR(100),$ptclassname +sourcetag,TAG,VARCHAR(100),$sourcetag +tag,TAG,VARCHAR(100),$tag +descriptor,TAG,VARCHAR(100),$descriptor +exdesc,TAG,VARCHAR(100),$exdesc +engunits,TAG,VARCHAR(100),$engunits +pointsource,TAG,VARCHAR(100),$pointsource +step,TAG,VARCHAR(100),$step +future,TAG,VARCHAR(100),$future +element_paths,TAG,VARCHAR(512),`$element_paths.replace("\\", ".")` + +SuperTable,milliampere_float32 +SubTable,${point_name} +Filter, +ts,KEY,TIMESTAMP,$ts +value,COLUMN,FLOAT,$value +status,COLUMN,INT,$status +path,TAG,VARCHAR(200),$path +point_name,TAG,VARCHAR(100),$point_name +ptclassname,TAG,VARCHAR(100),$ptclassname +sourcetag,TAG,VARCHAR(100),$sourcetag +tag,TAG,VARCHAR(100),$tag +descriptor,TAG,VARCHAR(100),$descriptor +exdesc,TAG,VARCHAR(100),$exdesc +engunits,TAG,VARCHAR(100),$engunits +pointsource,TAG,VARCHAR(100),$pointsource +step,TAG,VARCHAR(100),$step +future,TAG,VARCHAR(100),$future +element_paths,TAG,VARCHAR(512),`$element_paths.replace("\\", ".")` + +Meter_1000004_Voltage,POINT,volt_float32 +Meter_1000004_Current,POINT,milliampere_float32 +Meter_1000001_Voltage,POINT,volt_float32 +Meter_1000001_Current,POINT,milliampere_float32 +Meter_1000474_Voltage,POINT,volt_float32 +Meter_1000474_Current,POINT,milliampere_float32 +``` + +单列模型配置文件分为两个部分,第一部分通多列模型配置文件,由若干个超级表的定义组成。第二部分是点位列表,这一部分配置了点位和超级表的映射关系。默认生成的配置把 UOM 相同且数据类型相同的点映射到同一个超级表。 + +### Backfill 配置 + +1. 对于 PI 任务,可配置“重启补偿时间”,如果任务意外中断,重启时配置这个参数非常有用,它会让 taosX 自动回填一段时间的数据。 +2. 对于 PI backfill 任务,必须配置 backfill 的开始和结束时间。 + +### 高级选项 + +对于不同的任务类型高级选项部分有所不同。通用的高级选项有: +1. 连接器日志级别 +2. 连接器查询和发送数据的批次大小 +3. 单次读取最大延迟 + +对于**多列模型的实时任务**,还有以下开关选项: + +1. 是否同步新增的元素。如果打开,则 PI 连接器会监听模板下新增的元素,无需重启任务,即可自动同步新增元素的数据。 +2. 是否同步静态属性的变化。如果打开,PI 连接器会同步所有静态属性(非 PI Point 属性)的变化。也就是说如果在 PI AF Server 一个元素的静态属性值做了修改,那么 TDengine 表相应 TAG 的值也会修改。 +3. 是否同步删除元素的操作。如果打开,则 PI 连接器会监听配置的模板下删除元素的事件,并同步删除 TDengine 对应子表。 +4. 是否同步删除历史数据。如果打开,则对于某个元素的时序数据,如果在 PI 中某个时间的数据被删除了,TDengine 对应时间对应列的数据会被置空。 +5. 是否同步修改历史数据。如果打开,则对于某个元素的时序数据,如果在 PI 中历史数据被修改了,TDengine 对应时间的数据也会更新。 -高级配置部分可以配置连接器日志的级别、批次大小、和批次延迟。用于 Debug 和 性能优化。 \ No newline at end of file