From 0b21b7dff175950abba00dcb4d551afac2dd09af Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 14:25:50 +0800 Subject: [PATCH 01/32] fix indirect leak --- source/libs/transport/src/trans.c | 12 ++- source/libs/transport/src/transCli.c | 109 ++++++++------------ source/libs/transport/src/transSvr.c | 143 ++++++++++----------------- 3 files changed, 98 insertions(+), 166 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1c783f44fe..f017909341 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -103,7 +103,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->timeToGetConn = 10 * 1000; } pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; - + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -124,14 +124,20 @@ _end: } void rpcClose(void* arg) { tInfo("start to close rpc"); + if (arg == NULL) { + return; + } (void)transRemoveExHandle(transGetInstMgt(), (int64_t)arg); (void)transReleaseExHandle(transGetInstMgt(), (int64_t)arg); tInfo("end to close rpc"); return; } void rpcCloseImpl(void* arg) { + if (arg == NULL) return; SRpcInfo* pRpc = (SRpcInfo*)arg; - (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + if (pRpc->tcphandle != NULL) { + (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + } taosMemoryFree(pRpc); } @@ -168,7 +174,7 @@ int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64 return transSendRequest(shandle, pEpSet, pMsg, NULL); } int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { - if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0|| pRid == NULL) { + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { return transSendRequest(shandle, pEpSet, pMsg, pCtx); } else { return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 78320c450c..5052d79ef3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -99,12 +99,11 @@ typedef struct SCliMsg { } SCliMsg; typedef struct SCliThrd { - TdThread thread; // tid - int64_t pid; // pid - uv_loop_t* loop; - SAsyncPool* asyncPool; - uv_prepare_t* prepare; - void* pool; // conn pool + TdThread thread; // tid + int64_t pid; // pid + uv_loop_t* loop; + SAsyncPool* asyncPool; + void* pool; // conn pool // timer handles SArray* timerList; // msg queue @@ -167,7 +166,6 @@ static void cliSendCb(uv_write_t* req, int status); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); static void cliIdleCb(uv_idle_t* handle); -static void cliPrepareCb(uv_prepare_t* handle); static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd); static void cliSendBatchCb(uv_write_t* req, int status); @@ -231,7 +229,9 @@ static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static int32_t createThrdObj(void* trans, SCliThrd** pThrd); static void destroyThrdObj(SCliThrd* pThrd); -static void cliWalkCb(uv_handle_t* handle, void* arg); + +int32_t cliSendQuit(SCliThrd* thrd); +static void cliWalkCb(uv_handle_t* handle, void* arg); #define CLI_RELEASE_UV(loop) \ do { \ @@ -2119,33 +2119,6 @@ static void cliAsyncCb(uv_async_t* handle) { if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); } -static void cliPrepareCb(uv_prepare_t* handle) { - SCliThrd* thrd = handle->data; - tTrace("prepare work start"); - - SAsyncPool* pool = thrd->asyncPool; - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - (void)taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); - - int count = 0; - while (!QUEUE_IS_EMPTY(&wq)) { - queue* h = QUEUE_HEAD(&wq); - QUEUE_REMOVE(h); - - SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - (*cliAsyncHandle[pMsg->type])(pMsg, thrd); - count++; - } - } - tTrace("prepare work end"); - if (thrd->stopMsg != NULL) cliHandleQuit(thrd->stopMsg, thrd); -} void cliDestroyConnMsgs(SCliConn* conn, bool destroy) { transCtxCleanup(&conn->ctx); @@ -2260,6 +2233,12 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, _err: if (cli) { + for (int i = 0; i < cli->numOfThreads; i++) { + if (cli->pThreadObj[i]) { + (void)cliSendQuit(cli->pThreadObj[i]); + destroyThrdObj(cli->pThreadObj[i]); + } + } taosMemoryFree(cli->pThreadObj); taosMemoryFree(cli); } @@ -2339,37 +2318,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(code, NULL, _end); } - pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); - if (pThrd->prepare == NULL) { - tError("failed to create prepre since:%s", tstrerror(code)); - TAOS_CHECK_GOTO(code, NULL, _end); - } - - code = uv_prepare_init(pThrd->loop, pThrd->prepare); - if (code != 0) { - tError("failed to create prepre since:%s", uv_err_name(code)); - TAOS_CHECK_GOTO(TSDB_CODE_THIRDPARTY_ERROR, NULL, _end); - } - pThrd->prepare->data = pThrd; - - int32_t timerSize = 64; - pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); - if (pThrd->timerList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - - for (int i = 0; i < timerSize; i++) { - uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); - if (timer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - (void)uv_timer_init(pThrd->loop, timer); - if (taosArrayPush(pThrd->timerList, &timer) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); - } - } - pThrd->pool = createConnPool(4); if (pThrd->pool == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -2402,6 +2350,23 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); } + int32_t timerSize = 64; + pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); + if (pThrd->timerList == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + for (int i = 0; i < timerSize; i++) { + uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + if (timer == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + (void)uv_timer_init(pThrd->loop, timer); + if (taosArrayPush(pThrd->timerList, &timer) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + } pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = trans; pThrd->quit = false; @@ -2411,17 +2376,21 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { _end: if (pThrd) { + (void)taosThreadMutexDestroy(&pThrd->msgMtx); + (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); - taosMemoryFree(pThrd->prepare); - (void)taosThreadMutexDestroy(&pThrd->msgMtx); transAsyncPoolDestroy(pThrd->asyncPool); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); - taosMemoryFree(pThrd->prepare); + + destroyConnPool(pThrd); + transDQDestroy(pThrd->delayQueue, NULL); + transDQDestroy(pThrd->timeoutQueue, NULL); + transDQDestroy(pThrd->waitConnQueue, NULL); taosHashCleanup(pThrd->fqdn2ipCache); taosHashCleanup(pThrd->failFastCache); taosHashCleanup(pThrd->batchCache); @@ -2450,8 +2419,8 @@ static void destroyThrdObj(SCliThrd* pThrd) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } + uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); - taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); taosHashCleanup(pThrd->failFastCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 11aa468b19..0727e49d8b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -82,14 +82,13 @@ typedef struct { int64_t ver; } SIpWhiteListTab; typedef struct SWorkThrd { - TdThread thread; - uv_connect_t connect_req; - uv_pipe_t* pipe; - uv_os_fd_t fd; - uv_loop_t* loop; - SAsyncPool* asyncPool; - uv_prepare_t* prepare; - queue msg; + TdThread thread; + uv_connect_t connect_req; + uv_pipe_t* pipe; + uv_os_fd_t fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + queue msg; queue conn; void* pTransInst; @@ -98,6 +97,7 @@ typedef struct SWorkThrd { SIpWhiteListTab* pWhiteList; int64_t whiteListVer; int8_t enableIpWhiteList; + int8_t inited; } SWorkThrd; typedef struct SServerObj { @@ -139,7 +139,6 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); static void uvShutDownCb(uv_shutdown_t* req, int status); -static void uvPrepareCb(uv_prepare_t* handle); static bool uvRecvReleaseReq(SSvrConn* conn, STransMsgHead* pHead); @@ -180,6 +179,11 @@ static void uvDestroyConn(uv_handle_t* handle); static void* transWorkerThread(void* arg); static void* transAcceptThread(void* arg); +static void destroyWorkThrd(SWorkThrd* pThrd); +static void destroyWorkThrdObj(SWorkThrd* pThrd); + +static void sendQuitToWorkThrd(SWorkThrd* pThrd); + // add handle loop static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); @@ -849,52 +853,6 @@ static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { } return false; } -static void uvPrepareCb(uv_prepare_t* handle) { - // prepare callback - SWorkThrd* pThrd = handle->data; - SAsyncPool* pool = pThrd->asyncPool; - - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - (void)taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); - - while (!QUEUE_IS_EMPTY(&wq)) { - queue* head = QUEUE_HEAD(&wq); - QUEUE_REMOVE(head); - - SSvrMsg* msg = QUEUE_DATA(head, SSvrMsg, q); - if (msg == NULL) { - tError("unexcept occurred, continue"); - continue; - } - // release handle to rpc init - if (msg->type == Quit || msg->type == Update) { - (*transAsyncHandle[msg->type])(msg, pThrd); - continue; - } else { - STransMsg transMsg = msg->msg; - - SExHandle* exh1 = transMsg.info.handle; - int64_t refId = transMsg.info.refId; - SExHandle* exh2 = transAcquireExHandle(transGetSvrRefMgt(), refId); - if (exh2 == NULL || exh1 != exh2) { - tTrace("handle except msg %p, ignore it", exh1); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); - destroySmsg(msg); - continue; - } - msg->pConn = exh1->handle; - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); - (*transAsyncHandle[msg->type])(msg, pThrd); - } - } - } -} static void uvWorkDoTask(uv_work_t* req) { // doing time-consumeing task @@ -1101,25 +1059,6 @@ static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { QUEUE_INIT(&pThrd->msg); - pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); - if (pThrd->prepare == NULL) { - tError("failed to init prepare"); - return TSDB_CODE_OUT_OF_MEMORY; - } - - code = uv_prepare_init(pThrd->loop, pThrd->prepare); - if (code != 0) { - tError("failed to init prepare since %s", uv_err_name(code)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - - code = uv_prepare_start(pThrd->prepare, uvPrepareCb); - if (code != 0) { - tError("failed to start prepare since %s", uv_err_name(code)); - return TSDB_CODE_THIRDPARTY_ERROR; - } - pThrd->prepare->data = pThrd; - // conn set QUEUE_INIT(&pThrd->conn); @@ -1421,11 +1360,11 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - if (false == taosValidIpAndPort(srv->ip, srv->port)) { - code = TAOS_SYSTEM_ERROR(errno); - tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - goto End; - } + // if (false == taosValidIpAndPort(srv->ip, srv->port)) { + // code = TAOS_SYSTEM_ERROR(errno); + // tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + // goto End; + // } char pipeName[PATH_MAX]; #if defined(WINDOWS) || defined(DARWIN) @@ -1485,12 +1424,14 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, code = TSDB_CODE_OUT_OF_MEMORY; goto End; } + srv->pThreadObj[i] = thrd; thrd->pTransInst = shandle; thrd->quit = false; thrd->pTransInst = shandle; thrd->pWhiteList = uvWhiteListCreate(); if (thrd->pWhiteList == NULL) { + destroyWorkThrdObj(thrd); code = TSDB_CODE_OUT_OF_MEMORY; goto End; } @@ -1501,8 +1442,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - srv->pThreadObj[i] = thrd; - uv_os_sock_t fds[2]; if ((code = uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE)) != 0) { tError("failed to create pipe, errmsg: %s", uv_err_name(code)); @@ -1539,6 +1478,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, tError("failed to create worker-thread:%d", i); goto End; } + thrd->inited = 1; } #endif @@ -1560,6 +1500,12 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->inited = true; return srv; End: + for (int i = 0; i < srv->numOfThreads; i++) { + if (srv->pThreadObj[i] != NULL) { + SWorkThrd* thrd = srv->pThreadObj[i]; + destroyWorkThrd(thrd); + } + } transCloseServer(srv); terrno = code; return NULL; @@ -1663,20 +1609,27 @@ void uvHandleUpdate(SSvrMsg* msg, SWorkThrd* thrd) { taosMemoryFree(msg); } +void destroyWorkThrdObj(SWorkThrd* pThrd) { + if (pThrd == NULL) { + return; + } + transAsyncPoolDestroy(pThrd->asyncPool); + uvWhiteListDestroy(pThrd->pWhiteList); + uv_loop_close(pThrd->loop); + taosMemoryFree(pThrd->loop); + taosMemoryFree(pThrd); +} void destroyWorkThrd(SWorkThrd* pThrd) { if (pThrd == NULL) { return; } - (void)taosThreadJoin(pThrd->thread, NULL); - SRV_RELEASE_UV(pThrd->loop); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); - transAsyncPoolDestroy(pThrd->asyncPool); - - uvWhiteListDestroy(pThrd->pWhiteList); - - taosMemoryFree(pThrd->prepare); - taosMemoryFree(pThrd->loop); - taosMemoryFree(pThrd); + if (pThrd->inited) { + sendQuitToWorkThrd(pThrd); + (void)taosThreadJoin(pThrd->thread, NULL); + SRV_RELEASE_UV(pThrd->loop); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); + } + destroyWorkThrdObj(pThrd); } void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); @@ -1693,13 +1646,15 @@ void transCloseServer(void* arg) { tDebug("send quit msg to accept thread"); (void)uv_async_send(srv->pAcceptAsync); (void)taosThreadJoin(srv->thread, NULL); + SRV_RELEASE_UV(srv->loop); + (void)uv_loop_close(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); } } else { + SRV_RELEASE_UV(srv->loop); (void)uv_loop_close(srv->loop); } @@ -1708,7 +1663,9 @@ void transCloseServer(void* arg) { taosMemoryFree(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - taosMemoryFree(srv->pipe[i]); + if (srv->pipe[i] != NULL) { + taosMemoryFree(srv->pipe[i]); + } } taosMemoryFree(srv->pipe); From b6b0e266eb3033821a1655cadaafa26581124229 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 14:28:34 +0800 Subject: [PATCH 02/32] fix indirect leak --- source/libs/transport/src/transSvr.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 0727e49d8b..7e06767cb5 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1360,11 +1360,11 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } - // if (false == taosValidIpAndPort(srv->ip, srv->port)) { - // code = TAOS_SYSTEM_ERROR(errno); - // tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - // goto End; - // } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + code = TAOS_SYSTEM_ERROR(errno); + tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + goto End; + } char pipeName[PATH_MAX]; #if defined(WINDOWS) || defined(DARWIN) From 0973fb6eed8a96bc0ec30dfa6e5b59fcfc35b6b3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 16:14:19 +0800 Subject: [PATCH 03/32] fix indirect leak --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5052d79ef3..85c8f7c21a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2387,7 +2387,7 @@ _end: } taosArrayDestroy(pThrd->timerList); - destroyConnPool(pThrd); + (void)destroyConnPool(pThrd); transDQDestroy(pThrd->delayQueue, NULL); transDQDestroy(pThrd->timeoutQueue, NULL); transDQDestroy(pThrd->waitConnQueue, NULL); @@ -2419,7 +2419,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); } - uv_loop_close(pThrd->loop); + (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7e06767cb5..38c7edeeaf 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1615,7 +1615,7 @@ void destroyWorkThrdObj(SWorkThrd* pThrd) { } transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); - uv_loop_close(pThrd->loop); + (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } From 3b283b63507a64fddd30fa7a219e15e559f733ee Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 16:36:19 +0800 Subject: [PATCH 04/32] fix indirect leak --- source/libs/transport/src/transCli.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 85c8f7c21a..c2ce00c1cf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2383,6 +2383,7 @@ _end: transAsyncPoolDestroy(pThrd->asyncPool); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + (void)uv_timer_stop(timer); taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); @@ -2417,8 +2418,10 @@ static void destroyThrdObj(SCliThrd* pThrd) { tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + (void)uv_timer_stop(timer); taosMemoryFree(timer); } + (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); From 46a79c0be333809745ea9e69580bd5d71af0b934 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Sep 2024 17:50:35 +0800 Subject: [PATCH 05/32] fix indirect leak --- source/libs/transport/src/transCli.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c2ce00c1cf..803e50ce94 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -619,7 +619,11 @@ void* createConnPool(int size) { return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); } void* destroyConnPool(SCliThrd* pThrd) { - void* pool = pThrd->pool; + void* pool = pThrd->pool; + if (pool == NULL) { + return NULL; + } + SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conns)) { From 520336e129eeda517a4f28a81504fb41d63431b2 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 9 Sep 2024 15:47:17 +0800 Subject: [PATCH 06/32] osDir --- source/client/src/clientMonitor.c | 5 +-- source/common/src/cos.c | 1 - source/libs/index/src/indexTfile.c | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 6 ++-- source/libs/stream/src/streamCheckpoint.c | 2 +- source/libs/stream/src/streamSnapshot.c | 2 +- source/libs/tdb/src/db/tdbPager.c | 4 +-- source/libs/wal/src/walMeta.c | 4 +-- source/os/src/osDir.c | 34 ++++++++++++++++--- source/util/src/tcompare.c | 4 +-- 10 files changed, 44 insertions(+), 20 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 9ed6512352..6a0a09f9bd 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -821,9 +821,10 @@ int32_t monitorInit() { return code; } - if (taosMulModeMkDir(tmpSlowLogPath, 0777, true) != 0) { + code = taosMulModeMkDir(tmpSlowLogPath, 0777, true); + if (code != 0) { tscError("failed to create dir:%s since %s", tmpSlowLogPath, terrstr()); - return TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); + return code; } if (tsem2_init(&monitorSem, 0, 0) != 0) { diff --git a/source/common/src/cos.c b/source/common/src/cos.c index f02130d468..064e06eb43 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -1881,7 +1881,6 @@ void s3EvictCache(const char *path, long object_size) { // 1, list data files' atime under dir(path) tdbDirPtr pDir = taosOpenDir(dir_name); if (pDir == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); vError("failed to open %s since %s", dir_name, terrstr()); } SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile)); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 55a9bb06d9..8e1c4a2d2f 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -1125,7 +1125,7 @@ static int32_t tfileGetFileList(const char* path, SArray** ppResult) { TdDirPtr pDir = taosOpenDir(path); if (NULL == pDir) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _exception); + TAOS_CHECK_GOTO(terrno, NULL, _exception); } TdDirEntryPtr pDirEntry; while ((pDirEntry = taosReadDir(pDir)) != NULL) { diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4793a8951a..465968dc50 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -599,7 +599,7 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) { // copy file to dst TdDirPtr pDir = taosOpenDir(src); if (pDir == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _ERROR; } @@ -1428,7 +1428,7 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI code = taosMulModeMkDir(pChkpDir, 0755, true); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code)); goto _EXIT; } @@ -4633,7 +4633,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { TdDirPtr pDir = taosOpenDir(p->buf); if (pDir == NULL) { (void)taosThreadRwlockUnlock(&p->rwLock); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } TdDirEntryPtr de = NULL; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 916aee4e6e..05dcab3aae 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -1224,7 +1224,7 @@ static int32_t uploadCheckpointToS3(const char* id, const char* path) { TdDirPtr pDir = taosOpenDir(path); if (pDir == NULL) { - return TAOS_SYSTEM_ERROR(errno); + return terrno; } TdDirEntryPtr de = NULL; diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 0390422623..eb222efce6 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -271,7 +271,7 @@ int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { int32_t code = 0; TdDirPtr pDir = taosOpenDir(pSnapFile->path); if (NULL == pDir) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("%s failed to open %s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, tstrerror(code)); return code; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index b0f32136a3..4a5ad9d663 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -1136,7 +1136,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } SArray *pTxnList = taosArrayInit(16, sizeof(int64_t)); @@ -1182,7 +1182,7 @@ int tdbPagerRollback(SPager *pPager) { tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); - return terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } while ((pDirEntry = tdbReadDir(pDir)) != NULL) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 0eb011113a..fb30dd3376 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -403,7 +403,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { regfree(&logRegPattern); regfree(&idxRegPattern); wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - TAOS_RETURN(TSDB_CODE_FAILED); + return terrno; } SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo)); @@ -422,7 +422,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { regfree(&idxRegPattern); (void)taosCloseDir(&pDir); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + return terrno; } } } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 9f3908183e..d233a4fe0e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -145,7 +145,10 @@ int32_t taosMulMkDir(const char *dirname) { char *pos = temp; int32_t code = 0; #ifdef WINDOWS - taosRealPath(dirname, temp, sizeof(temp)); + code = taosRealPath(dirname, temp, sizeof(temp)); + if(code != 0) { + return code; + } if (temp[1] == ':') pos += 3; #else (void)strcpy(temp, dirname); @@ -207,7 +210,10 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { char *pos = temp; int32_t code = 0; #ifdef WINDOWS - taosRealPath(dirname, temp, sizeof(temp)); + code = taosRealPath(dirname, temp, sizeof(temp)); + if(code != 0) { + return code; + } if (temp[1] == ':') pos += 3; #else (void)strcpy(temp, dirname); @@ -430,6 +436,9 @@ TdDirPtr taosOpenDir(const char *dirname) { HANDLE hFind; TdDirPtr pDir = taosMemoryMalloc(sizeof(TdDir)); + if(pDir == NULL) { + return NULL; + } strcpy(szFind, dirname); strcat(szFind, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 @@ -437,6 +446,8 @@ TdDirPtr taosOpenDir(const char *dirname) { pDir->hFind = FindFirstFile(szFind, &(pDir->dirEntry.findFileData)); if (INVALID_HANDLE_VALUE == pDir->hFind) { taosMemoryFree(pDir); + DWORD errorCode = GetLastError(); + terrno = TAOS_SYSTEM_ERROR(errorCode); return NULL; } return pDir; @@ -444,6 +455,11 @@ TdDirPtr taosOpenDir(const char *dirname) { DIR *pDir = opendir(dirname); if (pDir == NULL) return NULL; TdDirPtr dirPtr = (TdDirPtr)taosMemoryMalloc(sizeof(TdDir)); + if (dirPtr == NULL) { + (void)closedir(pDir); + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } dirPtr->dirEntryPtr = (TdDirEntryPtr) & (dirPtr->dirEntry1); dirPtr->pDir = pDir; return dirPtr; @@ -506,22 +522,30 @@ char *taosGetDirEntryName(TdDirEntryPtr pDirEntry) { } int32_t taosCloseDir(TdDirPtr *ppDir) { + int32_t code = 0; if (ppDir == NULL || *ppDir == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } #ifdef WINDOWS - FindClose((*ppDir)->hFind); + if(!FindClose((*ppDir)->hFind)) { + terrno = TAOS_SYSTEM_ERROR(GetLastError()); + return terrno; + } taosMemoryFree(*ppDir); *ppDir = NULL; return 0; #elif defined(DARWIN) - closedir((*ppDir)->pDir); + code = closedir((*ppDir)->pDir); + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } taosMemoryFree(*ppDir); *ppDir = NULL; return 0; #else - int32_t code = closedir((DIR *)*ppDir); + code = closedir((DIR *)*ppDir); *ppDir = NULL; if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index afa048f0ac..b1f4ed0ed3 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1227,8 +1227,8 @@ static void checkRegexCache(void* param, void* tmrId) { if(sRegexCache.exit) { goto _exit; } - bool ret = taosTmrReset(checkRegexCache, REGEX_CACHE_CLEAR_TIME * 1000, param, sRegexCache.regexCacheTmr, &tmrId); - if (!ret) { + bool stopped = taosTmrReset(checkRegexCache, REGEX_CACHE_CLEAR_TIME * 1000, param, sRegexCache.regexCacheTmr, &tmrId); + if (stopped) { uError("failed to reset regex cache timer"); goto _exit; } From 2a3917029f3cc5967f42b99952b7deb100845da8 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 10 Sep 2024 11:29:50 +0800 Subject: [PATCH 07/32] osFile --- include/os/osSocket.h | 2 +- include/util/taoserror.h | 3 + source/common/src/cos_cp.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 7 +- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 8 +- source/dnode/mgmt/node_util/src/dmEps.c | 7 +- source/dnode/mgmt/node_util/src/dmFile.c | 24 ++---- source/dnode/mnode/sdb/src/sdbFile.c | 32 +++---- source/dnode/vnode/src/tsdb/tsdbFS.c | 7 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 7 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 7 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 2 +- source/libs/executor/src/groupcacheoperator.c | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 9 +- source/libs/stream/src/streamSnapshot.c | 4 +- source/libs/sync/src/syncRaftCfg.c | 6 +- source/libs/sync/src/syncRaftStore.c | 2 +- source/libs/tdb/src/db/tdbUtil.c | 4 +- source/libs/wal/src/walMeta.c | 10 +-- source/libs/wal/src/walRead.c | 10 +-- source/libs/wal/src/walWrite.c | 4 +- source/os/src/osDir.c | 4 +- source/os/src/osEnv.c | 5 +- source/os/src/osFile.c | 83 +++++++++---------- source/os/src/osSocket.c | 14 ++-- source/os/src/osSysinfo.c | 2 +- source/util/src/terror.c | 10 +++ source/util/src/tpagedbuf.c | 2 +- 29 files changed, 131 insertions(+), 152 deletions(-) diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 0427c6b22f..48478c8f49 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -143,7 +143,7 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); -void taosWinSocketInit(); +int32_t taosWinSocketInit(); /* * set timeout(ms) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4591c7fbcc..4e909b0a1c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -35,6 +35,8 @@ extern STaosError errors[]; #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) +#define TAOS_SYSTEM_WINAPI_ERROR(code) (0x81ff0000 | (code)) +#define TAOS_SYSTEM_WINSOCKET_ERROR(code) (0x82ff0000 | (code)) #define TAOS_SUCCEEDED(err) ((err) >= 0) #define TAOS_FAILED(err) ((err) < 0) @@ -153,6 +155,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MSG_PREPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0136) // internal #define TSDB_CODE_OUT_OF_BUFFER TAOS_DEF_ERROR_CODE(0, 0x0137) #define TSDB_CODE_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0138) +#define TSDB_CODE_SOCKET_ERROR TAOS_DEF_ERROR_CODE(0, 0x0139) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index e9af9c0306..793433bba7 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -175,7 +175,7 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { int64_t n = taosReadFile(fd, cp_body, size); if (n < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } else if (n != size) { TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 563d123b83..d9f6920fcb 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -96,7 +96,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { } if (taosReadFile(pFile, pData, size) != size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to read mnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -215,10 +215,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { code = TAOS_SYSTEM_ERROR(errno); goto _OVER; } - if (taosRenameFile(file, realfile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _OVER; - } + TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 8513d31695..d5b1e61aed 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -133,7 +133,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t } if (taosReadFile(pFile, pData, size) != size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to read vnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -253,12 +253,8 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { code = TAOS_SYSTEM_ERROR(errno); goto _OVER; } - if (taosRenameFile(file, realfile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _OVER; - } + TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); - code = 0; dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes); _OVER: diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 315c4d7430..7c0272965a 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -222,7 +222,7 @@ int32_t dmReadEps(SDnodeData *pData) { } if (taosReadFile(pFile, content, size) != size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to read dnode file:%s since %s", file, terrstr()); goto _OVER; } @@ -340,9 +340,8 @@ int32_t dmWriteEps(SDnodeData *pData) { if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER); (void)taosCloseFile(&pFile); - if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER); + TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); - code = 0; pData->updateTime = taosGetTimestampMs(); dInfo("succeed to write dnode file:%s, num:%d ver:%" PRId64, realfile, (int32_t)taosArrayGetSize(pData->dnodeEps), pData->dnodeVer); @@ -626,7 +625,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { } if (taosReadFile(pFile, content, size) != size) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = terrno; dError("failed to read dnode file:%s since %s", file, terrstr()); goto _OVER; } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index c8b24d5469..3412d5ed3a 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -75,7 +75,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { } if (taosReadFile(pFile, content, size) != size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to read file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -166,12 +166,8 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { code = TAOS_SYSTEM_ERROR(errno); goto _OVER; } - if (taosRenameFile(file, realfile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _OVER; - } + TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); - code = 0; dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed); _OVER: @@ -262,10 +258,7 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool goto _OVER; } - if (taosRenameFile(file, realfile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _OVER; - } + TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); encryptDebug("succeed to write checkCode file:%s", realfile); @@ -302,10 +295,7 @@ static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptC goto _OVER; } - if (taosRenameFile(file, realfile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _OVER; - } + TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); encryptDebug("succeed to write encryptCode file:%s", realfile); @@ -338,12 +328,12 @@ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) { content = taosMemoryMalloc(size); if (content == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OVER; } if (taosReadFile(pFile, content, size) != size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; encryptError("failed to read dnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -484,7 +474,7 @@ static int32_t dmReadEncryptCodeFile(char *file, char **output) { } if (taosReadFile(pFile, content, size) != size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to read dnode file:%s since %s", file, tstrerror(code)); goto _OVER; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 94826405dc..813579ac26 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -86,8 +86,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { int64_t sver = 0; int32_t ret = taosReadFile(pFile, &sver, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (ret != sizeof(int64_t)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -100,8 +99,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { ret = taosReadFile(pFile, &pSdb->applyIndex, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (ret != sizeof(int64_t)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -110,8 +108,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { ret = taosReadFile(pFile, &pSdb->applyTerm, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (ret != sizeof(int64_t)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -120,8 +117,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { ret = taosReadFile(pFile, &pSdb->applyConfig, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (ret != sizeof(int64_t)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -132,8 +128,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { int64_t maxId = 0; ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (ret != sizeof(int64_t)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -148,8 +143,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { int64_t ver = 0; ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (ret != sizeof(int64_t)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -163,8 +157,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { char reserve[SDB_RESERVE_SIZE] = {0}; ret = taosReadFile(pFile, reserve, sizeof(reserve)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (ret != sizeof(reserve)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -274,7 +267,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { if (ret == 0) break; if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mError("failed to read sdb file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -305,7 +298,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { ret = taosReadFile(pFile, pRaw->pData, readLen); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mError("failed to read sdb file:%s since %s, ret:%" PRId64 " readLen:%d", file, tstrerror(code), ret, readLen); goto _OVER; } @@ -523,7 +516,6 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { if (code == 0) { code = taosRenameFile(tmpfile, curfile); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); mError("failed to write sdb file:%s since %s", curfile, tstrerror(code)); } } @@ -693,7 +685,7 @@ int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) { int32_t readlen = taosReadFile(pIter->file, pBuf, maxlen); if (readlen < 0 || readlen > maxlen) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, tstrerror(code), pIter->total); *ppBuf = NULL; *len = 0; @@ -758,8 +750,8 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i char datafile[PATH_MAX] = {0}; snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); - if (taosRenameFile(pIter->name, datafile) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosRenameFile(pIter->name, datafile); + if (code != 0) { mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, tstrerror(code)); goto _OVER; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index ed80b5d412..cde4d7ded8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -310,7 +310,7 @@ static int32_t load_fs(const char *fname, STsdbFS *pFS) { } if (taosReadFile(pFD, pData, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; (void)taosCloseFile(&pFD); TSDB_CHECK_CODE(code, lino, _exit); } @@ -711,10 +711,7 @@ static int32_t tsdbFSCommit(STsdb *pTsdb) { if (!taosCheckExistFile(current_t)) goto _exit; // rename the file - if (taosRenameFile(current_t, current) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } + TSDB_CHECK_CODE(taosRenameFile(current_t, current), lino, _exit); // Load the new FS code = tsdbFSCreate(&fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 24916bf47c..843e700fd3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -130,7 +130,7 @@ static int32_t load_json(const char *fname, cJSON **json) { } if (taosReadFile(fp, data, size) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } data[size] = '\0'; @@ -309,10 +309,7 @@ static int32_t commit_edit(STFileSystem *fs) { int32_t code; int32_t lino; - if ((code = taosRenameFile(current_t, current))) { - code = TAOS_SYSTEM_ERROR(code); - TSDB_CHECK_CODE(code, lino, _exit); - } + TSDB_CHECK_CODE(taosRenameFile(current_t, current), lino, _exit); code = apply_commit(fs); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e044743bea..c34b29f97d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -226,7 +226,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor // read n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); if (n < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } else if (n < pFD->szPage) { TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } @@ -378,7 +378,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo ret = taosReadFile(pFD->pFD, buf + n, nRead); if (ret < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } else if (ret < nRead) { TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index ddd1ac9d4a..c272e540ba 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -211,8 +211,9 @@ int vnodeCommitInfo(const char *dir) { snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME); snprintf(tfname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP); - if (taosRenameFile(tfname, fname) < 0) { - return terrno = TAOS_SYSTEM_ERROR(errno); + int32_t code = taosRenameFile(tfname, fname); + if (code < 0) { + return code; } vInfo("vnode info is committed, dir:%s", dir); @@ -245,7 +246,7 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) { } if (taosReadFile(pFile, pData, size) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } pData[size] = '\0'; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 86c25009bc..7d8b5d4eda 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -289,7 +289,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) { taosMemoryFree(*ppData); (void)taosCloseFile(&pFile); - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } (void)taosCloseFile(&pFile); diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 1796aa7b64..576f3932b6 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -562,7 +562,7 @@ static int32_t readBlockFromDisk(SGroupCacheOperatorInfo* pGCache, SGroupCacheDa ret = (int32_t)taosReadFile(pFileFd->fd, *ppBuf, pBasic->bufSize); if (ret != pBasic->bufSize) { taosMemoryFreeClear(*ppBuf); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _return; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 465968dc50..4e14df9a40 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -296,7 +296,7 @@ int32_t remoteChkp_readMetaData(char* path, SSChkpMetaOnS3** pMeta) { char buf[256] = {0}; if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _EXIT; } @@ -367,8 +367,8 @@ int32_t remoteChkp_validAndCvtMeta(char* path, SSChkpMetaOnS3* pMeta, int64_t ch goto _EXIT; } - if (taosRenameFile(src, dst) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosRenameFile(src, dst); + if (code != 0) { goto _EXIT; } @@ -507,7 +507,6 @@ int32_t rebuildFromRemoteChkp_s3(const char* key, char* chkpPath, int64_t chkpId if (taosIsDir(defaultPath)) { code = taosRenameFile(defaultPath, defaultTmp); if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); goto _EXIT; } else { rename = 1; @@ -1609,7 +1608,7 @@ int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId) } if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("failed to read file to load extra info, file:%s, reason:%s", pDst, tstrerror(code)); goto _EXIT; } diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index eb222efce6..2b69da6007 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -696,7 +696,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) { int64_t bytes = taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); if (bytes != pHdr->size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); goto _err; } else { @@ -735,7 +735,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t // open fd again, let's close fd during handle errors. if (taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset) != pHdr->size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); goto _err; } diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 82cc86ed86..ac3bba8c42 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -164,9 +164,7 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { } (void)taosCloseFile(&pFile); - if (taosRenameFile(file, realfile) != 0) { - TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); - } + TAOS_CHECK_EXIT(taosRenameFile(file, realfile)); sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d, lastConfigIndex:%" PRId64 ", changeVersion:%d", pNode->vgId, realfile, len, pNode->raftCfg.lastConfigIndex, pNode->raftCfg.cfg.changeVersion); @@ -280,7 +278,7 @@ int32_t syncReadCfgFile(SSyncNode *pNode) { } if (taosReadFile(pFile, pData, size) != size) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, tstrerror(code)); goto _OVER; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index cc2aa7d91e..d6482ba2da 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -72,7 +72,7 @@ int32_t raftStoreReadFile(SSyncNode *pNode) { if (taosReadFile(pFile, pData, size) != size) { sError("vgId:%d, failed to read raft store file:%s since %s", pNode->vgId, file, terrstr()); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } pData[size] = '\0'; diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index d6d33bc7d8..2249123ef5 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -39,7 +39,9 @@ int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) { int64_t stDev = 0, stIno = 0; int32_t code = taosDevInoFile(fd, &stDev, &stIno); - return code; + if (TSDB_CODE_SUCCESS != code) { + return code; + } memset(fileid, 0, TDB_FILE_ID_LEN); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index fb30dd3376..378e9d1903 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -112,7 +112,7 @@ static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, in if (readSize != taosReadFile(pFile, buf, readSize)) { wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), readSize, fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } char* candidate = NULL; @@ -177,7 +177,7 @@ static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, in if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; break; } } @@ -522,7 +522,7 @@ static int32_t walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pC } if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } if (walValidHeadCksum(pCkHead) != 0) { @@ -588,7 +588,7 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), offset, fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } if (idxEntry.ver > pFileInfo->lastVer) { @@ -1028,7 +1028,7 @@ int32_t walLoadMeta(SWal* pWal) { (void)taosCloseFile(&pFile); taosMemoryFree(buf); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } // load into fileInfoSet code = walMetaDeserialize(pWal, buf); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index deb5a07672..6476cd39ac 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -144,7 +144,7 @@ static int32_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int if (ret < 0) { wError("vgId:%d, failed to read idx file, since %s", pReader->pWal->cfg.vgId, terrstr()); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } else { wError("vgId:%d, read idx file incompletely, read bytes %" PRId64 ", bytes should be %ld", pReader->pWal->cfg.vgId, ret, sizeof(SWalIdxEntry)); @@ -270,7 +270,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { continue; } else { if (contLen < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } else { TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); } @@ -345,7 +345,7 @@ int32_t walFetchBody(SWalReader *pRead) { wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%" PRIx64, vgId, pReadHead->version, ver, tstrerror(terrno), id); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } else { wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted, 0x%" PRIx64, @@ -426,7 +426,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex)); if (contLen < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } else { TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); } @@ -467,7 +467,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex)); if (contLen < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } else { TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 52af3e8528..a11831c437 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -214,7 +214,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); @@ -240,7 +240,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if (size != sizeof(SWalCkHead)) { TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } code = walValidHeadCksum(&head); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index d233a4fe0e..dcce5543b2 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -447,7 +447,7 @@ TdDirPtr taosOpenDir(const char *dirname) { if (INVALID_HANDLE_VALUE == pDir->hFind) { taosMemoryFree(pDir); DWORD errorCode = GetLastError(); - terrno = TAOS_SYSTEM_ERROR(errorCode); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errorCode); return NULL; } return pDir; @@ -529,7 +529,7 @@ int32_t taosCloseDir(TdDirPtr *ppDir) { } #ifdef WINDOWS if(!FindClose((*ppDir)->hFind)) { - terrno = TAOS_SYSTEM_ERROR(GetLastError()); + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return terrno; } taosMemoryFree(*ppDir); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index c0ea49e2cb..8334d609b1 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -64,7 +64,10 @@ int32_t osDefaultInit() { } #ifdef WINDOWS - taosWinSocketInit(); + code = taosWinSocketInit(); + if (code != TSDB_CODE_SUCCESS) { + return code; + } const char *tmpDir = getenv("tmp"); if (tmpDir == NULL) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 144baeb148..f4afaeabd1 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -132,6 +132,7 @@ int64_t taosCopyFile(const char *from, const char *to) { if (CopyFile(from, to, 0)) { return 1; } else { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return -1; } #else @@ -231,8 +232,9 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { HANDLE transactionHandle = CreateTransaction(NULL, NULL, 0, 0, 0, INFINITE, NULL); if (transactionHandle == INVALID_HANDLE_VALUE) { - printf("failed to rename file %s to %s, reason: CreateTransaction failed.\n", oldName, newName); - return -1; + DWORD error = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(error); + return terrno; } BOOL result = MoveFileTransacted(oldName, newName, NULL, NULL, MOVEFILE_REPLACE_EXISTING, transactionHandle); @@ -241,18 +243,18 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { finished = CommitTransaction(transactionHandle); if (!finished) { DWORD error = GetLastError(); - printf("failed to rename file %s to %s, reason: CommitTransaction errcode %d.\n", oldName, newName, error); + terrno = TAOS_SYSTEM_WINAPI_ERROR(error); } } else { RollbackTransaction(transactionHandle); DWORD error = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(error); finished = false; - printf("failed to rename file %s to %s, reason: MoveFileTransacted errcode %d.\n", oldName, newName, error); } CloseHandle(transactionHandle); - return finished ? 0 : -1; + return finished ? 0 : terrno; #else int32_t code = rename(oldName, newName); if (-1 == code) { @@ -260,7 +262,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { return terrno; } - return code; + return TSDB_CODE_SUCCESS; #endif } @@ -294,12 +296,14 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *a int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #ifdef WINDOWS if (pFile == NULL || pFile->hFile == NULL) { - return -1; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } BY_HANDLE_FILE_INFORMATION bhfi; if (GetFileInformationByHandle(pFile->hFile, &bhfi) == FALSE) { - printf("taosFStatFile get file info fail."); - return -1; + DWORD error = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(error); + return terrno; } if (stDev != NULL) { @@ -395,35 +399,38 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { if (h != INVALID_HANDLE_VALUE && (tdFileOptions & TD_FILE_APPEND) && (tdFileOptions & TD_FILE_WRITE)) { SetFilePointer(h, 0, NULL, FILE_END); } - // if (h == INVALID_HANDLE_VALUE) { - // DWORD dwError = GetLastError(); - // LPVOID lpMsgBuf; - // FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, - // 0, - // (LPTSTR)&lpMsgBuf, 0, NULL); - // printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); - // LocalFree(lpMsgBuf); - // } + if (h == INVALID_HANDLE_VALUE) { + DWORD dwError = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(error); + // LPVOID lpMsgBuf; + // FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, 0, (LPTSTR)&lpMsgBuf, 0, + // NULL); + // printf("CreateFile failed with error %d: %s", dwError, (char *)lpMsgBuf); + // LocalFree(lpMsgBuf); + } return h; } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK - taosThreadRwlockRdlock(&(pFile->rwlock)); + (void)taosThreadRwlockRdlock(&(pFile->rwlock)); #endif if (pFile->hFile == NULL) { #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif - return -1; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } DWORD bytesRead; if (!ReadFile(pFile->hFile, buf, count, &bytesRead, NULL)) { + DWORD errCode = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errCode); bytesRead = -1; } #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return bytesRead; } @@ -449,14 +456,15 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { if (pFile == NULL) { + terrno = TSDB_CODE_INVALID_PARA; return 0; } #if FILE_WITH_LOCK - taosThreadRwlockWrlock(&(pFile->rwlock)); + (void)taosThreadRwlockWrlock(&(pFile->rwlock)); #endif if (pFile->hFile == NULL) { #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return 0; } @@ -470,11 +478,12 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t BOOL result = WriteFile(pFile->hFile, buf, count, &ret, &ol); if (!result) { errno = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); ret = -1; } #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return ret; } @@ -484,7 +493,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return -1; } #if FILE_WITH_LOCK - taosThreadRwlockRdlock(&(pFile->rwlock)); + (void)taosThreadRwlockRdlock(&(pFile->rwlock)); #endif LARGE_INTEGER liOffset; @@ -498,7 +507,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { return -1; } #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return liOffset.QuadPart; } @@ -826,25 +835,11 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t } #endif -#ifdef WINDOWS - DWORD ret = 0; - OVERLAPPED ol = {0}; - ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20); - ol.Offset = (uint32_t)(offset & 0xFFFFFFFFLL); - - HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd); - SetLastError(0); - BOOL result = WriteFile(handle, buf, count, &ret, &ol); - if (!result) { - errno = GetLastError(); - ret = -1; - } -#else int64_t ret = pwrite(pFile->fd, buf, count, offset); if (-1 == ret) { code = TAOS_SYSTEM_ERROR(errno); } -#endif + #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -867,14 +862,10 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { int32_t code = 0; -#ifdef WINDOWS - int64_t ret = _lseeki64(pFile->fd, offset, whence); -#else int64_t ret = lseek(pFile->fd, offset, whence); if (-1 == ret) { code = TAOS_SYSTEM_ERROR(errno); } -#endif #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 2d160b277b..48ca36b5e5 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -1170,19 +1170,23 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { return (int)fd; } -void taosWinSocketInit() { +int32_t taosWinSocketInit() { #ifdef WINDOWS - static char flag = 0; - if (flag == 0) { + static int8_t flag = 0; + if (atomic_val_compare_exchange_8(&flag, 0, 1) == 0) { WORD wVersionRequested; WSADATA wsaData; wVersionRequested = MAKEWORD(1, 1); - if (WSAStartup(wVersionRequested, &wsaData) == 0) { - flag = 1; + if (WSAStartup(wVersionRequested, &wsaData) != 0) { + atomic_store_8(&flag, 0) + int errorCode = WSAGetLastError(); + return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); } } + return 0; #else #endif + return 0; } uint64_t taosHton64(uint64_t val) { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 9a4f0f8a98..137bb8f0db 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -300,7 +300,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { if (taosReadFile(pFile, (void *)email, maxLen) < 0) { taosCloseFile(&pFile); - return -1; + return terrno; } taosCloseFile(&pFile); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 58dde5cd23..5ceec33831 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -850,6 +850,7 @@ static void tsSortError(void) { taosSort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); } +static char WinAPIErrDesc[256] = {0}; const char* tstrerror(int32_t err) { (void)taosThreadOnce(&tsErrorInit, tsSortError); @@ -860,6 +861,15 @@ const char* tstrerror(int32_t err) { // invalid code return Unknown error return strerror(code); } + #ifdef WINDOWS + if ((err & 0x01ff0000) == 0x01ff0000) { + snprintf(WinAPIErrDesc, 256, "windows api error, code: 0x%08x", err & 0x0000ffff); + return WinAPIErrDesc; + } else if ((err & 0x02ff0000) == 0x02ff0000) { + snprintf(WinAPIErrDesc, 256, "windows socket error, code: 0x%08x", err & 0x0000ffff); + return WinAPIErrDesc; + } + #endif int32_t s = 0; int32_t e = sizeof(errors) / sizeof(errors[0]); diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 32b9b6e18d..de27cf4df9 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -256,7 +256,7 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { void* pPage = (void*)GET_PAYLOAD_DATA(pg); ret = (int32_t)taosReadFile(pBuf->pFile, pPage, pg->length); if (ret != pg->length) { - ret = TAOS_SYSTEM_ERROR(errno); + ret = terrno; return ret; } From a12e96d2c30a2a2edf7b5855de26ef3e3da4635c Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 10 Sep 2024 13:51:25 +0800 Subject: [PATCH 08/32] fix: build on windows --- source/os/src/osFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index f4afaeabd1..3be2b1e351 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -401,7 +401,7 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { } if (h == INVALID_HANDLE_VALUE) { DWORD dwError = GetLastError(); - terrno = TAOS_SYSTEM_WINAPI_ERROR(error); + terrno = TAOS_SYSTEM_WINAPI_ERROR(dwError); // LPVOID lpMsgBuf; // FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, 0, (LPTSTR)&lpMsgBuf, 0, // NULL); From aa5683cfd9fb9933924e3e0f0a6af0d8e9e2580f Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 10 Sep 2024 14:55:14 +0800 Subject: [PATCH 09/32] osfile --- source/client/src/clientMonitor.c | 4 +-- source/common/src/cos.c | 6 ++-- source/common/src/cos_cp.c | 4 +-- source/common/src/rsync.c | 4 +-- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 6 ++-- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 6 ++-- source/dnode/mgmt/node_util/src/dmEps.c | 10 +++--- source/dnode/mgmt/node_util/src/dmFile.c | 18 +++++----- source/dnode/mnode/sdb/src/sdbFile.c | 30 ++++++---------- source/dnode/vnode/src/tsdb/tsdbFS.c | 6 ++-- source/dnode/vnode/src/tsdb/tsdbFS2.c | 7 ++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 8 ++--- source/dnode/vnode/src/vnd/vnodeCommit.c | 7 ++-- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 5 +-- source/libs/executor/src/groupcacheoperator.c | 10 +++--- source/libs/stream/src/streamBackendRocksdb.c | 4 +-- source/libs/sync/src/syncRaftCfg.c | 6 ++-- source/libs/sync/src/syncRaftStore.c | 8 ++--- source/libs/wal/src/walMeta.c | 20 +++++------ source/libs/wal/src/walRead.c | 6 ++-- source/libs/wal/src/walWrite.c | 12 +++---- source/os/src/osFile.c | 34 ++++++++++++------- source/os/src/osSocket.c | 2 +- source/util/src/tconfig.c | 7 +++- source/util/src/tlog.c | 5 ++- source/util/src/tpagedbuf.c | 5 ++- 26 files changed, 122 insertions(+), 118 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 6a0a09f9bd..36dc474dd8 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -397,7 +397,7 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP } if (taosLSeekFile(pFile, 0, SEEK_END) < 0) { - tscError("failed to seek file:%p code: %d", pFile, errno); + tscError("failed to seek file:%p code: %d", pFile, terrno); return; } if (taosWriteFile(pFile, slowLogData->data, strlen(slowLogData->data) + 1) < 0) { @@ -409,7 +409,7 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP static char* readFile(TdFilePtr pFile, int64_t* offset, int64_t size) { tscDebug("[monitor] readFile slow begin pFile:%p, offset:%" PRId64 ", size:%" PRId64, pFile, *offset, size); if (taosLSeekFile(pFile, *offset, SEEK_SET) < 0) { - tscError("failed to seek file:%p code: %d", pFile, errno); + tscError("failed to seek file:%p code: %d", pFile, terrno); return NULL; } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 064e06eb43..1c06d9206e 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -132,7 +132,7 @@ int32_t s3CheckCfg() { } if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { (void)fprintf(stderr, "failed to write test file: %s.\n", path); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _next); + TAOS_CHECK_GOTO(terrno, &lino, _next); } if (taosFsyncFile(fp) < 0) { (void)fprintf(stderr, "failed to fsync test file: %s.\n", path); @@ -872,7 +872,7 @@ upload: if (i > 0 && cp.parts[i - 1].completed) { if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } } @@ -1076,7 +1076,7 @@ static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *objec } if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) { (void)taosCloseFile(&data.infileFD); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 793433bba7..1caf6a7189 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -210,10 +210,10 @@ static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) { TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } if (taosLSeekFile(fp, 0, SEEK_SET) < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (taosWriteFile(fp, data, strlen(data)) < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (taosFsyncFile(fp) < 0) { diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index b53c4e416e..525e1e0aea 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -90,10 +90,10 @@ static int32_t generateConfigFile(char* confDir) { #endif ); uDebug("[rsync] conf:%s", confContent); - if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) { + if (taosWriteFile(pFile, confContent, strlen(confContent)) != TSDB_CODE_SUCCESS) { uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); (void)taosCloseFile(&pFile); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; return code; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index d9f6920fcb..05a09b9a97 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -83,8 +83,8 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { dError("failed to fstat mnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -203,7 +203,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { int32_t len = strlen(buffer); if (taosWriteFile(pFile, buffer, len) <= 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } if (taosFsyncFile(pFile) < 0) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index d5b1e61aed..b7bf25c8d3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -120,8 +120,8 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { dError("failed to fstat mnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -240,7 +240,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { int32_t len = strlen(buffer); if (taosWriteFile(pFile, buffer, len) <= 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } if (taosFsyncFile(pFile) < 0) { diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 7c0272965a..685d70e1ae 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -209,8 +209,8 @@ int32_t dmReadEps(SDnodeData *pData) { } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { dError("failed to fstat dnode file:%s since %s", file, terrstr()); goto _OVER; } @@ -336,7 +336,7 @@ int32_t dmWriteEps(SDnodeData *pData) { if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER); int32_t len = strlen(buffer); - if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER); + if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(terrno, NULL, _OVER); if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER); (void)taosCloseFile(&pFile); @@ -612,8 +612,8 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { dError("failed to fstat dnode file:%s since %s", file, terrstr()); goto _OVER; } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 3412d5ed3a..c524f4eacf 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -62,8 +62,8 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { dError("failed to fstat file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -154,7 +154,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { int32_t len = strlen(buffer); if (taosWriteFile(pFile, buffer, len) <= 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } if (taosFsyncFile(pFile) < 0) { @@ -244,7 +244,7 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool } if (taosWriteFile(pFile, opts.result, len) <= 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } @@ -282,7 +282,7 @@ static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptC int32_t len = strlen(encryptCode); if (taosWriteFile(pFile, encryptCode, len) <= 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } if (taosFsyncFile(pFile) < 0) { @@ -320,8 +320,8 @@ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) { goto _OVER; } - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { encryptError("failed to fstat dnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -461,8 +461,8 @@ static int32_t dmReadEncryptCodeFile(char *file, char **output) { } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { dError("failed to fstat dnode file:%s since %s", file, tstrerror(code)); goto _OVER; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 813579ac26..e19f57ad46 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -168,26 +168,21 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { } static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { - int32_t code = 0; int64_t sver = SDB_FILE_VER; if (taosWriteFile(pFile, &sver, sizeof(int64_t)) != sizeof(int64_t)) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (taosWriteFile(pFile, &pSdb->applyIndex, sizeof(int64_t)) != sizeof(int64_t)) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (taosWriteFile(pFile, &pSdb->applyTerm, sizeof(int64_t)) != sizeof(int64_t)) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } if (taosWriteFile(pFile, &pSdb->applyConfig, sizeof(int64_t)) != sizeof(int64_t)) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { @@ -196,8 +191,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { maxId = pSdb->maxId[i]; } if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } } @@ -207,15 +201,13 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { ver = pSdb->tableVer[i]; } if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } } char reserve[SDB_RESERVE_SIZE] = {0}; if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { - code = TAOS_SYSTEM_ERROR(errno); - TAOS_RETURN(code); + return terrno; } return 0; @@ -440,7 +432,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { pRaw->status = pRow->status; if (taosWriteFile(pFile, pRaw, sizeof(SSdbRaw)) != sizeof(SSdbRaw)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); break; @@ -472,7 +464,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { } if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); break; @@ -484,7 +476,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { - code = TAOS_SYSTEM_ERROR(errno); + code = errno; taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); break; @@ -784,7 +776,7 @@ int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) { int32_t code = 0; int32_t writelen = taosWriteFile(pIter->file, pBuf, len); if (writelen != len) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mError("failed to write len:%d since %s, total:%" PRId64, len, tstrerror(code), pIter->total); TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index cde4d7ded8..3e2d8c026f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -116,7 +116,7 @@ static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { int64_t n = taosWriteFile(pFD, pData, size); if (n < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } if (taosFsyncFile(pFD) < 0) { @@ -296,8 +296,8 @@ static int32_t load_fs(const char *fname, STsdbFS *pFS) { } int64_t size; - if (taosFStatFile(pFD, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFD, &size, NULL); + if (code != 0) { (void)taosCloseFile(&pFD); TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 843e700fd3..ea6f10b1bd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -93,7 +93,7 @@ static int32_t save_json(const cJSON *json, const char *fname) { } if (taosWriteFile(fp, data, strlen(data)) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } if (taosFsyncFile(fp) < 0) { @@ -120,8 +120,9 @@ static int32_t load_json(const char *fname, cJSON **json) { } int64_t size; - if (taosFStatFile(fp, &size, NULL) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + code = taosFStatFile(fp, &size, NULL); + if (code != 0) { + TSDB_CHECK_CODE(code, lino, _exit); } data = taosMemoryMalloc(size + 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c34b29f97d..b0fbc19ec0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -154,7 +154,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); if (n < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } (void)taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); @@ -183,7 +183,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); if (n < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } if (pFD->szFile < pFD->pgno) { @@ -220,7 +220,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor // seek int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); if (n < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } // read @@ -373,7 +373,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo // read last chunk int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET); if (ret < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } ret = taosReadFile(pFD->pFD, buf + n, nRead); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index c272e540ba..485499a28f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -185,7 +185,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } if (taosWriteFile(pFile, data, strlen(data)) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } if (taosFsyncFile(pFile) < 0) { @@ -236,9 +236,8 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) { TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } - if (taosFStatFile(pFile, &size, NULL) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); - } + code = taosFStatFile(pFile, &size, NULL); + TSDB_CHECK_CODE(code, lino, _exit); pData = taosMemoryMalloc(size + 1); if (pData == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 7d8b5d4eda..e1ac42ca8c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -272,9 +272,10 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } int64_t size; - if (taosFStatFile(pFile, &size, NULL) < 0) { + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { (void)taosCloseFile(&pFile); - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code, lino, _exit); } *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1); diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 576f3932b6..15350bdfef 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -315,16 +315,16 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC } int32_t ret = taosLSeekFile(pFd->fd, pHead->basic.offset, SEEK_SET); - if (ret == -1) { + if (ret < 0) { releaseFdToFileCtx(pFd); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _return; } ret = (int32_t)taosWriteFile(pFd->fd, pHead->pBuf, pHead->basic.bufSize); if (ret != pHead->basic.bufSize) { releaseFdToFileCtx(pFd); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _return; } @@ -548,8 +548,8 @@ static int32_t readBlockFromDisk(SGroupCacheOperatorInfo* pGCache, SGroupCacheDa } int32_t ret = taosLSeekFile(pFileFd->fd, pBasic->offset, SEEK_SET); - if (ret == -1) { - code = TAOS_SYSTEM_ERROR(errno); + if (ret < 0) { + code = terrno; goto _return; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4e14df9a40..993d9d91af 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1669,7 +1669,7 @@ int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId) { } if (nBytes != taosWriteFile(pFile, buf, nBytes)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("failed to write file to add extra info, file:%s, reason:%s", pDst, tstrerror(code)); goto _EXIT; } @@ -4970,7 +4970,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { nBytes = taosWriteFile(pFile, content, strlen(content)); if (nBytes != strlen(content)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("chkp failed to write meta file: %s,reason:%s", dstDir, tstrerror(code)); (void)taosCloseFile(&pFile); goto _ERROR; diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index ac3bba8c42..c7811ed093 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -156,7 +156,7 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { int32_t len = strlen(buffer); if (taosWriteFile(pFile, buffer, len) <= 0) { - TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); + TAOS_CHECK_EXIT(terrno); } if (taosFsyncFile(pFile) < 0) { @@ -265,8 +265,8 @@ int32_t syncReadCfgFile(SSyncNode *pNode) { } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { sError("vgId:%d, failed to fstat sync cfg file:%s since %s", pNode->vgId, file, tstrerror(code)); goto _OVER; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index d6482ba2da..a4377858d8 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -58,10 +58,10 @@ int32_t raftStoreReadFile(SSyncNode *pNode) { } int64_t size = 0; - if (taosFStatFile(pFile, &size, NULL) < 0) { + code = taosFStatFile(pFile, &size, NULL); + if (code != 0) { sError("vgId:%d, failed to fstat raft store file:%s since %s", pNode->vgId, file, terrstr()); - - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + TAOS_CHECK_GOTO(code, &lino, _OVER); } pData = taosMemoryMalloc(size + 1); @@ -130,7 +130,7 @@ int32_t raftStoreWriteFile(SSyncNode *pNode) { if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); int32_t len = strlen(buffer); - if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(terrno, &lino, _OVER); if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 378e9d1903..3c144cae23 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -106,7 +106,7 @@ static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, in int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); if (ret < 0) { wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } if (readSize != taosReadFile(pFile, buf, readSize)) { @@ -169,9 +169,9 @@ static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, in } int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); if (ret < 0) { - wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(terrno), offset); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; break; } if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { @@ -518,7 +518,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { static int32_t walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { @@ -578,14 +578,14 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { // determine the last valid entry end, i.e. offset while ((offset -= sizeof(SWalIdxEntry)) >= 0) { if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { - wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(terrno), offset, fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(terrno), offset, fnameStr); TAOS_CHECK_GOTO(terrno, &lino, _err); @@ -628,7 +628,7 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { wError("vgId:%d, failed to seek file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset, fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } int64_t count = 0; @@ -654,7 +654,7 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } count++; } @@ -950,7 +950,7 @@ int32_t walSaveMeta(SWal* pWal) { if (pWal->cfg.level != TAOS_WAL_SKIP && len != taosWriteFile(pMetaFile, serialized, len)) { wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pMetaFile) < 0) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 6476cd39ac..73cf6d46ef 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -137,7 +137,7 @@ static int32_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int wError("vgId:%d, failed to seek idx file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, offset, terrstr()); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } SWalIdxEntry entry = {0}; if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) { @@ -158,7 +158,7 @@ static int32_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int wError("vgId:%d, failed to seek log file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, entry.offset, terrstr()); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -303,7 +303,7 @@ int32_t walSkipFetchBody(SWalReader *pRead) { } int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR); if (code < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } pRead->curVersion++; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a11831c437..43d5c07270 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -207,7 +207,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if (code < 0) { TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } // read idx file and get log file pos SWalIdxEntry entry; @@ -232,7 +232,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // TODO TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } // validate offset SWalCkHead head; @@ -536,13 +536,13 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { pWal->stopDnode(); } - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } // check alignment of idx entries int64_t endOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END); if (endOffset < 0) { - wFatal("vgId:%d, failed to seek end of WAL idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + wFatal("vgId:%d, failed to seek end of WAL idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(terrno), ver); taosMsleep(100); exit(EXIT_FAILURE); @@ -578,7 +578,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -634,7 +634,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy } if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 3be2b1e351..042be9c71f 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -437,19 +437,22 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { if (pFile == NULL || pFile->hFile == NULL) { + terrno = TSDB_CODE_INVALID_PARA; return 0; } #if FILE_WITH_LOCK - taosThreadRwlockWrlock(&(pFile->rwlock)); + (void)taosThreadRwlockWrlock(&(pFile->rwlock)); #endif DWORD bytesWritten; if (!WriteFile(pFile->hFile, buf, count, &bytesWritten, NULL)) { + errno = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); bytesWritten = -1; } #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); + (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif return bytesWritten; } @@ -490,6 +493,7 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { if (pFile == NULL || pFile->hFile == NULL) { + terrno = TSDB_CODE_INVALID_PARA; return -1; } #if FILE_WITH_LOCK @@ -499,11 +503,15 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { LARGE_INTEGER liOffset; liOffset.QuadPart = offset; if (!SetFilePointerEx(pFile->hFile, liOffset, NULL, whence)) { + errno = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); return -1; } liOffset.QuadPart = 0; if (!SetFilePointerEx(pFile->hFile, liOffset, &liOffset, FILE_CURRENT)) { + errno = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); return -1; } #if FILE_WITH_LOCK @@ -514,13 +522,16 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL || pFile->hFile == NULL) { - return 0; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } if (size != NULL) { LARGE_INTEGER fileSize; if (!GetFileSizeEx(pFile->hFile, &fileSize)) { - return -1; // Error getting file size + errno = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + return terrno; // Error getting file size } *size = fileSize.QuadPart; } @@ -528,7 +539,9 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (mtime != NULL) { FILETIME creationTime, lastAccessTime, lastWriteTime; if (!GetFileTime(pFile->hFile, &creationTime, &lastAccessTime, &lastWriteTime)) { - return -1; // Error getting file time + errno = GetLastError(); + terrno = TAOS_SYSTEM_WINAPI_ERROR(errno); + return terrno; // Error getting file time } // Convert the FILETIME structure to a time_t value ULARGE_INTEGER ull; @@ -854,7 +867,7 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { if (pFile == NULL || pFile->fd < 0) { terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return -1; } #if FILE_WITH_LOCK (void)taosThreadRwlockRdlock(&(pFile->rwlock)); @@ -873,7 +886,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { if (code) { terrno = code; - return terrno; + return -1; } return ret; @@ -890,16 +903,11 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { return terrno; } -#ifdef WINDOWS - struct __stat64 fileStat; - int32_t code = _fstat64(pFile->fd, &fileStat); -#else struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); -#endif if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); - return code; + return terrno; } if (size != NULL) { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 48ca36b5e5..4d650715f9 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -1178,7 +1178,7 @@ int32_t taosWinSocketInit() { WSADATA wsaData; wVersionRequested = MAKEWORD(1, 1); if (WSAStartup(wVersionRequested, &wsaData) != 0) { - atomic_store_8(&flag, 0) + atomic_store_8(&flag, 0); int errorCode = WSAGetLastError(); return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index c6da1537f5..48fe271d33 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1255,11 +1255,16 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); } size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END); + if(fileSize <= 0) { + (void)taosCloseFile(&pFile); + (void)printf("load json file error: %s\n", filepath); + TAOS_CHECK_EXIT(terrno); + } char *buf = taosMemoryMalloc(fileSize + 1); if (!buf) { (void)taosCloseFile(&pFile); (void)printf("load json file error: %s, failed to alloc memory\n", filepath); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } buf[fileSize] = 0; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 49e1d87d80..98e17c5db9 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -545,9 +545,9 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { // only an estimate for number of lines int64_t filesize = 0; - if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { + if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) != 0) { (void)printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } tsLogObj.lines = (int32_t)(filesize / 60); @@ -1003,7 +1003,6 @@ _return: if (pFile) (void)taosCloseFile(&pFile); - terrno = TAOS_SYSTEM_ERROR(errno); taosPrintLog(flags, level, dflag, "crash signal is %d", signum); #ifdef _TD_DARWIN_64 diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index de27cf4df9..265f06cbfd 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -136,13 +136,12 @@ static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize static int32_t doFlushBufPageImpl(SDiskbasedBuf* pBuf, int64_t offset, const char* pData, int32_t size) { int32_t ret = taosLSeekFile(pBuf->pFile, offset, SEEK_SET); if (ret == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = terrno; return terrno; } ret = (int32_t)taosWriteFile(pBuf->pFile, pData, size); if (ret != size) { - terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } @@ -249,7 +248,7 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret == -1) { - ret = TAOS_SYSTEM_ERROR(errno); + ret = terrno; return ret; } From aea768d2c07e414a68fbe22405acbb856d619da3 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 10 Sep 2024 17:40:19 +0800 Subject: [PATCH 10/32] osFile --- source/client/src/clientMonitor.c | 12 +- source/common/src/cos.c | 16 +- source/common/src/cos_cp.c | 6 +- source/common/src/rsync.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 8 +- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 6 +- source/dnode/mgmt/node_util/src/dmEps.c | 10 +- source/dnode/mgmt/node_util/src/dmFile.c | 20 +- source/dnode/mnode/impl/src/mndDump.c | 2 +- source/dnode/mnode/sdb/src/sdbFile.c | 10 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 14 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 6 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 8 +- source/dnode/vnode/src/tsdb/tsdbRetention.c | 26 +-- source/dnode/vnode/src/vnd/vnodeCommit.c | 6 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 2 +- source/libs/executor/src/groupcacheoperator.c | 2 +- source/libs/function/src/udfd.c | 2 +- source/libs/parser/src/parInsertSql.c | 2 +- source/libs/parser/src/parTranslater.c | 6 +- source/libs/stream/src/streamBackendRocksdb.c | 12 +- source/libs/stream/src/streamSnapshot.c | 4 +- source/libs/sync/src/syncRaftCfg.c | 4 +- source/libs/sync/src/syncRaftStore.c | 8 +- source/libs/wal/src/walMeta.c | 22 +- source/libs/wal/src/walRead.c | 4 +- source/libs/wal/src/walWrite.c | 44 ++-- source/os/src/osFile.c | 212 ++++++------------ source/util/src/tconfig.c | 8 +- source/util/src/tlog.c | 9 +- source/util/src/tpagedbuf.c | 2 +- 31 files changed, 202 insertions(+), 293 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 36dc474dd8..b890b61668 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -34,11 +34,11 @@ static void processFileInTheEnd(TdFilePtr pFile, char* path) { return; } if (taosFtruncateFile(pFile, 0) != 0) { - tscError("failed to truncate file:%s, errno:%d", path, errno); + tscError("failed to truncate file:%s, errno:%d", path, terrno); return; } if (taosUnLockFile(pFile) != 0) { - tscError("failed to unlock file:%s, errno:%d", path, errno); + tscError("failed to unlock file:%s, errno:%d", path, terrno); return; } if (taosCloseFile(&(pFile)) != 0) { @@ -367,7 +367,7 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpP tscInfo("[monitor] create slow log file:%s", path); pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC); if (pFile == NULL) { - tscError("failed to open file:%s since %d", path, errno); + tscError("failed to open file:%s since %d", path, terrno); return; } @@ -549,7 +549,7 @@ static void monitorSendSlowLogAtRunning(int64_t clusterId) { int64_t size = getFileSize(pClient->path); if (size <= pClient->offset) { if (taosFtruncateFile(pClient->pFile, 0) < 0) { - tscError("failed to truncate file:%p code: %d", pClient->pFile, errno); + tscError("failed to truncate file:%p code: %d", pClient->pFile, terrno); } tscDebug("[monitor] monitorSendSlowLogAtRunning truncate file to 0 file:%p", pClient->pFile); pClient->offset = 0; @@ -606,7 +606,7 @@ static void monitorSendAllSlowLogAtQuit() { static void processFileRemoved(SlowLogClient* pClient) { if (taosUnLockFile(pClient->pFile) != 0) { - tscError("failed to unlock file:%s since %d", pClient->path, errno); + tscError("failed to unlock file:%s since %d", pClient->path, terrno); return; } (void)taosCloseFile(&(pClient->pFile)); @@ -614,7 +614,7 @@ static void processFileRemoved(SlowLogClient* pClient) { TdFilePtr pFile = taosOpenFile(pClient->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC); if (pFile == NULL) { - tscError("failed to open file:%s since %d", pClient->path, errno); + tscError("failed to open file:%s since %d", pClient->path, terrno); } else { pClient->pFile = pFile; } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 1c06d9206e..d3b3f1f87d 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -128,7 +128,7 @@ int32_t s3CheckCfg() { if (!fp) { (void)fprintf(stderr, "failed to open test file: %s.\n", path); // uError("ERROR: %s Failed to open %s", __func__, path); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _next); + TAOS_CHECK_GOTO(terrno, &lino, _next); } if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { (void)fprintf(stderr, "failed to write test file: %s.\n", path); @@ -136,7 +136,7 @@ int32_t s3CheckCfg() { } if (taosFsyncFile(fp) < 0) { (void)fprintf(stderr, "failed to fsync test file: %s.\n", path); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _next); + TAOS_CHECK_GOTO(terrno, &lino, _next); } (void)taosCloseFile(&fp); @@ -988,12 +988,12 @@ int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8 if (taosStatFile(file, (int64_t *)&contentLength, &lmtime, NULL) < 0) { uError("ERROR: %s Failed to stat file %s: ", __func__, file); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { uError("ERROR: %s Failed to open file %s: ", __func__, file); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = @@ -1065,14 +1065,14 @@ static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *objec if (taosStatFile(file, (int64_t *)&contentLength, &lmtime, NULL) < 0) { uError("ERROR: %s Failed to stat file %s: ", __func__, file); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } contentLength = size; if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { uError("ERROR: %s Failed to open file %s: ", __func__, file); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) { (void)taosCloseFile(&data.infileFD); @@ -1412,8 +1412,8 @@ static int32_t s3GetObjectToFileByEp(const char *object_name, const char *fileNa TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - uError("[s3] open file error, errno:%d, fileName:%s", TAOS_SYSTEM_ERROR(errno), fileName); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName); + TAOS_RETURN(terrno); } TS3GetData cbd = {0}; diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 1caf6a7189..ce5f666f21 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -10,7 +10,7 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) { TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH); if (!fd) { uError("%s Failed to open %s", __func__, cp_path); - TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_CHECK_RETURN(terrno); } checkpoint->thefile = fd; @@ -162,7 +162,7 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ); if (!fd) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } int64_t size = -1; @@ -207,7 +207,7 @@ static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) { TdFilePtr fp = checkpoint->thefile; if (taosFtruncateFile(fp, 0) < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (taosLSeekFile(fp, 0, SEEK_SET) < 0) { TAOS_CHECK_GOTO(terrno, &lino, _exit); diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index 525e1e0aea..77787db3ce 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -58,7 +58,7 @@ static int32_t generateConfigFile(char* confDir) { TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } #ifdef WINDOWS diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 05a09b9a97..073ba293e2 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -71,13 +71,13 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { } if (taosStatFile(file, NULL, NULL, NULL) < 0) { - dInfo("mnode file:%s not exist, reason:%s", file, tstrerror(TAOS_SYSTEM_ERROR(errno))); + dInfo("mnode file:%s not exist, reason:%s", file, tstrerror(terrno)); return 0; } pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to open mnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -197,7 +197,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } @@ -207,7 +207,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { goto _OVER; } if (taosFsyncFile(pFile) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index b7bf25c8d3..2f93d51ea2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -106,7 +106,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); if (taosStatFile(file, NULL, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dInfo("vnode file:%s not exist, reason:%s", file, tstrerror(code)); code = 0; return code; @@ -114,7 +114,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to open vnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -234,7 +234,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 685d70e1ae..982c139c7b 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -203,7 +203,7 @@ int32_t dmReadEps(SDnodeData *pData) { pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to open dnode file:%s since %s", file, terrstr()); goto _OVER; } @@ -333,11 +333,11 @@ int32_t dmWriteEps(SDnodeData *pData) { } pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); - if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER); + if (pFile == NULL) TAOS_CHECK_GOTO(terrno, NULL, _OVER); int32_t len = strlen(buffer); if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(terrno, NULL, _OVER); - if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER); + if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(terrno, NULL, _OVER); (void)taosCloseFile(&pFile); TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER); @@ -598,7 +598,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); if (taosStatFile(file, NULL, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dDebug("dnode file:%s not exist, reason:%s", file, tstrerror(code)); code = 0; goto _OVER; @@ -606,7 +606,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) { pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to open dnode file:%s since %s", file, terrstr()); goto _OVER; } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index c524f4eacf..61a72f2aeb 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -56,7 +56,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to open file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -148,7 +148,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } @@ -158,7 +158,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { goto _OVER; } if (taosFsyncFile(pFile) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } @@ -188,7 +188,7 @@ int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile) { *pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC); if (*pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to open file:%s since %s", filepath, tstrerror(code)); return code; } @@ -199,7 +199,7 @@ int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile) { ret = taosLockFile(*pFile); if (ret == 0) break; - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; taosMsleep(1000); retryTimes++; dError("failed to lock file:%s since %s, retryTimes:%d", filepath, tstrerror(code), retryTimes); @@ -239,7 +239,7 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } @@ -276,7 +276,7 @@ static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptC pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } @@ -286,7 +286,7 @@ static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptC goto _OVER; } if (taosFsyncFile(pFile) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _OVER; } @@ -315,7 +315,7 @@ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) { pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; encryptError("failed to open dnode file:%s since %s", file, tstrerror(code)); goto _OVER; } @@ -455,7 +455,7 @@ static int32_t dmReadEncryptCodeFile(char *file, char **output) { pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; dError("failed to open dnode file:%s since %s", file, tstrerror(code)); goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 04c9093c11..7613282ef8 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -616,7 +616,7 @@ void mndDumpSdb() { char file[] = "sdb.json"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = terrno; mError("failed to write %s since %s", file, terrstr()); return; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index e19f57ad46..fd4819dadd 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -234,7 +234,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { taosMemoryFree(pRaw); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mInfo("read sdb file:%s finished since %s", file, tstrerror(code)); return 0; } @@ -386,7 +386,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mError("failed to open sdb file:%s for write since %s", tmpfile, tstrerror(code)); TAOS_RETURN(code); } @@ -638,8 +638,8 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter int64_t commitTerm = pSdb->commitTerm; int64_t commitConfig = pSdb->commitConfig; if (taosCopyFile(datafile, pIter->name) < 0) { + code = terrno; (void)taosThreadMutexUnlock(&pSdb->filelock); - code = TAOS_SYSTEM_ERROR(errno); mError("failed to copy sdb file %s to %s since %s", datafile, pIter->name, tstrerror(code)); sdbCloseIter(pIter); TAOS_RETURN(code); @@ -648,7 +648,7 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter pIter->file = taosOpenFile(pIter->name, TD_FILE_READ); if (pIter->file == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mError("failed to open sdb file:%s since %s", pIter->name, tstrerror(code)); sdbCloseIter(pIter); TAOS_RETURN(code); @@ -708,7 +708,7 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) { pIter->file = taosOpenFile(pIter->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pIter->file == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; mError("failed to open %s since %s", pIter->name, tstrerror(code)); sdbCloseIter(pIter); TAOS_RETURN(code); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 3e2d8c026f..ecf838b761 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -111,7 +111,7 @@ static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { // save to file pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFD == NULL) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } int64_t n = taosWriteFile(pFD, pData, size); @@ -176,7 +176,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { if (pTsdb->fs.pDelFile) { tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname); if (taosStatFile(fname, &size, NULL, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } } @@ -190,7 +190,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { // head ========= tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); if (taosStatFile(fname, &size, NULL, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } // if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) { @@ -201,7 +201,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { // data ========= tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); if (taosStatFile(fname, &size, NULL, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } // if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { @@ -216,7 +216,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { // sma ============= tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); if (taosStatFile(fname, &size, NULL, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } // if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { @@ -232,7 +232,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); if (taosStatFile(fname, &size, NULL, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } // if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) { @@ -291,7 +291,7 @@ static int32_t load_fs(const char *fname, STsdbFS *pFS) { // load binary TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ); if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index ea6f10b1bd..f7c50e4260 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -89,7 +89,7 @@ static int32_t save_json(const cJSON *json, const char *fname) { fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (fp == NULL) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } if (taosWriteFile(fp, data, strlen(data)) < 0) { @@ -97,7 +97,7 @@ static int32_t save_json(const cJSON *json, const char *fname) { } if (taosFsyncFile(fp) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } _exit: @@ -116,7 +116,7 @@ static int32_t load_json(const char *fname, cJSON **json) { TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); if (fp == NULL) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } int64_t size; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index b0fbc19ec0..c0a1e4b047 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -46,14 +46,14 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { pFD->pFD = taosOpenFile(lc_path, flag); if (pFD->pFD == NULL) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } } else { tsdbInfo("no file: %s", path); - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } pFD->s3File = 1; } @@ -73,7 +73,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { // not check file size when reading data files. if (flag != TD_FILE_READ /* && !pFD->s3File*/) { if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 96b752104d..c80e7c651e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -48,7 +48,7 @@ static int32_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_ int64_t n; int64_t last = taosGetTimestampMs(); if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) { - TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_CHECK_RETURN(terrno); } remain -= n; @@ -76,14 +76,14 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi fdFrom = taosOpenFile(fname_from, TD_FILE_READ); if (fdFrom == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size); fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (fdTo == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config; @@ -91,7 +91,7 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1); if (taosFSendFile(fdTo, fdFrom, 0, lc_size) < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } _exit: @@ -116,14 +116,14 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile fdFrom = taosOpenFile(from->fname, TD_FILE_READ); if (fdFrom == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, from->f->size); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (fdTo == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } TSDB_CHECK_CODE(code, lino, _exit); @@ -429,7 +429,7 @@ static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile fdFrom = taosOpenFile(from->fname, TD_FILE_READ); if (fdFrom == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } char *object_name = taosDirEntryBaseName(fname); @@ -519,7 +519,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int fdFrom = taosOpenFile(fname, TD_FILE_READ); if (fdFrom == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } tsdbInfo("vgId:%d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size); @@ -527,12 +527,12 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (fdTo == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); if (n < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } _exit: @@ -618,19 +618,19 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ); if (fdFrom == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (fdTo == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); if (n < 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 485499a28f..53d590bfbd 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -181,7 +181,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // save info to a vnode_tmp.json pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } if (taosWriteFile(pFile, data, strlen(data)) < 0) { @@ -189,7 +189,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } if (taosFsyncFile(pFile) < 0) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } _exit: @@ -233,7 +233,7 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) { // read info pFile = taosOpenFile(fname, TD_FILE_READ); if (pFile == NULL) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + TSDB_CHECK_CODE(code = terrno, lino, _exit); } code = taosFStatFile(pFile, &size, NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index e1ac42ca8c..edf6c5a814 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -267,7 +267,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ); if (NULL == pFile) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 15350bdfef..8dfc08f26e 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -204,7 +204,7 @@ static FORCE_INLINE int32_t initOpenCacheFile(SGroupCacheFileFd* pFileFd, char* TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE|TD_FILE_AUTO_DEL); //TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE); if (NULL == newFd) { - QRY_ERR_RET(TAOS_SYSTEM_ERROR(errno)); + QRY_ERR_RET(terrno); } pFileFd->fd = newFd; int32_t code = taosThreadMutexInit(&pFileFd->mutex, NULL); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index b17af44260..6ae718c101 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -994,7 +994,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) { TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (file == NULL) { - fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); + fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(terrno)); return TSDB_CODE_FILE_CORRUPTED; } int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9412735ded..51ecc67ec9 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2268,7 +2268,7 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } pStmt->fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); if (NULL == pStmt->fp) { - return TAOS_SYSTEM_ERROR(errno); + return terrno; } return parseDataFromFileImpl(pCxt, pStmt, rowsDataCxt); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 62943cb6d5..b084fd8c5d 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -11374,7 +11374,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) static int32_t readFromFile(char* pName, int32_t* len, char** buf) { int64_t filesize = 0; if (taosStatFile(pName, &filesize, NULL, NULL) < 0) { - return TAOS_SYSTEM_ERROR(errno); + return terrno; } *len = filesize; @@ -11391,7 +11391,7 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { TdFilePtr tfile = taosOpenFile(pName, O_RDONLY | O_BINARY); if (NULL == tfile) { taosMemoryFreeClear(*buf); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } int64_t s = taosReadFile(tfile, *buf, *len); @@ -14070,7 +14070,7 @@ static int32_t prepareReadCsvFile(STranslateContext* pCxt, SCreateSubTableFromFi if (NULL == pModifyStmt->fp) { fp = taosOpenFile(pCreateStmt->filePath, TD_FILE_READ | TD_FILE_STREAM); if (NULL == fp) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _ERR; } } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 993d9d91af..a177db688d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -290,7 +290,7 @@ int32_t remoteChkp_readMetaData(char* path, SSChkpMetaOnS3** pMeta) { pFile = taosOpenFile(path, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _EXIT; } @@ -357,7 +357,7 @@ int32_t remoteChkp_validAndCvtMeta(char* path, SSChkpMetaOnS3* pMeta, int64_t ch } if (taosStatFile(src, NULL, NULL, NULL) != 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; goto _EXIT; } @@ -1603,7 +1603,7 @@ int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId) // compatible with previous version *processId = -1; code = 0; - stWarn("failed to open file to load extra info, file:%s, reason:%s", pDst, tstrerror(TAOS_SYSTEM_ERROR(errno))); + stWarn("failed to open file to load extra info, file:%s, reason:%s", pDst, tstrerror(terrno)); goto _EXIT; } @@ -1656,7 +1656,7 @@ int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId) { pFile = taosOpenFile(pDst, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("failed to open file to add extra info, file:%s, reason:%s", pDst, tstrerror(code)); goto _EXIT; } @@ -4940,7 +4940,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { } if (taosCopyFile(srcBuf, dstBuf) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("failed to copy file from %s to %s, reason:%s", srcBuf, dstBuf, tstrerror(code)); goto _ERROR; } @@ -4953,7 +4953,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { TdFilePtr pFile = taosOpenFile(dstDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("chkp failed to create meta file: %s, reason:%s", dstDir, tstrerror(code)); goto _ERROR; } diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 2b69da6007..4e804f9ca7 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -556,9 +556,9 @@ _NEXT: uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); int64_t nread = taosPReadFile(pSnapFile->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pSnapFile->offset); - if (nread == -1) { + if (nread < 0) { taosMemoryFree(buf); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; stError("%s snap failed to read snap, file name:%s, type:%d,reason:%s", STREAM_STATE_TRANSFER, item->name, item->type, tstrerror(code)); return code; diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index c7811ed093..e8ed73b089 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -150,7 +150,7 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { - code = terrno ? terrno : TAOS_SYSTEM_ERROR(errno); + code = terrno; TAOS_CHECK_EXIT(code); } @@ -259,7 +259,7 @@ int32_t syncReadCfgFile(SSyncNode *pNode) { pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; sError("vgId:%d, failed to open sync cfg file:%s since %s", pNode->vgId, file, tstrerror(code)); goto _OVER; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index a4377858d8..3bb40286bc 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -54,7 +54,7 @@ int32_t raftStoreReadFile(SSyncNode *pNode) { if (pFile == NULL) { sError("vgId:%d, failed to open raft store file:%s since %s", pNode->vgId, file, terrstr()); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + TAOS_CHECK_GOTO(terrno, &lino, _OVER); } int64_t size = 0; @@ -127,15 +127,15 @@ int32_t raftStoreWriteFile(SSyncNode *pNode) { if (buffer == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _OVER); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); - if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + if (pFile == NULL) TAOS_CHECK_GOTO(terrno, &lino, _OVER); int32_t len = strlen(buffer); if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(terrno, &lino, _OVER); - if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(terrno, &lino, _OVER); (void)taosCloseFile(&pFile); - if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _OVER); + if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(terrno, &lino, _OVER); code = 0; sInfo("vgId:%d, succeed to write raft store file:%s, term:%" PRId64, pNode->vgId, realfile, pStore->currentTerm); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 3c144cae23..5f1ae02166 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -63,7 +63,7 @@ static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, in if (pFile == NULL) { wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); *lastVer = retVer; - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } // ensure size as non-negative @@ -223,8 +223,8 @@ static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, in lastEntryEndOffset, fileSize); if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { - wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + wError("failed to truncate file due to %s. file:%s", strerror(terrno), fnameStr); + TAOS_CHECK_GOTO(terrno, &lino, _err); } if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) { @@ -315,7 +315,7 @@ static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) { if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) { wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } if (updateMeta != NULL) *updateMeta = true; @@ -375,7 +375,7 @@ static int32_t walTrimIdxFile(SWal* pWal, int32_t fileIdx) { TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } wInfo("vgId:%d, trim idx file. file: %s, size: %" PRId64 ", offset: %" PRId64, pWal->cfg.vgId, fnameStr, fileSize, @@ -462,7 +462,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { if (code < 0) { wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { @@ -545,7 +545,7 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) { wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { @@ -565,14 +565,14 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { if (pIdxFile == NULL) { wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); if (pLogFile == NULL) { wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } // determine the last valid entry end, i.e. offset @@ -619,7 +619,7 @@ static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { wError("vgId:%d, failed to ftruncate file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset, fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } } @@ -941,7 +941,7 @@ int32_t walSaveMeta(SWal* pWal) { if (pMetaFile == NULL) { wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } char* serialized = NULL; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 73cf6d46ef..aed7db327f 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -175,7 +175,7 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) { if (pLogFile == NULL) { wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr()); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } pReader->pLogFile = pLogFile; @@ -185,7 +185,7 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) { if (pIdxFile == NULL) { wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr()); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } pReader->pIdxFile = pIdxFile; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 43d5c07270..224a7dc711 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -53,7 +53,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { wError("vgId:%d, restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); @@ -62,7 +62,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { wError("vgId:%d, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); } @@ -111,20 +111,20 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } } if (pWal->pIdxFile != NULL) { if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } } @@ -141,7 +141,7 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { if (pIdxTFile == NULL) { pWal->pIdxFile = NULL; - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } walBuildLogName(pWal, fileFirstVer, fnameStr); pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); @@ -149,7 +149,7 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { TAOS_UNUSED(taosCloseFile(&pIdxTFile)); pWal->pLogFile = NULL; - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } pWal->pLogFile = pLogTFile; @@ -200,7 +200,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if (pIdxFile == NULL) { TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); @@ -225,7 +225,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // TODO TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } code = taosLSeekFile(pLogFile, entry.offset, SEEK_SET); if (code < 0) { @@ -260,13 +260,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if (code < 0) { TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(code); } code = taosFtruncateFile(pIdxFile, idxOff); if (code < 0) { TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(code); } pWal->vers.lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; @@ -294,21 +294,21 @@ static int32_t walRollImpl(SWal *pWal) { if (pWal->pIdxFile != NULL) { if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } } if (pWal->pLogFile != NULL) { if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } } @@ -319,13 +319,13 @@ static int32_t walRollImpl(SWal *pWal) { walBuildIdxName(pWal, newFileFirstVer, fnameStr); pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pIdxFile == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } walBuildLogName(pWal, newFileFirstVer, fnameStr); pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr); if (pLogFile == NULL) { - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } TAOS_CHECK_GOTO(walRollFileInfo(pWal), &lino, _exit); @@ -672,7 +672,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy _exit: // recover in a reverse order if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); wFatal("vgId:%d, failed to recover WAL logfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId, terrstr(), offset); taosMsleep(100); @@ -681,7 +680,6 @@ _exit: int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); wFatal("vgId:%d, failed to recover WAL idxfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId, terrstr(), idxOffset); taosMsleep(100); @@ -700,12 +698,12 @@ static int32_t walInitWriteFile(SWal *pWal) { walBuildIdxName(pWal, fileFirstVer, fnameStr); pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pIdxTFile == NULL) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } walBuildLogName(pWal, fileFirstVer, fnameStr); pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pLogTFile == NULL) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } // switch file pWal->pIdxFile = pIdxTFile; @@ -755,7 +753,7 @@ int32_t walFsync(SWal *pWal, bool forceFsync) { if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; } } TAOS_UNUSED(taosThreadMutexUnlock(&pWal->mutex)); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 042be9c71f..a9cd099d54 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -554,7 +554,8 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { int32_t taosLockFile(TdFilePtr pFile) { if (pFile == NULL || pFile->hFile == NULL) { - return -1; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } BOOL fSuccess = FALSE; @@ -568,7 +569,7 @@ int32_t taosLockFile(TdFilePtr pFile) { &overlapped // overlapped structure ); if (!fSuccess) { - return GetLastError(); + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } return 0; } @@ -582,18 +583,20 @@ int32_t taosUnLockFile(TdFilePtr pFile) { fSuccess = UnlockFileEx(pFile->hFile, 0, ~0, ~0, &overlapped); if (!fSuccess) { - return GetLastError(); + return TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); } return 0; } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (pFile == NULL) { - return 0; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } if (pFile->hFile == NULL) { printf("Ftruncate file error, hFile was null\n"); - return -1; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } LARGE_INTEGER li_0; @@ -601,7 +604,8 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { BOOL cur = SetFilePointerEx(pFile->hFile, li_0, NULL, FILE_CURRENT); if (!cur) { printf("SetFilePointerEx Error getting current position in file.\n"); - return -1; + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; } LARGE_INTEGER li_size; @@ -609,7 +613,6 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { BOOL cur2 = SetFilePointerEx(pFile->hFile, li_size, NULL, FILE_BEGIN); if (cur2 == 0) { int error = GetLastError(); - printf("SetFilePointerEx GetLastError is: %d\n", error); switch (error) { case ERROR_INVALID_HANDLE: errno = EBADF; @@ -618,7 +621,8 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { errno = EIO; break; } - return -1; + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } if (!SetEndOfFile(pFile->hFile)) { @@ -632,23 +636,27 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { errno = EIO; break; } - return -1; + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } return 0; } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { if (pFileOut == NULL || pFileIn == NULL) { - return 0; + terrno = TSDB_CODE_INVALID_PARA; + return -1; } if (pFileIn->hFile == NULL || pFileOut->hFile == NULL) { - return 0; + terrno = TSDB_CODE_INVALID_PARA; + return -1; } LARGE_INTEGER fileOffset; fileOffset.QuadPart = *offset; if (!SetFilePointerEx(pFileIn->hFile, fileOffset, &fileOffset, FILE_BEGIN)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return -1; } @@ -659,6 +667,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in DWORD bytesWritten; for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { if (!ReadFile(pFileIn->hFile, buffer, _SEND_FILE_STEP_, &bytesRead, NULL)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return writeLen; } @@ -666,12 +675,14 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in return writeLen; } else if (bytesRead < _SEND_FILE_STEP_) { if (!WriteFile(pFileOut->hFile, buffer, bytesRead, &bytesWritten, NULL)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return -1; } else { return (int64_t)(writeLen + bytesRead); } } else { if (!WriteFile(pFileOut->hFile, buffer, _SEND_FILE_STEP_, &bytesWritten, NULL)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return -1; } else { writeLen += _SEND_FILE_STEP_; @@ -683,6 +694,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in if (remain > 0) { DWORD bytesRead; if (!ReadFile(pFileIn->hFile, buffer, (DWORD)remain, &bytesRead, NULL)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return -1; } @@ -691,6 +703,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } else { DWORD bytesWritten; if (!WriteFile(pFileOut->hFile, buffer, bytesRead, &bytesWritten, NULL)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return -1; } else { writeLen += bytesWritten; @@ -740,7 +753,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return -1; } int64_t leftbytes = count; @@ -763,7 +776,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif terrno = code; - return terrno; + return -1; } } else if (readbytes == 0) { #if FILE_WITH_LOCK @@ -926,32 +939,12 @@ int32_t taosLockFile(TdFilePtr pFile) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - -#ifdef WINDOWS - BOOL fSuccess = FALSE; - LARGE_INTEGER fileSize; - OVERLAPPED overlapped = {0}; - - HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); - - fSuccess = LockFileEx(hFile, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, - 0, // reserved - ~0, // number of bytes to lock low - ~0, // number of bytes to lock high - &overlapped // overlapped structure - ); - if (!fSuccess) { - return GetLastError(); - } - return 0; -#else int32_t code = (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return code; -#endif + return 0; } int32_t taosUnLockFile(TdFilePtr pFile) { @@ -959,25 +952,12 @@ int32_t taosUnLockFile(TdFilePtr pFile) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - -#ifdef WINDOWS - BOOL fSuccess = FALSE; - OVERLAPPED overlapped = {0}; - HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); - - fSuccess = UnlockFileEx(hFile, 0, ~0, ~0, &overlapped); - if (!fSuccess) { - return GetLastError(); - } - return 0; -#else int32_t code = (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return code; -#endif + return 0; } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { @@ -986,104 +966,29 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { return terrno; } -#ifdef WINDOWS - - HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); - - LARGE_INTEGER li_0; - li_0.QuadPart = (int64_t)0; - BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); - if (!cur) { - printf("SetFilePointerEx Error getting current position in file.\n"); - return -1; - } - - LARGE_INTEGER li_size; - li_size.QuadPart = l_size; - BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); - if (cur2 == 0) { - int error = GetLastError(); - printf("SetFilePointerEx GetLastError is: %d\n", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - if (!SetEndOfFile(h)) { - int error = GetLastError(); - printf("SetEndOfFile GetLastError is:%d", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - return 0; -#else int32_t code = ftruncate(pFile->fd, l_size); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return code; -#endif + return 0; } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { if (pFileOut == NULL || pFileIn == NULL) { terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return -1; } if (pFileIn->fd < 0 || pFileOut->fd < 0) { terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return -1; } -#ifdef WINDOWS - - _lseeki64(pFileIn->fd, *offset, 0); - int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = {0}; - - for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = _read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); - if (rlen <= 0) { - return writeLen; - } else if (rlen < _SEND_FILE_STEP_) { - write(pFileOut->fd, (void *)buffer, (uint32_t)rlen); - return (int64_t)(writeLen + rlen); - } else { - write(pFileOut->fd, (void *)buffer, _SEND_FILE_STEP_); - writeLen += _SEND_FILE_STEP_; - } +#ifdef _TD_DARWIN_64 + if(lseek(pFileIn->fd, (int32_t)(*offset), 0) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } - - int64_t remain = size - writeLen; - if (remain > 0) { - size_t rlen = _read(pFileIn->fd, (void *)buffer, (size_t)remain); - if (rlen <= 0) { - return writeLen; - } else { - write(pFileOut->fd, (void *)buffer, (uint32_t)remain); - writeLen += remain; - } - } - return writeLen; - -#elif defined(_TD_DARWIN_64) - - lseek(pFileIn->fd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; @@ -1112,7 +1017,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } return writeLen; -#else +#else // for linux int64_t leftbytes = size; int64_t sentbytes; @@ -1128,7 +1033,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in continue; } else { terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + return -1; } } else if (sentbytes == 0) { return (int64_t)(size - leftbytes); @@ -1223,6 +1128,7 @@ int32_t taosCloseFile(TdFilePtr *ppFile) { if ((*ppFile)->hFile != NULL) { // FlushFileBuffers((*ppFile)->hFile); if (!CloseHandle((*ppFile)->hFile)) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); code = -1; } (*ppFile)->hFile = NULL; @@ -1251,7 +1157,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) STUB_RAND_IO_ERR(terrno) if (pFile == NULL) { terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return -1; } int32_t code = 0; @@ -1267,7 +1173,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #endif terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return -1; } DWORD ret = 0; @@ -1278,7 +1184,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) SetLastError(0); BOOL result = ReadFile(pFile->hFile, buf, count, &ret, &ol); if (!result && GetLastError() != ERROR_HANDLE_EOF) { - errno = GetLastError(); + code = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); ret = -1; } #else @@ -1291,7 +1197,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return -1; } int64_t ret = pread(pFile->fd, buf, count, offset); if (-1 == ret) { @@ -1304,10 +1210,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) if (code) { terrno = code; - return code; + return -1; } - return ret; + return 0; } int32_t taosFsyncFile(TdFilePtr pFile) { @@ -1333,7 +1239,12 @@ int32_t taosFsyncFile(TdFilePtr pFile) { if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) { return 0; } - return !FlushFileBuffers(pFile->hFile); + bool ret = FlushFileBuffers(pFile->hFile); + if (!ret) { + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); + return terrno; + } + return 0; } #else if (pFile->fd >= 0) { @@ -1397,7 +1308,9 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { #ifdef WINDOWS size_t bufferSize = 512; *ptrBuf = taosMemoryMalloc(bufferSize); - if (*ptrBuf == NULL) goto END; + if (*ptrBuf == NULL) { + goto END; + } size_t bytesRead = 0; size_t totalBytesRead = 0; @@ -1405,8 +1318,14 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { while (1) { char *result = fgets(*ptrBuf + totalBytesRead, bufferSize - totalBytesRead, pFile->fp); if (result == NULL) { - taosMemoryFreeClear(*ptrBuf); - goto END; + if (feof(pFile->fp)) { + break; + } else { + ret = -1; + terrno = TAOS_SYSTEM_ERROR(ferror(pFile->fp)); + taosMemoryFreeClear(*ptrBuf); + goto END; + } } bytesRead = strlen(*ptrBuf + totalBytesRead); totalBytesRead += bytesRead; @@ -1431,7 +1350,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { size_t len = 0; ret = getline(ptrBuf, &len, pFile->fp); if (-1 == ret) { - code = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); } #endif @@ -1441,10 +1360,6 @@ END: (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif - if (code) { - terrno = code; - } - return ret; } @@ -1636,8 +1551,7 @@ int taosSetAutoDelFile(char *path) { if (succ) { return 0; } else { - DWORD error = GetLastError(); - terrno = TAOS_SYSTEM_ERROR(error); + terrno = TAOS_SYSTEM_WINAPI_ERROR(GetLastError()); return terrno; } #else diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 48fe271d33..bcbf09abdc 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -136,7 +136,7 @@ static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { char fullDir[PATH_MAX] = {0}; if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { - int32_t code = TAOS_SYSTEM_ERROR(errno); + int32_t code = terrno; uError("failed to expand dir:%s since %s", inputDir, tstrerror(code)); TAOS_RETURN(code); } @@ -1009,7 +1009,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + TAOS_RETURN(terrno); } while (!taosEOFFile(pFile)) { @@ -1066,7 +1066,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { // success when the file does not exist - code = TAOS_SYSTEM_ERROR(errno); + code = terrno; if (errno == ENOENT) { uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, tstrerror(code)); TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -1252,7 +1252,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) { - TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); + TAOS_CHECK_EXIT(terrno); } size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END); if(fileSize <= 0) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 98e17c5db9..bde3d48a6d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -207,7 +207,7 @@ int32_t taosInitSlowLog() { tsLogObj.slowHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (tsLogObj.slowHandle->pFile == NULL) { (void)printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } return 0; @@ -539,7 +539,7 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { tsLogObj.logHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE); if (tsLogObj.logHandle->pFile == NULL) { (void)printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } (void)taosLockLogFile(tsLogObj.logHandle->pFile); @@ -973,7 +973,6 @@ void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, vo pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); taosPrintLog(flags, level, dflag, "failed to open file:%s since %s", filepath, terrstr()); goto _return; } @@ -1029,11 +1028,10 @@ void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr if (NULL == *pFd) { int64_t filesize = 0; if (taosStatFile(filepath, &filesize, NULL, NULL) < 0) { - if (ENOENT == errno) { + if (TAOS_SYSTEM_ERROR(ENOENT) == terrno) { return; } - terrno = TAOS_SYSTEM_ERROR(errno); taosPrintLog(flags, level, dflag, "failed to stat file:%s since %s", filepath, terrstr()); return; } @@ -1048,7 +1046,6 @@ void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr return; } - terrno = TAOS_SYSTEM_ERROR(errno); taosPrintLog(flags, level, dflag, "failed to open file:%s since %s", filepath, terrstr()); return; } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 265f06cbfd..534da9cd77 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -64,7 +64,7 @@ static int32_t createDiskFile(SDiskbasedBuf* pBuf) { pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (pBuf->pFile == NULL) { - return TAOS_SYSTEM_ERROR(errno); + return terrno; } return TSDB_CODE_SUCCESS; From 2fa083c82feb626f4ec261d50f73b8ed0f17192a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 10 Sep 2024 19:03:27 +0800 Subject: [PATCH 11/32] opt transport --- source/libs/transport/src/transCli.c | 1 - source/libs/transport/src/transSvr.c | 16 +++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 803e50ce94..a0c7315fe7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2426,7 +2426,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(timer); } - (void)uv_loop_close(pThrd->loop); taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 38c7edeeaf..3a597fe6ad 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -188,11 +188,13 @@ static void sendQuitToWorkThrd(SWorkThrd* pThrd); static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); -#define SRV_RELEASE_UV(loop) \ - do { \ - (void)uv_walk(loop, uvWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ +#define SRV_RELEASE_UV(loop) \ + do { \ + if (loop && uv_loop_alive(loop)) { \ + (void)uv_walk(loop, uvWalkCb, NULL); \ + (void)uv_run(loop, UV_RUN_DEFAULT); \ + (void)uv_loop_close(loop); \ + } \ } while (0); #define ASYNC_ERR_JRET(thrd) \ @@ -1615,7 +1617,6 @@ void destroyWorkThrdObj(SWorkThrd* pThrd) { } transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); - (void)uv_loop_close(pThrd->loop); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } @@ -1648,14 +1649,11 @@ void transCloseServer(void* arg) { (void)taosThreadJoin(srv->thread, NULL); SRV_RELEASE_UV(srv->loop); - (void)uv_loop_close(srv->loop); - for (int i = 0; i < srv->numOfThreads; i++) { destroyWorkThrd(srv->pThreadObj[i]); } } else { SRV_RELEASE_UV(srv->loop); - (void)uv_loop_close(srv->loop); } taosMemoryFree(srv->pThreadObj); From 79de2c52f4990712314d5764a9cc6e85fb8dc356 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 10 Sep 2024 19:36:20 +0800 Subject: [PATCH 12/32] fix: return length --- source/os/src/osFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index a9cd099d54..94ead5c129 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1213,7 +1213,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) return -1; } - return 0; + return ret; } int32_t taosFsyncFile(TdFilePtr pFile) { From 25c0b928854a9ee958e0fb83d2bc82fddf796b1b Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 01:11:56 +0000 Subject: [PATCH 13/32] feat:Optimising the duration and keep --- include/libs/command/command.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 20 +++++--- source/libs/command/src/command.c | 52 ++++++--------------- tests/script/tsim/db/create_all_options.sim | 4 +- 4 files changed, 32 insertions(+), 45 deletions(-) diff --git a/include/libs/command/command.h b/include/libs/command/command.h index b788b03386..284f54e5ae 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -29,5 +29,6 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp); int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp); void qExplainFreeCtx(SExplainCtx *pCtx); +int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes); #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 20c5d09c9e..9c02fd6e5d 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -34,6 +34,7 @@ #include "systable.h" #include "thttp.h" #include "tjson.h" +#include "command.h" #define DB_VER_NUMBER 1 #define DB_RESERVE_SIZE 27 @@ -2321,18 +2322,25 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, (void)colDataSetVal(pColInfo, rows, (const char *)strictVstr, false); char durationVstr[128] = {0}; - int32_t len = sprintf(&durationVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.daysPerFile); + int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], pDb->cfg.daysPerFile); + varDataSetLen(durationVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); (void)colDataSetVal(pColInfo, rows, (const char *)durationVstr, false); - char keepVstr[128] = {0}; + char keepVstr[512] = {0}; + char keep0Str[128] = {0}; + char keep1Str[128] = {0}; + char keep2Str[128] = {0}; + + formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0); + formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1); + formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2); + if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, - pDb->cfg.daysToKeep0); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); } else { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, - pDb->cfg.daysToKeep2); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep0Str, keep1Str, keep2Str); } varDataSetLen(keepVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 138782495f..0e5f2f4cbd 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -344,27 +344,18 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { return TSDB_CACHE_MODEL_NONE_STR; } -static int32_t formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { +int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes) { int len = 0; if (timeInMinutes % 1440 == 0) { int days = timeInMinutes / 1440; - len = snprintf(NULL, 0, "%dd", days); - *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); - if(*buffer == NULL) return terrno; - sprintf(*buffer, "%dd", days); + len = sprintf(buffer, "%dd", days); } else if (timeInMinutes % 60 == 0) { int hours = timeInMinutes / 60; - len = snprintf(NULL, 0, "%dh", hours); - *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); - if(*buffer == NULL) return terrno; - sprintf(*buffer, "%dh", hours); + len = sprintf(buffer, "%dh", hours); } else { - len = snprintf(NULL, 0, "%dm", timeInMinutes); - *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); - if(*buffer == NULL) return terrno; - sprintf(*buffer, "%dm", timeInMinutes); + len = sprintf(buffer, "%dm", timeInMinutes); } - return TSDB_CODE_SUCCESS; + return len; } static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { @@ -404,25 +395,16 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, } else if (pCfg->hashPrefix < 0) { hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; } - char* durationStr = NULL; - char* keep0Str = NULL; - char* keep1Str = NULL; - char* keep2Str = NULL; - int32_t codeDuration = formatDurationOrKeep(&durationStr, pCfg->daysPerFile); - int32_t codeKeep0 = formatDurationOrKeep(&keep0Str, pCfg->daysToKeep0); - int32_t codeKeep1 = formatDurationOrKeep(&keep1Str, pCfg->daysToKeep1); - int32_t codeKeep2 = formatDurationOrKeep(&keep2Str, pCfg->daysToKeep2); - if(codeDuration != TSDB_CODE_SUCCESS || codeKeep0 != TSDB_CODE_SUCCESS || codeKeep1 != TSDB_CODE_SUCCESS || codeKeep2 != TSDB_CODE_SUCCESS) { - int32_t firstErrorCode = codeDuration != TSDB_CODE_SUCCESS ? codeDuration : - codeKeep0 != TSDB_CODE_SUCCESS ? codeKeep0 : - codeKeep1 != TSDB_CODE_SUCCESS ? codeKeep1 : codeKeep2; - taosMemoryFree(pRetentions); - taosMemoryFree(durationStr); - taosMemoryFree(keep0Str); - taosMemoryFree(keep1Str); - taosMemoryFree(keep2Str); - return firstErrorCode; - } + char durationStr[128] = {0}; + char keep0Str[128] = {0}; + char keep1Str[128] = {0}; + char keep2Str[128] = {0}; + + int32_t lenDuration = formatDurationOrKeep(durationStr, pCfg->daysPerFile); + int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pCfg->daysToKeep0); + int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pCfg->daysToKeep1); + int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pCfg->daysToKeep2); + if (IS_SYS_DBNAME(dbName)) { len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); } else { @@ -449,10 +431,6 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, } taosMemoryFree(pRetentions); - taosMemoryFree(durationStr); - taosMemoryFree(keep0Str); - taosMemoryFree(keep1Str); - taosMemoryFree(keep2Str); (varDataLen(buf2)) = len; diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index e4f29cc74e..e402223d93 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -92,10 +92,10 @@ endi if $data5_db != on then # strict return -1 endi -if $data6_db != 14400m then # duration +if $data6_db != 10d then # duration return -1 endi -if $data7_db != 5256000m,5256000m,5256000m then # keep +if $data7_db != 3650d,3650d,3650d then # keep return -1 endi if $data8_db != 256 then # buffer From fede52d5b7743ec553d10eaac72af5d694f6568d Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 10 Sep 2024 17:48:47 +0800 Subject: [PATCH 14/32] enh: (last) rm rwritebatch --- source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/tsdb/tsdbCache.c | 31 +++++++++++-------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 59d70889e5..c85c376c47 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -342,7 +342,6 @@ typedef struct { rocksdb_writeoptions_t *writeoptions; rocksdb_readoptions_t *readoptions; rocksdb_writebatch_t *writebatch; - rocksdb_writebatch_t *rwritebatch; STSchema *pTSchema; } SRocksCache; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3a3e20612e..f4a423bfa7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -214,10 +214,8 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { } rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); - rocksdb_writebatch_t *rwritebatch = rocksdb_writebatch_create(); pTsdb->rCache.writebatch = writebatch; - pTsdb->rCache.rwritebatch = rwritebatch; pTsdb->rCache.my_comparator = cmp; pTsdb->rCache.options = options; pTsdb->rCache.writeoptions = writeoptions; @@ -248,7 +246,6 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { rocksdb_close(pTsdb->rCache.db); rocksdb_flushoptions_destroy(pTsdb->rCache.flushoptions); rocksdb_writebatch_destroy(pTsdb->rCache.writebatch); - rocksdb_writebatch_destroy(pTsdb->rCache.rwritebatch); rocksdb_readoptions_destroy(pTsdb->rCache.readoptions); rocksdb_writeoptions_destroy(pTsdb->rCache.writeoptions); rocksdb_options_destroy(pTsdb->rCache.options); @@ -258,8 +255,8 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { taosMemoryFree(pTsdb->rCache.pTSchema); } -static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) { - rocksdb_writebatch_t *wb = read ? pTsdb->rCache.rwritebatch : pTsdb->rCache.writebatch; +static void rocksMayWrite(STsdb *pTsdb, bool force) { + rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; int count = rocksdb_writebatch_count(wb); if ((force && count > 0) || count >= ROCKS_BATCH_SIZE) { @@ -516,8 +513,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); - rocksMayWrite(pTsdb, true, false); - rocksMayWrite(pTsdb, true, true); + rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -667,8 +663,7 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); - rocksMayWrite(pTsdb, true, false); - rocksMayWrite(pTsdb, true, true); + rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); if (NULL != err) { @@ -862,7 +857,7 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra taosMemoryFree(pTSchema); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -903,7 +898,7 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { taosMemoryFree(pTSchema); - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -934,7 +929,7 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -973,7 +968,7 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1243,7 +1238,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosMemoryFreeClear(pToFree); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1582,7 +1577,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } // store result back to rocks cache - wb = pTsdb->rCache.rwritebatch; + wb = pTsdb->rCache.writebatch; char *value = NULL; size_t vlen = 0; code = tsdbCacheSerialize(pLastCol, &value, &vlen); @@ -1597,7 +1592,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } if (wb) { - rocksMayWrite(pTsdb, false, true); + rocksMayWrite(pTsdb, false); } _exit: @@ -1797,6 +1792,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache if (remainCols && TARRAY_SIZE(remainCols) > 0) { (void)taosThreadMutexLock(&pTsdb->lruMutex); + rocksMayWrite(pTsdb, false); + for (int i = 0; i < TARRAY_SIZE(remainCols);) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; LRUHandle *h = taosLRUCacheLookup(pCache, &idxKey->key, ROCKS_KEY_LEN); @@ -1944,7 +1941,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE taosMemoryFreeClear(pLastCol); } - rocksMayWrite(pTsdb, true, false); + rocksMayWrite(pTsdb, true); _exit: (void)taosThreadMutexUnlock(&pTsdb->lruMutex); From cffa2a496ced1d33c425f36ea5a7577b101cbca9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 09:37:09 +0800 Subject: [PATCH 15/32] fix mem leak --- source/libs/transport/src/transSvr.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3a597fe6ad..48042b84c1 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -188,13 +188,11 @@ static void sendQuitToWorkThrd(SWorkThrd* pThrd); static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); -#define SRV_RELEASE_UV(loop) \ - do { \ - if (loop && uv_loop_alive(loop)) { \ - (void)uv_walk(loop, uvWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ - } \ +#define SRV_RELEASE_UV(loop) \ + do { \ + (void)uv_walk(loop, uvWalkCb, NULL); \ + (void)uv_run(loop, UV_RUN_DEFAULT); \ + (void)uv_loop_close(loop); \ } while (0); #define ASYNC_ERR_JRET(thrd) \ From abbcc4afa087dec2984e5c5b90987ddc81d4b472 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 11 Sep 2024 03:24:59 +0000 Subject: [PATCH 16/32] fix/TS-5404-use-temp-transid-when-timeout --- source/dnode/mnode/impl/src/mndTrans.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 63cab4168a..18ceddb7d0 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -64,6 +64,8 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); +int32_t tsMaxTransId = 0; + int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_TRANS, @@ -602,7 +604,9 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, tstrncpy(pTrans->opername, opername, TSDB_TRANS_OPER_LEN); } - pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); + int32_t sdbMaxId = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); + int32_t oldId = atomic_load_32(&tsMaxTransId); + pTrans->id = TMAX(sdbMaxId, oldId); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->conflict = conflict; @@ -1027,6 +1031,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); + atomic_store_32(&tsMaxTransId, pTrans->id); TAOS_RETURN(code); } mInfo("trans:%d, prepare finished", pTrans->id); From 44a955218cfcedb9b9445e058fcb3a87e4177c95 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 13:48:23 +0800 Subject: [PATCH 17/32] fix: (last) call rocksdb_write before rocksdb_multi_get --- source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/tsdb/tsdbCache.c | 111 +++++++++--------------- 2 files changed, 40 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c85c376c47..d4a33a5478 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -362,7 +362,6 @@ struct STsdb { SMemTable *imem; STsdbFS fs; // old SLRUCache *lruCache; - SCacheFlushState flushState; TdThreadMutex lruMutex; SLRUCache *biCache; TdThreadMutex biMutex; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f4a423bfa7..53e20d459f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -267,7 +267,6 @@ static void rocksMayWrite(STsdb *pTsdb, bool force) { tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, count, err); rocksdb_free(err); - // pTsdb->flushState.flush_count = 0; } rocksdb_writebatch_clear(wb); @@ -456,47 +455,23 @@ static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size TAOS_RETURN(TSDB_CODE_SUCCESS); } -static void tsdbCachePutBatch(SLastCol *pLastCol, const void *key, size_t klen, SCacheFlushState *state) { - int32_t code = 0; - STsdb *pTsdb = state->pTsdb; - SRocksCache *rCache = &pTsdb->rCache; - rocksdb_writebatch_t *wb = rCache->writebatch; - char *rocks_value = NULL; - size_t vlen = 0; - - code = tsdbCacheSerialize(pLastCol, &rocks_value, &vlen); - if (code) { - tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); - return; - } - - rocksdb_writebatch_put(wb, (char *)key, klen, rocks_value, vlen); - - taosMemoryFree(rocks_value); - - if (++state->flush_count >= ROCKS_BATCH_SIZE) { - char *err = NULL; - - rocksdb_write(rCache->db, rCache->writeoptions, wb, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - state->flush_count, err); - rocksdb_free(err); - } - - rocksdb_writebatch_clear(wb); - - state->flush_count = 0; - } -} +static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol); int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + STsdb *pTsdb = (STsdb *)ud; + + int32_t code = tsdbCachePutToRocksdb(pTsdb, (SLastKey *)key, pLastCol); + if (code) { + tsdbError("tsdb/cache: vgId:%d, flush dirty lru failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; + } pLastCol->dirty = 0; + + rocksMayWrite(pTsdb, false); } return 0; @@ -511,7 +486,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { (void)taosThreadMutexLock(&pTsdb->lruMutex); - taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); + taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); @@ -602,7 +577,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + tsdbCacheFlushDirty(key, klen, pLastCol, ud); } for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { @@ -639,11 +614,11 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i SLastKey *pLastKey = &(SLastKey){.lflag = lflag, .uid = uid, .cid = cid}; LRUStatus status = taosLRUCacheInsert(pCache, pLastKey, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + TAOS_LRU_PRIORITY_LOW, pTsdb); if (status != TAOS_LRU_STATUS_OK) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); code = TSDB_CODE_FAILED; + pLastCol = NULL; } _exit: @@ -661,7 +636,7 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->lruCache; rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; - taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); + taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); @@ -733,6 +708,10 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, char **values_list = NULL; size_t *values_list_sizes = NULL; + + // was written by caller + // rocksMayWrite(pTsdb, true); // flush writebatch cache + TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, 2, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), NULL, _exit); @@ -857,7 +836,7 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra taosMemoryFree(pTSchema); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -898,7 +877,7 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { taosMemoryFree(pTSchema); - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -929,7 +908,7 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -968,7 +947,7 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool (void)tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1078,11 +1057,11 @@ static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLa TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(pLRULastCol, &charge)); LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, pLastKey, ROCKS_KEY_LEN, pLRULastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLRULastCol); code = TSDB_CODE_FAILED; + pLRULastCol = NULL; } _exit: @@ -1172,6 +1151,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray keys_list_sizes[i] = ROCKS_KEY_LEN; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + code = tsdbCacheGetValuesFromRocks(pTsdb, num_keys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes); if (code) { @@ -1238,7 +1219,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray taosMemoryFreeClear(pToFree); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1568,32 +1549,22 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); - taosMemoryFree(pLastCol); + pLastCol = NULL; TAOS_CHECK_EXIT(TSDB_CODE_FAILED); } // store result back to rocks cache - wb = pTsdb->rCache.writebatch; - char *value = NULL; - size_t vlen = 0; - code = tsdbCacheSerialize(pLastCol, &value, &vlen); + code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, pLastCol); if (code) { - tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); - } else { - SLastKey *key = &idxKey->key; - size_t klen = ROCKS_KEY_LEN; - rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); - taosMemoryFree(value); + tsdbError("vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + TAOS_CHECK_EXIT(code); } } - if (wb) { - rocksMayWrite(pTsdb, false); - } + rocksMayWrite(pTsdb, false); _exit: taosArrayDestroy(lastrowTmpIndexArray); @@ -1633,6 +1604,8 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA keys_list_sizes[i] = ROCKS_KEY_LEN; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + code = tsdbCacheGetValuesFromRocks(pTsdb, num_keys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes); if (code) { @@ -1681,11 +1654,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA } LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - tsdbCacheFreeSLastColItem(pLastCol); - taosMemoryFreeClear(pLastCol); taosMemoryFreeClear(pToFree); TAOS_CHECK_EXIT(TSDB_CODE_FAILED); } @@ -1792,7 +1763,6 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache if (remainCols && TARRAY_SIZE(remainCols) > 0) { (void)taosThreadMutexLock(&pTsdb->lruMutex); - rocksMayWrite(pTsdb, false); for (int i = 0; i < TARRAY_SIZE(remainCols);) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; @@ -1907,6 +1877,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE keys_list_sizes[i] = klen; } + rocksMayWrite(pTsdb, true); // flush writebatch cache + TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, numKeys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), NULL, _exit); @@ -1941,7 +1913,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE taosMemoryFreeClear(pLastCol); } - rocksMayWrite(pTsdb, true); + rocksMayWrite(pTsdb, false); _exit: (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -1983,9 +1955,6 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { (void)taosThreadMutexInit(&pTsdb->lruMutex, NULL); - pTsdb->flushState.pTsdb = pTsdb; - pTsdb->flushState.flush_count = 0; - _err: if (code) { tsdbError("tsdb/cache: vgId:%d, open failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); From 64bb49e0dbfd2a3087ccdfe54fb28e4af6528e71 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Mon, 9 Sep 2024 18:44:16 +0800 Subject: [PATCH 18/32] fix:[TS-5839] Fix crash when use last function on a column with all null value with partition subclause. --- include/common/tcommon.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 65 +++++++++++++++++++------ tests/script/tsim/query/cache_last.sim | 2 +- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5714990dd5..753f84d774 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -110,6 +110,8 @@ typedef struct SFirstLastRes { int32_t pkBytes; int8_t pkType; STuplePos pos; + STuplePos nullTuplePos; + bool nullTupleSaved; char buf[]; } SFirstLastRes; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 16f31321ce..41e2cadace 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -152,6 +152,7 @@ int32_t getIrateInfoSize(int32_t pkBytes); int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t firstFunction(SqlFunctionCtx* pCtx); int32_t firstFunctionMerge(SqlFunctionCtx* pCtx); int32_t lastFunction(SqlFunctionCtx* pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 3ec63a6d34..976d15e7d8 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -3469,7 +3469,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLast, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, - .initFunc = functionSetup, + .initFunc = firstLastFunctionSetup, .processFunc = lastFunction, .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6d655528c7..a84e8b66ad 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2617,6 +2617,22 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } +int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (pResInfo->initialized) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResInfo)) { + return TSDB_CODE_FUNC_SETUP_ERROR; + } + + SFirstLastRes * pRes = GET_ROWCELL_INTERBUF(pResInfo); + SInputColumnInfoData* pInput = &pCtx->input; + + pRes->nullTupleSaved = false; + pRes->nullTuplePos.pageId = -1; + return TSDB_CODE_SUCCESS; +} + static int32_t prepareBuf(SqlFunctionCtx* pCtx) { if (pCtx->subsidiaries.rowLen == 0) { int32_t rowLen = 0; @@ -2635,7 +2651,7 @@ static int32_t prepareBuf(SqlFunctionCtx* pCtx) { } static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, - SFirstLastRes* pInfo) { + SFirstLastRes* pInfo, bool noElements) { int32_t code = TSDB_CODE_SUCCESS; if (pCtx->subsidiaries.num <= 0) { @@ -2643,7 +2659,7 @@ static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowI } if (!pInfo->hasResult) { - code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + code = saveTupleData(pCtx, rowIndex, pSrcBlock, noElements ? &pInfo->nullTuplePos : &pInfo->pos); } else { code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); } @@ -2669,7 +2685,7 @@ static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t } pInfo->ts = currentTs; - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo, false); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2708,10 +2724,11 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; return TSDB_CODE_SUCCESS; } @@ -2802,10 +2819,11 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (numOfElems == 0) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2841,10 +2859,11 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; return TSDB_CODE_SUCCESS; } @@ -2983,10 +3002,11 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { // save selectivity value for column consisted of all null values if (numOfElems == 0) { - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); if (code != TSDB_CODE_SUCCESS) { return code; } + pInfo->nullTupleSaved = true; } return TSDB_CODE_SUCCESS; @@ -3031,7 +3051,7 @@ static bool firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOut static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, int32_t rowIndex) { if (firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput, pOutput->nullTupleSaved); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -3079,6 +3099,14 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer } } + if (numOfElems == 0) { + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pInfo->nullTupleSaved = true; + } + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -3099,6 +3127,11 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + + if (pResInfo->isNullRes) { + colDataSetNULL(pCol, pBlock->info.rows); + return setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); + } code = colDataSetVal(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); if (TSDB_CODE_SUCCESS != code) { return code; @@ -3134,12 +3167,16 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return TSDB_CODE_OUT_OF_RANGE; } - code = colDataSetVal(pCol, pBlock->info.rows, res, false); - if (TSDB_CODE_SUCCESS != code) { - return TSDB_CODE_OUT_OF_MEMORY; + if (pEntryInfo->numOfRes == 0) { + colDataSetNULL(pCol, pBlock->info.rows); + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); + } else { + code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + return TSDB_CODE_OUT_OF_MEMORY; + } + code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); } - code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); - taosMemoryFree(res); return code; } @@ -3185,7 +3222,7 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex pInfo->pkData = pInfo->buf + pInfo->bytes; } pInfo->ts = cts; - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo, false); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/tests/script/tsim/query/cache_last.sim b/tests/script/tsim/query/cache_last.sim index a06e6e1e6f..0a30bbd325 100644 --- a/tests/script/tsim/query/cache_last.sim +++ b/tests/script/tsim/query/cache_last.sim @@ -89,7 +89,7 @@ if $data10 != @ -> Merge (columns=3 width=24 input_order=unknown output_order= return -1 endi sql explain select count(*), last_row(f1), min(f1) from sta interval(1s); -if $data10 != @ -> Merge (columns=4 width=82 input_order=asc output_order=asc mode=sort)@ then +if $data10 != @ -> Merge (columns=4 width=106 input_order=asc output_order=asc mode=sort)@ then return -1 endi sql explain select distinct count(*), last_row(f1), min(f1) from tba1; From f9126f121187ddc3fabfbd1e876106ad4f72a492 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 14:11:51 +0800 Subject: [PATCH 19/32] enh:modify error code passing --- source/libs/sync/src/syncPipeline.c | 2 +- source/libs/sync/src/syncRespMgr.c | 2 +- source/libs/sync/src/syncSnapshot.c | 10 +++--- source/libs/tfs/src/tfs.c | 4 +-- source/libs/tfs/src/tfsDisk.c | 2 +- source/libs/transport/src/thttp.c | 4 +-- source/libs/transport/src/transCli.c | 52 +++++++++++++-------------- source/libs/transport/src/transComm.c | 16 ++++----- source/libs/transport/src/transSvr.c | 16 ++++----- source/util/src/talgo.c | 6 ++-- source/util/src/tbase58.c | 8 ++--- source/util/src/tbloomfilter.c | 6 ++-- source/util/src/tconfig.c | 4 +-- source/util/src/tgeosctx.c | 2 +- source/util/src/tjson.c | 2 +- source/util/src/tlist.c | 2 +- source/util/src/tlrucache.c | 2 +- source/util/src/tqueue.c | 8 ++--- source/util/src/tref.c | 6 ++-- source/util/src/tscalablebf.c | 4 +-- source/util/src/tworker.c | 10 +++--- 21 files changed, 84 insertions(+), 84 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index a6e9c7de32..86d4d83d29 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -1279,7 +1279,7 @@ int32_t syncLogBufferCreate(SSyncLogBuffer** ppBuf) { int32_t code = 0; SSyncLogBuffer* pBuf = taosMemoryCalloc(1, sizeof(SSyncLogBuffer)); if (pBuf == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exit); + TAOS_CHECK_GOTO(terrno, NULL, _exit); } pBuf->size = sizeof(pBuf->entries) / sizeof(pBuf->entries[0]); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index aa7a6da0a2..854d6bc314 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -25,7 +25,7 @@ int32_t syncRespMgrCreate(void *data, int64_t ttl, SSyncRespMgr **ppObj) { *ppObj = NULL; if ((pObj = taosMemoryCalloc(1, sizeof(SSyncRespMgr))) == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pObj->pRespHash = diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 2a3b164f03..0b77ab78a6 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -54,7 +54,7 @@ static int32_t syncSnapBufferCreate(SSyncSnapBuffer **ppBuf) { SSyncSnapBuffer *pBuf = taosMemoryCalloc(1, sizeof(SSyncSnapBuffer)); if (pBuf == NULL) { *ppBuf = NULL; - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pBuf->size = sizeof(pBuf->entries) / sizeof(void *); if (pBuf->size != TSDB_SYNC_SNAP_BUFFER_SIZE) return TSDB_CODE_SYN_INTERNAL_ERROR; @@ -74,7 +74,7 @@ int32_t snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex, SSyncSn SSyncSnapshotSender *pSender = taosMemoryCalloc(1, sizeof(SSyncSnapshotSender)); if (pSender == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pSender->start = false; @@ -285,7 +285,7 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (pSender->seq > SYNC_SNAPSHOT_SEQ_BEGIN) { pBlk = taosMemoryCalloc(1, sizeof(SyncSnapBlock)); if (pBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _OUT; } @@ -422,7 +422,7 @@ int32_t snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId, SSyncSnapsh SSyncSnapshotReceiver *pReceiver = taosMemoryCalloc(1, sizeof(SSyncSnapshotReceiver)); if (pReceiver == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pReceiver->start = false; @@ -697,7 +697,7 @@ static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshot // copy snap info from leader void *data = taosMemoryCalloc(1, pMsg->dataLen); if (data == NULL) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } pInfo->data = data; data = NULL; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 4954e81837..9727256a58 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -36,7 +36,7 @@ int32_t tfsOpen(SDiskCfg *pCfg, int32_t ndisk, STfs **ppTfs) { pTfs = taosMemoryCalloc(1, sizeof(STfs)); if (pTfs == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (taosThreadSpinInit(&pTfs->lock, 0) != 0) { @@ -429,7 +429,7 @@ int32_t tfsOpendir(STfs *pTfs, const char *rname, STfsDir **ppDir) { int32_t code = 0; STfsDir *pDir = taosMemoryCalloc(1, sizeof(STfsDir)); if (pDir == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exit); + TAOS_CHECK_GOTO(terrno, NULL, _exit); } SDiskID diskId = {.id = 0, .level = 0}; diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 07fa8d79b9..dfdf1262c9 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -22,7 +22,7 @@ int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *path, STfsDisk *pDisk = NULL; if ((pDisk = taosMemoryCalloc(1, sizeof(STfsDisk))) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if ((pDisk->path = taosStrdup(path)) == NULL) { diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index d19877dbf1..e0e2a0c323 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -446,7 +446,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { int32_t httpSendQuit(SHttpModule* http, int64_t chanId) { SHttpMsg* msg = taosMemoryCalloc(1, sizeof(SHttpMsg)); if (msg == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } msg->seq = atomic_fetch_add_64(&httpSeqNum, 1); msg->quit = 1; @@ -744,7 +744,7 @@ int64_t transInitHttpChanImpl() { int32_t code = 0; SHttpModule* http = taosMemoryCalloc(1, sizeof(SHttpModule)); if (http == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _ERROR; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 78320c450c..4dd3346f8b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -874,7 +874,7 @@ static int32_t allocConnRef(SCliConn* conn, bool update) { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); if (exh == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } exh->refId = transAddExHandle(transGetRefMgt(), exh); @@ -974,7 +974,7 @@ static int32_t cliCreateConn(SCliThrd* pThrd, SCliConn** pCliConn) { int32_t code = 0; SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); if (conn == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } // read/write stream handle @@ -996,7 +996,7 @@ static int32_t cliCreateConn(SCliThrd* pThrd, SCliConn** pCliConn) { if (timer == NULL) { timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); if (timer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _failed); + TAOS_CHECK_GOTO(terrno, NULL, _failed); } tDebug("no available timer, create a timer %p", timer); @@ -1951,7 +1951,7 @@ static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, SCliBatchList* pBatchList = taosMemoryCalloc(1, sizeof(SCliBatchList)); if (pBatchList == NULL) { tError("failed to create batch list, reason:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } QUEUE_INIT(&pBatchList->wq); pBatchList->port = port; @@ -1991,7 +1991,7 @@ static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliMsg* p SCliBatch* pBatch = taosMemoryCalloc(1, sizeof(SCliBatch)); if (pBatch == NULL) { tError("failed to create batch, reason:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } QUEUE_INIT(&pBatch->wq); @@ -2315,7 +2315,7 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { SCliThrd* pThrd = (SCliThrd*)taosMemoryCalloc(1, sizeof(SCliThrd)); if (pThrd == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + TAOS_CHECK_GOTO(terrno, NULL, _end); } QUEUE_INIT(&pThrd->msg); @@ -2323,7 +2323,7 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (pThrd->loop == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + TAOS_CHECK_GOTO(terrno, NULL, _end); } code = uv_loop_init(pThrd->loop); @@ -2362,7 +2362,7 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { for (int i = 0; i < timerSize; i++) { uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); if (timer == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + TAOS_CHECK_GOTO(terrno, NULL, _end); } (void)uv_timer_init(pThrd->loop, timer); if (taosArrayPush(pThrd->timerList, &timer) == NULL) { @@ -2486,7 +2486,7 @@ int32_t cliSendQuit(SCliThrd* thrd) { int32_t code = 0; SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (msg == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } msg->type = Quit; @@ -2894,7 +2894,7 @@ int32_t transReleaseCliHandle(void* handle) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pCtx->ahandle = tmsg.info.ahandle; @@ -2902,7 +2902,7 @@ int32_t transReleaseCliHandle(void* handle) { SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cmsg == NULL) { taosMemoryFree(pCtx); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } cmsg->msg = tmsg; cmsg->st = taosGetTimestampUs(); @@ -2923,7 +2923,7 @@ static int32_t transInitMsg(void* shandle, const SEpSet* pEpSet, STransMsg* pReq if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } epsetAssign(&pCtx->epSet, pEpSet); @@ -2937,7 +2937,7 @@ static int32_t transInitMsg(void* shandle, const SEpSet* pEpSet, STransMsg* pReq SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cliMsg == NULL) { taosMemoryFree(pCtx); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } cliMsg->ctx = pCtx; @@ -3071,7 +3071,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra STransMsg* pTransRsp = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransRsp == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); @@ -3081,7 +3081,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); if (sem == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } code = tsem_init(sem, 0, 0); @@ -3096,7 +3096,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra if (pCtx == NULL) { (void)tsem_destroy(sem); taosMemoryFree(sem); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } epsetAssign(&pCtx->epSet, pEpSet); @@ -3111,7 +3111,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra (void)tsem_destroy(sem); taosMemoryFree(sem); taosMemoryFree(pCtx); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN1); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } cliMsg->ctx = pCtx; @@ -3150,7 +3150,7 @@ int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { int32_t code = 0; tsem2_t* sem = taosMemoryCalloc(1, sizeof(tsem2_t)); if (sem == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(sem, 0, 0) != 0) { @@ -3159,7 +3159,7 @@ int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { STransSyncMsg* pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg)); if (pSyncMsg == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _EXIT); + TAOS_CHECK_GOTO(terrno, NULL, _EXIT); } taosInitRWLatch(&pSyncMsg->latch); @@ -3194,7 +3194,7 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransMsg == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN2); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); } SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); @@ -3206,7 +3206,7 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN2); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); } epsetAssign(&pCtx->epSet, pEpSet); @@ -3229,7 +3229,7 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cliMsg == NULL) { taosMemoryFree(pCtx); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _RETURN2); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); } cliMsg->ctx = pCtx; @@ -3294,7 +3294,7 @@ int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { for (int8_t i = 0; i < pTransInst->numOfThreads; i++) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } @@ -3303,7 +3303,7 @@ int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cliMsg == NULL) { taosMemoryFree(pCtx); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } @@ -3330,7 +3330,7 @@ int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { int32_t transAllocHandle(int64_t* refId) { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); if (exh == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } exh->refId = transAddExHandle(transGetRefMgt(), exh); @@ -3369,7 +3369,7 @@ int32_t transFreeConnById(void* shandle, int64_t transpointId) { SCliMsg* pCli = taosMemoryCalloc(1, sizeof(SCliMsg)); if (pCli == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + TAOS_CHECK_GOTO(terrno, NULL, _exception); } pCli->type = FreeById; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 1329489be6..05244dbce2 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -70,7 +70,7 @@ int32_t transDecompressMsg(char** msg, int32_t len) { char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STransMsgHead* pNewHead = (STransMsgHead*)buf; @@ -106,7 +106,7 @@ int transSockInfo2Str(struct sockaddr* sockname, char* dst) { int32_t transInitBuffer(SConnBuffer* buf) { buf->buf = taosMemoryCalloc(1, BUFFER_CAP); if (buf->buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } buf->cap = BUFFER_CAP; @@ -149,7 +149,7 @@ int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); if (*buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memcpy(*buf, p->buf, total); if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { @@ -249,7 +249,7 @@ int32_t transSetConnOption(uv_tcp_t* stream, int keepalive) { int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAsyncPool** pPool) { SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); if (pool == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; // return NULL; } int32_t code = 0; @@ -258,7 +258,7 @@ int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAs pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); if (pool->asyncs == NULL) { taosMemoryFree(pool); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int i = 0, err = 0; @@ -267,7 +267,7 @@ int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAs SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); if (item == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } item->pThrd = arg; @@ -560,7 +560,7 @@ int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue) { timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); if (timer == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } heap = heapCreate(timeCompare); @@ -852,7 +852,7 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { int32_t len = 0; char* pBuf = taosMemoryCalloc(1, pList->num * 36); if (pBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int i = 0; i < pList->num; i++) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 11aa468b19..f68561fedf 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -260,7 +260,7 @@ int32_t uvWhiteListToStr(SWhiteUserList* plist, char* user, char** ppBuf) { char* pBuf = taosMemoryCalloc(1, tlen + 64); if (pBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t len = sprintf(pBuf, "user: %s, ver: %" PRId64 ", ip: {%s}", user, plist->ver, tmp); @@ -299,7 +299,7 @@ int32_t uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* plist, if (ppUserList == NULL || *ppUserList == NULL) { SWhiteUserList* pUserList = taosMemoryCalloc(1, sizeof(SWhiteUserList)); if (pUserList == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pUserList->ver = ver; @@ -1104,7 +1104,7 @@ static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); if (pThrd->prepare == NULL) { tError("failed to init prepare"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } code = uv_prepare_init(pThrd->loop, pThrd->prepare); @@ -1155,7 +1155,7 @@ static int32_t addHandleToAcceptloop(void* arg) { srv->pAcceptAsync = taosMemoryCalloc(1, sizeof(uv_async_t)); if (srv->pAcceptAsync == NULL) { tError("failed to create async since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } code = uv_async_init(srv->loop, srv->pAcceptAsync, uvAcceptAsyncCb); @@ -1749,7 +1749,7 @@ int32_t transReleaseSrvHandle(void* handle) { SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (m == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return1; } @@ -1799,7 +1799,7 @@ int32_t transSendResponse(const STransMsg* msg) { SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (m == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return1; } @@ -1844,7 +1844,7 @@ int32_t transRegisterMsg(const STransMsg* msg) { SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (m == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return1; } @@ -1888,7 +1888,7 @@ int32_t transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); if (msg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 9c241516ca..f1ee40eccf 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -148,7 +148,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer if (NULL == buf) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); taosMemoryFreeClear(buf); @@ -393,7 +393,7 @@ int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompa char *buf = taosMemoryCalloc(1, size); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (base && size > 0) { @@ -454,7 +454,7 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c const int32_t THRESHOLD_SIZE = 6; char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t start = 0; start < numOfElem - 1; start += THRESHOLD_SIZE) { diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index b5d873f5ea..84ca86c084 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -43,7 +43,7 @@ int32_t base58_encode(const uint8_t *value, int32_t vlen, char **result) { size = (pe - pb) * 69 / 50 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } bfree = true; } @@ -65,7 +65,7 @@ int32_t base58_encode(const uint8_t *value, int32_t vlen, char **result) { uint8_t *pResult = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); if (!pResult) { if (bfree) taosMemoryFree(pbuf); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memset(pResult, '1', nz); while (pi != pbuf + size) pResult[nz++] = basis_58[*pi++]; @@ -117,7 +117,7 @@ int32_t base58_decode(const char *value, size_t inlen, int32_t *outlen, uint8_t size = (int32_t)(pe - pb) * 733 / 1000 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } bfree = true; } @@ -149,7 +149,7 @@ int32_t base58_decode(const char *value, size_t inlen, int32_t *outlen, uint8_t uint8_t *pResult = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); if (!pResult) { if (bfree) taosMemoryFree(pbuf); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memset(pResult, 0, nz); diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index c87c482167..841657e628 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -44,7 +44,7 @@ int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilte } SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (pBF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pBF->expectedEntries = expectedEntries; @@ -65,7 +65,7 @@ int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilte pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); if (pBF->buffer == NULL) { tBloomFilterDestroy(pBF); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } (*ppBF) = pBF; @@ -152,7 +152,7 @@ int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) { int32_t lino = 0; SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); if (!pBF) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pBF->buffer = NULL; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index c6da1537f5..d20857453d 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -45,7 +45,7 @@ extern char **environ; int32_t cfgInit(SConfig **ppCfg) { SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); if (pCfg == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); @@ -1453,7 +1453,7 @@ struct SConfigIter { int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { SConfigIter *pIter = taosMemoryCalloc(1, sizeof(SConfigIter)); if (pIter == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pIter->pConf = pConf; diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index ddd04b1e88..8de8d26fad 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -93,7 +93,7 @@ int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx) { SGeosContext *tlGeosCtxObj = (SGeosContext *)taosMemoryCalloc(1, sizeof(SGeosContext)); if (!tlGeosCtxObj) { - TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(terrno); } if ((taosThreadSetSpecific(tlGeosCtxKey, (const void *)tlGeosCtxObj)) != 0) { taosMemoryFreeClear(tlGeosCtxObj); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 1d2b3f003c..4cf5917f7c 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -320,7 +320,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v } *pObj = taosMemoryCalloc(1, objSize); if (NULL == *pObj) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return func(pJsonObj, *pObj); } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 3bc24328cc..0fa73677be 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -84,7 +84,7 @@ int32_t tdListPrepend(SList *list, void *data) { int32_t tdListAppend(SList *list, const void *data) { SListNode *node = (SListNode *)taosMemoryCalloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memcpy((void *)(node->data), data, list->eleSize); diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 7faff88155..cfbd875890 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -112,7 +112,7 @@ static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) { table->lengthBits = 4; table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry *)); if (!table->list) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } table->elems = 0; diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 780a6c94f1..de21a61127 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -62,7 +62,7 @@ void taosSetQueueCapacity(STaosQueue *queue, int64_t size) { queue->itemLimit = int32_t taosOpenQueue(STaosQueue **queue) { *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); if (*queue == NULL) { - return (terrno = TSDB_CODE_OUT_OF_MEMORY); + return terrno; } int32_t code = taosThreadMutexInit(&(*queue)->mutex, NULL); @@ -162,7 +162,7 @@ int32_t taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize, void ** STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); if (pNode == NULL) { (void)atomic_sub_fetch_64(&tsQueueMemoryUsed, size + dataSize); - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pNode->dataSize = dataSize; @@ -260,7 +260,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { int32_t taosAllocateQall(STaosQall **qall) { *qall = taosMemoryCalloc(1, sizeof(STaosQall)); if (*qall == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return 0; } @@ -333,7 +333,7 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { int32_t taosOpenQset(STaosQset **qset) { *qset = taosMemoryCalloc(sizeof(STaosQset), 1); if (*qset == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexInit(&(*qset)->mutex, NULL); diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 0eac7b4427..e7161f7284 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -67,13 +67,13 @@ int32_t taosOpenRef(int32_t max, RefFp fp) { nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max); if (lockedBy == NULL) { taosMemoryFree(nodeList); - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexLock(&tsRefMutex); @@ -157,7 +157,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { pNode = taosMemoryCalloc(sizeof(SRefNode), 1); if (pNode == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } rid = atomic_add_fetch_64(&pSet->rid, 1); diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 4fac5a2b4f..ebc076e02e 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -38,7 +38,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf* } SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (pSBf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; @@ -218,7 +218,7 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { int32_t lino = 0; SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); if (!pSBf) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } pSBf->hashFn1 = HASH_FUNCTION_1; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 1fc0d16d57..a1d2d8e38c 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -31,7 +31,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { pool->workers = taosMemoryCalloc(pool->max, sizeof(SQueueWorker)); if (pool->workers == NULL) { taosCloseQset(pool->qset); - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexInit(&pool->mutex, NULL); @@ -310,7 +310,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; pool->workers = taosMemoryCalloc(pool->max, sizeof(SWWorker)); if (pool->workers == NULL) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexInit(&pool->mutex, NULL); @@ -459,7 +459,7 @@ int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) case QWORKER_POOL: { SQWorkerPool *pPool = taosMemoryCalloc(1, sizeof(SQWorkerPool)); if (!pPool) { - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pPool->name = pCfg->name; pPool->min = pCfg->min; @@ -477,7 +477,7 @@ int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) case QUERY_AUTO_QWORKER_POOL: { SQueryAutoQWorkerPool *pPool = taosMemoryCalloc(1, sizeof(SQueryAutoQWorkerPool)); if (!pPool) { - return (terrno = TSDB_CODE_OUT_OF_MEMORY); + return terrno; } pPool->name = pCfg->name; pPool->min = pCfg->min; @@ -812,7 +812,7 @@ int32_t tQueryAutoQWorkerInit(SQueryAutoQWorkerPool *pool) { if (!pool->pCb) { pool->pCb = taosMemoryCalloc(1, sizeof(SQueryAutoQWorkerPoolCB)); - if (!pool->pCb) return TSDB_CODE_OUT_OF_MEMORY; + if (!pool->pCb) return terrno; pool->pCb->pPool = pool; pool->pCb->beforeBlocking = tQueryAutoQWorkerBeforeBlocking; pool->pCb->afterRecoverFromBlocking = tQueryAutoQWorkerRecoverFromBlocking; From b38d9da0e14225acec17a1039e9b18a7a2271b8f Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 06:26:22 +0000 Subject: [PATCH 20/32] feat:add return value --- source/dnode/mnode/impl/src/mndDb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9c02fd6e5d..6467dfcf11 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -2333,9 +2333,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, char keep1Str[128] = {0}; char keep2Str[128] = {0}; - formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0); - formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1); - formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2); + int32_t lenKeep0 = formatDurationOrKeep(keep0Str, pDb->cfg.daysToKeep0); + int32_t lenKeep1 = formatDurationOrKeep(keep1Str, pDb->cfg.daysToKeep1); + int32_t lenKeep2 = formatDurationOrKeep(keep2Str, pDb->cfg.daysToKeep2); if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); From f5e2a36f2382585af829bb6da0542adbe12c6425 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 06:29:35 +0000 Subject: [PATCH 21/32] feat:update --- source/libs/command/src/command.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 0e5f2f4cbd..70a9b39229 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -345,12 +345,12 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { } int32_t formatDurationOrKeep(char* buffer, int32_t timeInMinutes) { - int len = 0; + int32_t len = 0; if (timeInMinutes % 1440 == 0) { - int days = timeInMinutes / 1440; + int32_t days = timeInMinutes / 1440; len = sprintf(buffer, "%dd", days); } else if (timeInMinutes % 60 == 0) { - int hours = timeInMinutes / 60; + int32_t hours = timeInMinutes / 60; len = sprintf(buffer, "%dh", hours); } else { len = sprintf(buffer, "%dm", timeInMinutes); From 6c610e477755b9f4eafdc6bf6a2318224e28067c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 14:45:03 +0800 Subject: [PATCH 22/32] fix mem leak --- source/libs/transport/src/transCli.c | 1 - source/libs/transport/src/transSvr.c | 14 +++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a0c7315fe7..ce9adc836e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2422,7 +2422,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); - (void)uv_timer_stop(timer); taosMemoryFree(timer); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 48042b84c1..1db5d0a6d7 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1398,10 +1398,21 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, thrd->quit = false; thrd->pTransInst = shandle; thrd->pWhiteList = uvWhiteListCreate(); + if (thrd->pWhiteList == NULL) { + destroyWorkThrdObj(thrd); + code = TSDB_CODE_OUT_OF_MEMORY; + goto End; + } - srv->pThreadObj[i] = thrd; srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); + if (srv->pipe[i] == NULL) { + destroyWorkThrdObj(thrd); + code = TSDB_CODE_OUT_OF_MEMORY; + goto End; + } + thrd->pipe = &(srv->pipe[i][1]); // init read + srv->pThreadObj[i] = thrd; if ((code = addHandleToWorkloop(thrd, pipeName)) != 0) { goto End; @@ -1415,6 +1426,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, tError("failed to create worker-thread:%d", i); goto End; } + thrd->inited = 1; } #else From d6707788021138a4a41175b822f21d726d2eb63e Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Tue, 10 Sep 2024 09:16:35 +0800 Subject: [PATCH 23/32] fix:[TD-31974] fix memory leak when error occurs after mode function init. --- source/libs/executor/src/executorInt.c | 4 ++++ source/libs/executor/src/operator.c | 10 ++++++---- source/libs/function/src/builtinsimpl.c | 7 ++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 00138d6871..1659c1fa3e 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1010,6 +1010,10 @@ static void destroySqlFunctionCtx(SqlFunctionCtx* pCtx, SExprInfo* pExpr, int32_ } for (int32_t i = 0; i < numOfOutput; ++i) { + if (pCtx[i].fpSet.cleanup != NULL) { + pCtx[i].fpSet.cleanup(&pCtx[i]); + } + if (pExpr != NULL) { SExprInfo* pExprInfo = &pExpr[i]; for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) { diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 6fe270161f..8fe8c6363a 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -659,10 +659,6 @@ void destroyOperator(SOperatorInfo* pOperator) { freeResetOperatorParams(pOperator, OP_GET_PARAM, true); freeResetOperatorParams(pOperator, OP_NOTIFY_PARAM, true); - if (pOperator->fpSet.closeFn != NULL && pOperator->info != NULL) { - pOperator->fpSet.closeFn(pOperator->info); - } - if (pOperator->pDownstream != NULL) { for (int32_t i = 0; i < pOperator->numOfRealDownstream; ++i) { destroyOperator(pOperator->pDownstream[i]); @@ -673,6 +669,12 @@ void destroyOperator(SOperatorInfo* pOperator) { } cleanupExprSupp(&pOperator->exprSupp); + + // close operator after cleanup exprSupp, since we need to call cleanup of sqlFunctionCtx first to avoid mem leak. + if (pOperator->fpSet.closeFn != NULL && pOperator->info != NULL) { + pOperator->fpSet.closeFn(pOperator->info); + } + taosMemoryFreeClear(pOperator); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6d655528c7..c55839d033 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6036,9 +6036,10 @@ static void modeFunctionCleanup(SModeInfo * pInfo) { } void modeFunctionCleanupExt(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - modeFunctionCleanup(pInfo); + if (pCtx == NULL || GET_RES_INFO(pCtx) == NULL || GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)) == NULL) { + return; + } + modeFunctionCleanup(GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx))); } static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pInfo, STuplePos* pPos) { From 5cfe7b8ec9ea5556d335fd66a685677e0fe9b5f8 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 11 Sep 2024 14:58:06 +0800 Subject: [PATCH 24/32] fix(query):fix the issue of return db error when the timestamp is 0 --- source/libs/executor/src/timewindowoperator.c | 8 +---- tests/parallel_test/cases.task | 1 + tests/script/tsim/compute/interval1.sim | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 tests/script/tsim/compute/interval1.sim diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 93b3759c47..640ce5c98b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -894,13 +894,7 @@ int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo tsCols = (int64_t*)pColDataInfo->pData; if(tsCols[0] == 0) { - pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; - T_LONG_JMP(pTaskInfo->env, terrno); - } - - // no data in primary ts - if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) { - return NULL; + qWarn("%s at line %d.block start ts:%" PRId64 ",end ts:%" PRId64, __func__, __LINE__, tsCols[0], tsCols[pBlock->info.rows - 1]); } if (tsCols[0] != 0 && (pBlock->info.window.skey == 0 && pBlock->info.window.ekey == 0)) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c8e0a624d5..76ec6422b9 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1453,6 +1453,7 @@ ,,y,script,./test.sh -f tsim/compute/diff2.sim ,,y,script,./test.sh -f tsim/compute/first.sim ,,y,script,./test.sh -f tsim/compute/interval.sim +,,y,script,./test.sh -f tsim/compute/interval1.sim ,,y,script,./test.sh -f tsim/compute/last_row.sim ,,y,script,./test.sh -f tsim/compute/last.sim ,,y,script,./test.sh -f tsim/compute/leastsquare.sim diff --git a/tests/script/tsim/compute/interval1.sim b/tests/script/tsim/compute/interval1.sim new file mode 100644 index 0000000000..74af727917 --- /dev/null +++ b/tests/script/tsim/compute/interval1.sim @@ -0,0 +1,33 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c timezone -v UTC +system sh/exec.sh -n dnode1 -s start +sql connect + + +sql CREATE DATABASE `alphacloud_alphaess` BUFFER 512 CACHESIZE 1024 CACHEMODEL 'both' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 365000d; + +sql use alphacloud_alphaess; + +sql create stable st(ts TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `uk` VARCHAR(64) ENCODE 'disabled' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY ) tags(ta int,tb int,tc int); + +sql create table t1 using st tags(1,1,1); + +sql insert into t1 values ("1970-01-29 05:04:53.000"," 22:: "); + +sql select _wstart, count(*) from st interval(1y); + +if $rows != 1 then + print =====rows=$rows + return -1 +endi + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +if $data01 != 1 then + print =====data00=$data00 + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 9b418bee783bd7c53e36819bd408d05b590d6427 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Sep 2024 16:37:06 +0800 Subject: [PATCH 25/32] fix meta dead lock --- source/libs/executor/src/scanoperator.c | 139 ++++++++++++------------ 1 file changed, 72 insertions(+), 67 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 42e7e4ac3b..b0b6a91faa 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -90,7 +90,7 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order, bool* overlap) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; STimeWindow w = {0}; // 0 by default, which means it is not a interval operator of the upstream operator. @@ -101,7 +101,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc if (order == TSDB_ORDER_ASC) { w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); - if(w.ekey < pBlockInfo->window.skey) { + if (w.ekey < pBlockInfo->window.skey) { qError("w.ekey:%" PRId64 " < pBlockInfo->window.skey:%" PRId64, w.ekey, pBlockInfo->window.skey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -117,7 +117,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc break; } - if(w.ekey <= pBlockInfo->window.ekey) { + if (w.ekey <= pBlockInfo->window.ekey) { qError("w.ekey:%" PRId64 " <= pBlockInfo->window.ekey:%" PRId64, w.ekey, pBlockInfo->window.ekey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -128,7 +128,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc } } else { w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); - if(w.skey > pBlockInfo->window.ekey) { + if (w.skey > pBlockInfo->window.ekey) { qError("w.skey:%" PRId64 " > pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.ekey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -144,7 +144,7 @@ static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBloc break; } - if(w.skey >= pBlockInfo->window.skey){ + if (w.skey >= pBlockInfo->window.skey) { qError("w.skey:%" PRId64 " >= pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.skey); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } @@ -227,7 +227,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset); - EFuncDataRequired reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); + EFuncDataRequired reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { notLoadBlock = false; break; @@ -399,7 +399,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca } } - if(*status != FUNC_DATA_REQUIRED_DATA_LOAD) { + if (*status != FUNC_DATA_REQUIRED_DATA_LOAD) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); qError("%s loadDataBlock invalid status:%d", GET_TASKID(pTaskInfo), *status); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; @@ -588,14 +588,14 @@ static void freeTableCachedValObj(STableCachedVal* pVal) { int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; - bool freeReader = false; - LRUHandle* h = NULL; - STableCachedVal val = {0}; - SMetaReader mr = {0}; - const char* idStr = pTask->id.str; - int32_t insertRet = TAOS_LRU_STATUS_OK; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool freeReader = false; + LRUHandle* h = NULL; + STableCachedVal val = {0}; + SMetaReader mr = {0}; + const char* idStr = pTask->id.str; + int32_t insertRet = TAOS_LRU_STATUS_OK; STableCachedVal* pVal = NULL; // currently only the tbname pseudo column @@ -625,8 +625,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // append null value before return to caller, since the caller will ignore this error code and proceed doSetNullValue(pBlock, pExpr, numOfExpr); } else { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(code), - idStr); + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(code), idStr); } pHandle->api.metaReaderFn.clearReader(&mr); return code; @@ -743,7 +742,7 @@ _end: if (NULL != pVal) { insertRet = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal, - sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); + sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (insertRet != TAOS_LRU_STATUS_OK) { qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr); } @@ -1416,7 +1415,7 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr if (!pRecorder) { return terrno; } - STableScanInfo* pTableScanInfo = pOptr->info; + STableScanInfo* pTableScanInfo = pOptr->info; *pRecorder = pTableScanInfo->base.readRecorder; *pOptrExplain = pRecorder; *len = sizeof(SFileBlockLoadRecorder); @@ -1499,7 +1498,7 @@ int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHa initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); QUERY_CHECK_NULL(pInfo->pResBlock, code, lino, _error, terrno); - + code = prepareDataBlockBuf(pInfo->pResBlock, &pInfo->base.matchInfo); QUERY_CHECK_CODE(code, lino, _error); @@ -1538,7 +1537,7 @@ int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHa _error: if (pInfo != NULL) { - pInfo->base.pTableListInfo = NULL; // this attribute will be destroy outside of this function + pInfo->base.pTableListInfo = NULL; // this attribute will be destroy outside of this function destroyTableScanOperatorInfo(pInfo); } @@ -1660,7 +1659,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (void**)&pReader, GET_TASKID(pTaskInfo), NULL); + (void**)&pReader, GET_TASKID(pTaskInfo), NULL); QUERY_CHECK_CODE(code, lino, _end); bool hasNext = false; @@ -1813,7 +1812,7 @@ static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ QUERY_CHECK_NULL(pInfo->pUpdateInfo, code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, - pInfo->pUpdateInfo->maxDataVersion); + pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; if (pRes) { @@ -2236,9 +2235,9 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS } uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; - TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; - TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t ver = pSrcBlock->info.version - 1; + TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; + int64_t ver = pSrcBlock->info.version - 1; if (pInfo->partitionSup.needCalc && (srcStartTsCol[0] != srcEndTsCol[0] || (hasPrimaryKeyCol(pInfo) && mode == STREAM_DELETE_DATA))) { @@ -2804,8 +2803,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { - code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, - pInfo->pRes, pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache); + code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, + pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { code = 0; @@ -3182,7 +3181,7 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) return; } - void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); + void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); if (!pUpInfo) { return; } @@ -3471,7 +3470,8 @@ FETCH_NEXT_BLOCK: (*ppRes) = pInfo->pDeleteDataRes; return code; } - qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, lino); + qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, + lino); blockDataCleanup(pInfo->pUpdateDataRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } break; @@ -3484,7 +3484,8 @@ FETCH_NEXT_BLOCK: (*ppRes) = pInfo->pUpdateRes; return code; } - qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, lino); + qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, + lino); blockDataCleanup(pInfo->pUpdateDataRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } break; @@ -3671,7 +3672,7 @@ static int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** for (int32_t i = 0; i < size; ++i) { STableKeyInfo* pkeyInfo = tableListGetInfo(pTableListInfo, i); QUERY_CHECK_NULL(pkeyInfo, code, lino, _end, terrno); - void* tmp = taosArrayPush(tableIdList, &pkeyInfo->uid); + void* tmp = taosArrayPush(tableIdList, &pkeyInfo->uid); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } @@ -4066,7 +4067,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* int32_t numOfOutput = taosArrayGetSize(pInfo->matchInfo.pList); pColIds = taosArrayInit(numOfOutput, sizeof(int16_t)); QUERY_CHECK_NULL(pColIds, code, lino, _error, terrno); - + for (int32_t i = 0; i < numOfOutput; ++i) { SColMatchItem* id = taosArrayGet(pInfo->matchInfo.pList, i); QUERY_CHECK_NULL(id, code, lino, _error, terrno); @@ -4268,7 +4269,7 @@ _error: } static int32_t doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, - SStorageAPI* pAPI) { + SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4280,7 +4281,6 @@ static int32_t doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pR qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); tDecoderClear(&(*mr).coder); - pAPI->metaReaderFn.clearReader(mr); goto _end; } @@ -4289,7 +4289,6 @@ static int32_t doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pR if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - pAPI->metaReaderFn.clearReader(mr); goto _end; } @@ -4697,6 +4696,12 @@ static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock* while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { code = doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); + if (code == TSDB_CODE_OUT_OF_MEMORY) { + break; + } else { + // ignore other error + } + ++count; if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); @@ -4814,7 +4819,7 @@ int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* p nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx); } } - //TODO wjm check pInfo->filterCtx.code + // TODO wjm check pInfo->filterCtx.code __optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdxNext : doTagScanFromMetaEntryNext; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); @@ -4925,7 +4930,7 @@ static int32_t fetchNextSubTableBlockFromReader(SOperatorInfo* pOperator, STmsSu pInfo->base.dataReader = pInput->pReader; while (true) { - bool hasNext = false; + bool hasNext = false; code = pAPI->tsdReader.tsdNextDataBlock(pInfo->base.dataReader, &hasNext); if (code != 0) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->base.dataReader); @@ -5036,7 +5041,7 @@ static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) { static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { int32_t code = setGroupStartEndIndex(pInfo); if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); return code; } STmsSubTablesMergeInfo* pSubTblsInfo = taosMemoryCalloc(1, sizeof(STmsSubTablesMergeInfo)); @@ -5065,8 +5070,7 @@ static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { } int32_t bufPageSize = pInfo->bufPageSize; int32_t inMemSize = (pSubTblsInfo->numSubTables - pSubTblsInfo->numTableBlocksInMem) * bufPageSize; - code = - createDiskbasedBuf(&pSubTblsInfo->pBlocksBuf, pInfo->bufPageSize, inMemSize, "blocksExternalBuf", tsTempDir); + code = createDiskbasedBuf(&pSubTblsInfo->pBlocksBuf, pInfo->bufPageSize, inMemSize, "blocksExternalBuf", tsTempDir); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pSubTblsInfo->aInputs); taosMemoryFree(pSubTblsInfo); @@ -5199,7 +5203,7 @@ static int32_t appendChosenRowToDataBlock(STmsSubTablesMergeInfo* pSubTblsInfo, SColumnInfoData* pSrcColInfo = taosArrayGet(pInputBlock->pDataBlock, i); QUERY_CHECK_NULL(pSrcColInfo, code, lino, _end, terrno); - bool isNull = colDataIsNull(pSrcColInfo, pInputBlock->info.rows, pInput->rowIdx, NULL); + bool isNull = colDataIsNull(pSrcColInfo, pInputBlock->info.rows, pInput->rowIdx, NULL); if (isNull) { code = colDataSetVal(pColInfo, pBlock->info.rows, NULL, true); @@ -5223,7 +5227,8 @@ _end: return code; } -static int32_t getSubTablesSortedBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t capacity, SSDataBlock** pResBlock) { +static int32_t getSubTablesSortedBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t capacity, + SSDataBlock** pResBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; STableMergeScanInfo* pInfo = pOperator->info; @@ -5580,9 +5585,9 @@ static int32_t getBlockForTableMergeScan(void* param, SSDataBlock** ppBlock) { } int32_t generateSortByTsPkInfo(SArray* colMatchInfo, int32_t order, SArray** ppSortArray) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; - SArray* pSortInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SArray* pSortInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); QUERY_CHECK_NULL(pSortInfo, code, lino, _end, terrno); SBlockOrderInfo biTs = {0}; SBlockOrderInfo biPk = {0}; @@ -5983,7 +5988,7 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla if (!execInfo) { return terrno; } - STableMergeScanInfo* pInfo = pOptr->info; + STableMergeScanInfo* pInfo = pOptr->info; execInfo->blockRecorder = pInfo->base.readRecorder; execInfo->sortExecInfo = pInfo->sortExecInfo; @@ -6126,24 +6131,24 @@ _error: // ==================================================================================================================== // TableCountScanOperator -static void destoryTableCountScanOperator(void* param); -static int32_t buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI); -static int32_t buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI); -static int32_t buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); -static int32_t buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName); -static int32_t buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes); -static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, - size_t perfdbTableNum); -static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, - size_t infodbTableNum, size_t perfdbTableNum); -static const char* GROUP_TAG_DB_NAME = "db_name"; -static const char* GROUP_TAG_STABLE_NAME = "stable_name"; +static void destoryTableCountScanOperator(void* param); +static int32_t buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI); +static int32_t buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI); +static int32_t buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); +static int32_t buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName); +static int32_t buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes); +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum); +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum); +static const char* GROUP_TAG_DB_NAME = "db_name"; +static const char* GROUP_TAG_STABLE_NAME = "stable_name"; int32_t tblCountScanGetGroupTagsSlotId(const SNodeList* scanCols, STableCountScanSupp* supp) { if (scanCols != NULL) { @@ -6478,7 +6483,7 @@ _end: } static int32_t buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -6526,7 +6531,7 @@ _end: } static int32_t buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, - STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; From 0b6a934586e7d07b51b04d6054be096efc10faf7 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 11 Sep 2024 09:15:20 +0000 Subject: [PATCH 26/32] fix/TS-5404-use-temp-transid-when-timeout-lock --- source/dnode/mnode/impl/src/mndTrans.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 18ceddb7d0..724c9637e6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -64,7 +64,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); -int32_t tsMaxTransId = 0; +static int32_t tsMaxTransId = 0; int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { @@ -605,8 +605,9 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, } int32_t sdbMaxId = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); - int32_t oldId = atomic_load_32(&tsMaxTransId); - pTrans->id = TMAX(sdbMaxId, oldId); + sdbReadLock(pMnode->pSdb, SDB_TRANS); + pTrans->id = TMAX(sdbMaxId, tsMaxTransId + 1); + sdbUnLock(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->conflict = conflict; @@ -1031,7 +1032,9 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); - atomic_store_32(&tsMaxTransId, pTrans->id); + sdbReadLock(pMnode->pSdb, SDB_TRANS); + tsMaxTransId = TMAX(pTrans->id, tsMaxTransId); + sdbUnLock(pMnode->pSdb, SDB_TRANS); TAOS_RETURN(code); } mInfo("trans:%d, prepare finished", pTrans->id); From cf35e31289b86d885ae1cdd2812f6c0f13ce8fab Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Wed, 11 Sep 2024 09:43:55 +0000 Subject: [PATCH 27/32] modify test case --- tests/script/tsim/db/alter_option.sim | 6 +++--- tests/script/tsim/db/basic6.sim | 6 +++--- tests/script/tsim/db/keep.sim | 4 ++-- tests/script/tsim/parser/alter.sim | 14 +++++++------- .../tsim/parser/alter__for_community_version.sim | 12 ++++++------ tests/script/tsim/parser/create_db.sim | 16 ++++++++-------- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 749b956a9a..4c18fe171c 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -72,10 +72,10 @@ endi if $data5_db != on then # strict return -1 endi -if $data6_db != 345600m then # duration +if $data6_db != 240d then # duration return -1 endi -if $data7_db != 1440000m,1440000m,1440000m then # keep +if $data7_db != 1000d,1000d,1000d then # keep return -1 endi if $data8_db != 256 then # buffer @@ -220,7 +220,7 @@ print ============== modify keep sql alter database db keep 2400 sql select * from information_schema.ins_databases print keep $data7_db -if $data7_db != 3456000m,3456000m,3456000m then +if $data7_db != 2400d,2400d,2400d then return -1 endi diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 5043574787..8d16c3e975 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -34,10 +34,10 @@ endi if $data24 != 1 then return -1 endi -if $data26 != 2880m then +if $data26 != 2d then return -1 endi -if $data27 != 14400m,14400m,14400m then +if $data27 != 10d,10d,10d then return -1 endi #if $data28 != 32 then @@ -78,7 +78,7 @@ endi if $data24 != 1 then return -1 endi -if $data26 != 21600m then +if $data26 != 15d then return -1 endi diff --git a/tests/script/tsim/db/keep.sim b/tests/script/tsim/db/keep.sim index f0653c4801..28c595cd21 100644 --- a/tests/script/tsim/db/keep.sim +++ b/tests/script/tsim/db/keep.sim @@ -45,7 +45,7 @@ print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data22 != 2 then return -1 endi -if $data27 != 86400m,86400m,86400m then +if $data27 != 60d,60d,60d then return -1 endi @@ -86,7 +86,7 @@ sql select * from information_schema.ins_databases if $data22 != 2 then return -1 endi -if $data27 != 43200m,43200m,43200m then +if $data27 != 30d,30d,30d then return -1 endi diff --git a/tests/script/tsim/parser/alter.sim b/tests/script/tsim/parser/alter.sim index 99e014a011..ceeb5df34f 100644 --- a/tests/script/tsim/parser/alter.sim +++ b/tests/script/tsim/parser/alter.sim @@ -40,7 +40,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 30240m,30240m,30240m then +if $data27 != 21d,21d,21d then return -1 endi sql alter database $db keep 11,12 @@ -48,7 +48,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,17280m,17280m then +if $data27 != 11d,12d,12d then return -1 endi sql alter database $db keep 20,20,20 @@ -56,7 +56,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 28800m,28800m,28800m then +if $data27 != 20d,20d,20d then return -1 endi sql alter database $db keep 10,10,10 @@ -64,7 +64,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 14400m,14400m,14400m then +if $data27 != 10d,10d,10d then return -1 endi sql alter database $db keep 10,10,11 @@ -72,7 +72,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 14400m,14400m,15840m then +if $data27 != 10d,10d,11d then return -1 endi sql alter database $db keep 11,12,13 @@ -80,7 +80,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,17280m,18720m then +if $data27 != 11d,12d,13d then return -1 endi sql alter database $db keep 365000,365000,365000 @@ -88,7 +88,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 525600000m,525600000m,525600000m then +if $data27 != 365000d,365000d,365000d then return -1 endi diff --git a/tests/script/tsim/parser/alter__for_community_version.sim b/tests/script/tsim/parser/alter__for_community_version.sim index 29c748d441..b3534204fc 100644 --- a/tests/script/tsim/parser/alter__for_community_version.sim +++ b/tests/script/tsim/parser/alter__for_community_version.sim @@ -23,7 +23,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 28800m,28800m,28800m then +if $data27 != 20d,20d,20d then return -1 endi @@ -49,7 +49,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 28800m,28800m,28800m then +if $data27 != 20d,20d,20d then return -1 endi sql alter database $db keep 10 @@ -57,7 +57,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 14400m,14400m,14400m then +if $data27 != 10d,10d,10d then return -1 endi sql alter database $db keep 11 @@ -65,7 +65,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,15840m,15840m then +if $data27 != 11d,11d,11d then return -1 endi sql alter database $db keep 13 @@ -73,7 +73,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 18720m,18720m,18720m then +if $data27 != 13d,13d,13d then return -1 endi sql alter database $db keep 365000 @@ -81,7 +81,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 525600000m,525600000m,525600000m then +if $data27 != 365000d,365000d,365000d then return -1 endi diff --git a/tests/script/tsim/parser/create_db.sim b/tests/script/tsim/parser/create_db.sim index 5cb659586c..1057e0b465 100644 --- a/tests/script/tsim/parser/create_db.sim +++ b/tests/script/tsim/parser/create_db.sim @@ -118,10 +118,10 @@ endi if $data24 != $replica then return -1 endi -if $data26 != 14400m then +if $data26 != 10d then return -1 endi -if $data27 != 525600m,525600m,525600m then +if $data27 != 365d,365d,365d then return -1 endi @@ -155,7 +155,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 56160m,56160m,56160m then +if $data27 != 39d,39d,39d then return -1 endi sql drop database dbk0 @@ -164,7 +164,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 56160m,57600m,57600m then +if $data27 != 39d,40d,40d then return -1 endi sql drop database dbka @@ -174,7 +174,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,15840m,15840m then +if $data27 != 11d,11d,11d then return -1 endi sql drop database dbk1 @@ -183,7 +183,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,17280m,18720m then +if $data27 != 11d,12d,13d then return -1 endi sql drop database dbk2 @@ -192,7 +192,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,15840m,18720m then +if $data27 != 11d,11d,13d then return -1 endi sql drop database dbk3 @@ -201,7 +201,7 @@ sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -if $data27 != 15840m,18720m,18720m then +if $data27 != 11d,13d,13d then return -1 endi sql drop database dbk4 From 2bb9dc609a2cc25b3e99afe64b90249749e04dcb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Sep 2024 18:18:44 +0800 Subject: [PATCH 28/32] fix(query): fix race condition for table group list. --- source/libs/executor/src/scanoperator.c | 36 ++++++++++++++++++------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 42e7e4ac3b..f30329889a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1241,6 +1241,29 @@ _end: return code; } +static int32_t doInitReader(STableScanInfo* pInfo, SExecTaskInfo* pTaskInfo, SStorageAPI* pAPI, int32_t* pNum, + STableKeyInfo** pList) { + const char* idStr = GET_TASKID(pTaskInfo); + int32_t code = initNextGroupScan(pInfo, pList, pNum); + if (code) { + qError("%s failed to init groupScan Info, code:%s at line:%d", idStr, tstrerror(code), __LINE__); + return code; + } + + if (pInfo->base.dataReader != NULL) { + qError("%s tsdb reader should be null", idStr); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + + code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, *pList, *pNum, pInfo->pResBlock, + (void**)&pInfo->base.dataReader, idStr, &pInfo->pIgnoreTables); + if (code) { + qError("%s failed to open tsdbReader, code:%s at line:%d", idStr, tstrerror(code), __LINE__); + } + + return code; +} + static int32_t groupSeqTableScan(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -1250,6 +1273,7 @@ static int32_t groupSeqTableScan(SOperatorInfo* pOperator, SSDataBlock** pResBlo int32_t num = 0; STableKeyInfo* pList = NULL; SSDataBlock* pResult = NULL; + const char* idStr = GET_TASKID(pTaskInfo); QRY_PARAM_CHECK(pResBlock); @@ -1260,17 +1284,10 @@ static int32_t groupSeqTableScan(SOperatorInfo* pOperator, SSDataBlock** pResBlo } taosRLockLatch(&pTaskInfo->lock); - code = initNextGroupScan(pInfo, &pList, &num); - + code = doInitReader(pInfo, pTaskInfo, pAPI, &num, &pList); taosRUnLockLatch(&pTaskInfo->lock); QUERY_CHECK_CODE(code, lino, _end); - QUERY_CHECK_CONDITION((pInfo->base.dataReader == NULL), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); - - code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), &pInfo->pIgnoreTables); - QUERY_CHECK_CODE(code, lino, _end); - if (pInfo->filesetDelimited) { pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader); } @@ -1280,7 +1297,6 @@ static int32_t groupSeqTableScan(SOperatorInfo* pOperator, SSDataBlock** pResBlo } } - pResult = NULL; code = doGroupedTableScan(pOperator, &pResult); QUERY_CHECK_CODE(code, lino, _end); @@ -1305,7 +1321,7 @@ static int32_t groupSeqTableScan(SOperatorInfo* pOperator, SSDataBlock** pResBlo _end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + qError("%s %s failed at line %d since %s", idStr, __func__, lino, tstrerror(code)); pTaskInfo->code = code; } From 4319cacdc3871bf260a8b3945f42de9280f01141 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Sep 2024 19:09:17 +0800 Subject: [PATCH 29/32] fix(query):fix invalid read. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 15 ++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 7c68d31f84..294874b94a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -292,7 +292,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); +int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap, const char* id); void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); int32_t tsdbGetTableSchema(SMeta *pMeta, int64_t uid, STSchema **pSchema, int64_t *suid); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index c3364231e0..b24aa6fb1d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -473,7 +473,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } (void)taosThreadMutexLock(&pr->readerMutex); - code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); + code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap, pr->idstr); if (code != TSDB_CODE_SUCCESS) { goto _end; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 8f66df1ffb..c48e37c65d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5016,7 +5016,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { tsdbTrace("tsdb/reader: %p, take snapshot", pReader); - code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); + code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -5862,7 +5862,7 @@ int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_ return code; } -int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { +int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap, const char* id) { int32_t code = 0; STsdb* pTsdb = pReader->pTsdb; SVersionRange* pRange = &pReader->info.verRange; @@ -5872,7 +5872,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs // lock code = taosThreadMutexLock(&pTsdb->mutex); if (code != TSDB_CODE_SUCCESS) { - tsdbError("failed to lock tsdb, code:%s", tstrerror(code)); + tsdbError("%s failed to lock tsdb, code:%s", id, tstrerror(code)); return code; } @@ -5932,18 +5932,19 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs } (void) taosThreadMutexUnlock(&pTsdb->mutex); - TSDB_CHECK_CODE(code, lino, _exit);} + TSDB_CHECK_CODE(code, lino, _exit); + } // unlock (void) taosThreadMutexUnlock(&pTsdb->mutex); *ppSnap = pSnap; - tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); + tsdbTrace("%s vgId:%d, take read snapshot", id, TD_VID(pTsdb->pVnode)); return code; _exit: - tsdbError("%s vgId:%d take read snapshot failed, line:%d code:%s", pReader->idStr, TD_VID(pTsdb->pVnode), lino, - tstrerror(code)); + tsdbError("%s vgId:%d take read snapshot failed, line:%d code:%s", id, TD_VID(pTsdb->pVnode), lino, tstrerror(code)); + if (pSnap) { if (pSnap->pNode) taosMemoryFree(pSnap->pNode); if (pSnap->pINode) taosMemoryFree(pSnap->pINode); From 4d103c09d490f67fd5b6fb836d91449a8731433c Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 11 Sep 2024 19:22:04 +0800 Subject: [PATCH 30/32] fix: ignore tsdbCacheFlushDirty errcode --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 53e20d459f..74bd837775 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -577,7 +577,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud SLastCol *pLastCol = (SLastCol *)value; if (pLastCol->dirty) { - tsdbCacheFlushDirty(key, klen, pLastCol, ud); + (void)tsdbCacheFlushDirty(key, klen, pLastCol, ud); } for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { From 9923079693fb3eb24186812bed79f7519980ea95 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 11 Sep 2024 19:57:47 +0800 Subject: [PATCH 31/32] TD-31198: add 32/64 bit ODBC driver into Windows packages --- cmake/taosadapter_CMakeLists.txt.in | 2 +- packaging/tools/tdengine.iss | 41 +++++++++++++---------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index ca6b5a3e5f..8085c55e3e 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -55,7 +55,8 @@ Source: favicon.ico; DestDir: "{app}\include"; Flags: igNoreversion; Source: {#MyAppSourceDir}{#MyAppDLLName}; DestDir: "{win}\System32"; Flags: igNoreversion recursesubdirs createallsubdirs 64bit;Check:IsWin64; Source: {#MyAppSourceDir}{#MyAppCfgName}; DestDir: "{app}\cfg"; Flags: igNoreversion recursesubdirs createallsubdirs onlyifdoesntexist uninsneveruninstall Source: {#MyAppSourceDir}{#MyAppDriverName}; DestDir: "{app}\driver"; Flags: igNoreversion recursesubdirs createallsubdirs -Source: {#MyAppSourceDir}\taos_odbc\*; DestDir: "{app}\taos_odbc\"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}\taos_odbc\*; DestDir: "{app}\taos_odbc"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}\*.dll; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs ;Source: {#MyAppSourceDir}{#MyAppConnectorName}; DestDir: "{app}\connector"; Flags: igNoreversion recursesubdirs createallsubdirs ;Source: {#MyAppSourceDir}{#MyAppExamplesName}; DestDir: "{app}\examples"; Flags: igNoreversion recursesubdirs createallsubdirs Source: {#MyAppSourceDir}{#MyAppIncludeName}; DestDir: "{app}\include"; Flags: igNoreversion recursesubdirs createallsubdirs @@ -69,7 +70,9 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt [run] Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden -Filename: "{cmd}"; Parameters: "/c odbcconf /F ""C:\TDengine\taos_odbc\win_odbcinst.in"""; WorkingDir: "{app}"; Flags: runhidden; StatusMsg: "Configuring ODBC" + +Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64" +Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86" [UninstallRun] RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden @@ -100,32 +103,26 @@ begin end; function DeleteOdbcDsnRegistry: Boolean; -var - Names: TArrayOfString; - I: Integer; - Value: String; begin - if RegGetValueNames(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', Names) then - begin - for I := 0 to GetArrayLength(Names) - 1 do - begin - if RegQueryStringValue(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', Names[I], Value) then - begin - if Value = 'TDengine' then - begin - RegDeleteKeyIncludingSubkeys(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\' + Names[I]); - RegDeleteValue(HKCU64, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources\', Names[I]); - end; - end; - end; - end; + RegDeleteKeyIncludingSubkeys(HKCU, 'SOFTWARE\ODBC\ODBC.INI\TAOS_ODBC_DSN'); + RegDeleteKeyIncludingSubkeys(HKCU, 'SOFTWARE\ODBC\ODBC.INI\TAOS_ODBC_WS_DSN') + + RegDeleteValue(HKCU, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', 'TAOS_ODBC_DSN'); + RegDeleteValue(HKCU, 'SOFTWARE\ODBC\ODBC.INI\ODBC Data Sources', 'TAOS_ODBC_WS_DSN'); + Result := True; end; function DeleteOdbcDriverRegistry: Boolean; begin - RegDeleteKeyIncludingSubkeys(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\TDengine'); - RegDeleteValue(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\ODBC Drivers', 'TDengine'); + // Delete 64-bit ODBC driver registry + RegDeleteKeyIncludingSubkeys(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\TAOS_ODBC_DRIVER'); + RegDeleteValue(HKLM64, 'SOFTWARE\ODBC\ODBCINST.INI\ODBC Drivers', 'TAOS_ODBC_DRIVER'); + + // Delete 32-bit ODBC driver registry + RegDeleteKeyIncludingSubkeys(HKLM64, 'SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\TAOS_ODBC_DRIVER'); + RegDeleteValue(HKLM64, 'SOFTWARE\Wow6432Node\ODBC\ODBCINST.INI\ODBC Drivers', 'TAOS_ODBC_DRIVER'); + Result := True; end; From 28b3c92dddd0f9d3f91ef3b77e8d051e5abb2494 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 12 Sep 2024 02:36:07 +0000 Subject: [PATCH 32/32] fix/TS-5404-use-write-lock --- source/dnode/mnode/impl/src/mndTrans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 724c9637e6..43f8ffe61b 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1032,7 +1032,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mInfo("trans:%d, prepare transaction", pTrans->id); if ((code = mndTransSync(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code)); - sdbReadLock(pMnode->pSdb, SDB_TRANS); + sdbWriteLock(pMnode->pSdb, SDB_TRANS); tsMaxTransId = TMAX(pTrans->id, tsMaxTransId); sdbUnLock(pMnode->pSdb, SDB_TRANS); TAOS_RETURN(code);