From 6fbfda871fa92c9d648b77057a338d107459489a Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 15 Aug 2024 13:49:45 +0800 Subject: [PATCH 01/59] add test case for last and last_row --- tests/army/query/last/test_last.py | 192 +++++++++++++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 193 insertions(+) create mode 100644 tests/army/query/last/test_last.py diff --git a/tests/army/query/last/test_last.py b/tests/army/query/last/test_last.py new file mode 100644 index 0000000000..287cc97974 --- /dev/null +++ b/tests/army/query/last/test_last.py @@ -0,0 +1,192 @@ +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * + + +class TDTestCase(TBase): + """Add test case to verify TD-30816 (last/last_row accuracy) + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepare_data(self): + tdSql.execute("create database db_td30816 cachemodel 'both';") + tdSql.execute("use db_td30816;") + # create regular table + tdSql.execute("create table rt_int (ts timestamp, c1 int primary key, c2 int);") + tdSql.execute("create table rt_str (ts timestamp, c1 varchar(16) primary key, c2 varchar(16));") + + # create stable + tdSql.execute("create table st_pk_int (ts timestamp, c1 int primary key, c2 int) tags (t1 int);") + tdSql.execute("create table st_pk_str (ts timestamp, c1 varchar(16) primary key, c2 varchar(16)) tags (t1 int);") + + # create child table + tdSql.execute("create table ct1 using st_pk_int tags(1);") + tdSql.execute("create table ct2 using st_pk_int tags(2);") + + tdSql.execute("create table ct3 using st_pk_str tags(3);") + tdSql.execute("create table ct4 using st_pk_str tags(4);") + + # insert data to regular table + tdSql.execute("insert into rt_int values ('2021-01-01 00:00:00', 1, NULL);") + tdSql.execute("insert into rt_int values ('2021-01-01 00:00:01', 2, 1);") + tdSql.execute("insert into rt_str values ('2021-01-01 00:00:00', 'a', NULL);") + tdSql.execute("insert into rt_str values ('2021-01-01 00:00:01', 'b', '1');") + + # insert data to child table + tdSql.execute("insert into ct1 values ('2021-01-01 00:00:00', 1, 1);") + tdSql.execute("insert into ct1 values ('2021-01-01 00:00:01', 2, NULL);") + tdSql.execute("insert into ct2 values ('2021-01-01 00:00:00', 3, 3);") + tdSql.execute("insert into ct2 values ('2021-01-01 00:00:01', 4, NULL);") + + tdSql.execute("insert into ct3 values ('2021-01-01 00:00:00', 'a', '1');") + tdSql.execute("insert into ct3 values ('2021-01-01 00:00:01', 'b', NULL);") + tdSql.execute("insert into ct4 values ('2021-01-01 00:00:00', 'c', '3');") + tdSql.execute("insert into ct4 values ('2021-01-01 00:00:01', 'd', NULL);") + + def test_last_with_primarykey_int_ct(self): + tdSql.execute("use db_td30816;") + tdSql.query("select last(*) from st_pk_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 3) + + tdSql.query("select last_row(*) from st_pk_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, None) + + # delete and insert data + tdSql.execute("delete from ct1 where ts='2021-01-01 00:00:01';") + tdSql.execute("delete from ct2 where ts='2021-01-01 00:00:01';") + tdSql.execute("insert into ct1 values ('2021-01-01 00:00:00', 0, 5);") + tdSql.execute("insert into ct2 values ('2021-01-01 00:00:00', -1, 6);") + tdSql.query("select last(*) from st_pk_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 3) + tdSql.checkData(0, 2, 3) + + tdSql.query("select last_row(*) from st_pk_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 3) + tdSql.checkData(0, 2, 3) + tdLog.info("Finish test_last_with_primarykey_int_ct") + + def test_last_with_primarykey_str_ct(self): + tdSql.execute("use db_td30816;") + tdSql.query("select last(*) from st_pk_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 'd') + tdSql.checkData(0, 2, '3') + + tdSql.query("select last_row(*) from st_pk_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 'd') + tdSql.checkData(0, 2, None) + + # delete and insert data + tdSql.execute("delete from ct3 where ts='2021-01-01 00:00:01';") + tdSql.execute("delete from ct4 where ts='2021-01-01 00:00:01';") + tdSql.execute("insert into ct3 values ('2021-01-01 00:00:00', '6', '5');") + tdSql.execute("insert into ct4 values ('2021-01-01 00:00:00', '7', '6');") + + tdSql.query("select last(*) from st_pk_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 'c') + tdSql.checkData(0, 2, '3') + + tdSql.query("select last_row(*) from st_pk_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 'c') + tdSql.checkData(0, 2, 3) + tdLog.info("Finish test_last_with_primarykey_str_ct") + + def test_last_with_primarykey_int_rt(self): + tdSql.execute("use db_td30816;") + tdSql.query("select last(*) from rt_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 2, 1) + + tdSql.query("select last_row(*) from rt_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 2, 1) + + tdSql.execute("delete from rt_int where ts='2021-01-01 00:00:01';") + tdSql.execute("insert into rt_int values ('2021-01-01 00:00:00', 0, 5);") + + tdSql.query("select last(*) from rt_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 5) + + tdSql.query("select last_row(*) from rt_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, None) + tdLog.info("Finish test_last_with_primarykey_int_rt") + + def test_last_with_primarykey_str_rt(self): + tdSql.execute("use db_td30816;") + tdSql.query("select last(*) from rt_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 'b') + tdSql.checkData(0, 2, '1') + + tdSql.query("select last_row(*) from rt_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:01') + tdSql.checkData(0, 1, 'b') + tdSql.checkData(0, 2, '1') + + tdSql.execute("delete from rt_str where ts='2021-01-01 00:00:01';") + tdSql.execute("insert into rt_str values ('2021-01-01 00:00:00', '2', '5');") + + tdSql.query("select last(*) from rt_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 'a') + tdSql.checkData(0, 2, '5') + + tdSql.query("select last_row(*) from rt_str;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2021-01-01 00:00:00') + tdSql.checkData(0, 1, 'a') + tdSql.checkData(0, 2, None) + tdLog.info("Finish test_last_with_primarykey_str_rt") + + def run(self): + self.prepare_data() + # regular table + self.test_last_with_primarykey_int_rt() + self.test_last_with_primarykey_str_rt() + # child tables + self.test_last_with_primarykey_int_ct() + self.test_last_with_primarykey_str_ct() + + def stop(self): + tdSql.execute("drop database db_td30816;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3553f0d913..09e22a7672 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -38,6 +38,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/queryBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f tmq/tmqBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_compare_asc_desc.py +,,y,army,./pytest.sh python3 ./test.py -f query/last/test_last.py # # system test From 30ff2fc11b5b8dc70d7ae4bf93c9ab3d623f9a60 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 19 Aug 2024 17:19:05 +0800 Subject: [PATCH 02/59] add trace log --- source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/thttp.c | 103 ++++++++++++++++++-------- 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 820075787f..e8172385b2 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -240,6 +240,7 @@ typedef struct { void* pThrd; queue qmsg; TdThreadMutex mtx; // protect qmsg; + int64_t num; } SAsyncItem; typedef struct { diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 908468f094..6dee6aee09 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -30,6 +30,8 @@ static int32_t FAST_FAILURE_LIMIT = 1; static int64_t httpDefaultChanId = -1; +static int64_t httpSeqNum = 0; + typedef struct SHttpModule { uv_loop_t* loop; SAsyncPool* asyncPool; @@ -50,6 +52,7 @@ typedef struct SHttpMsg { EHttpCompFlag flag; int8_t quit; int64_t chanId; + int64_t seq; } SHttpMsg; typedef struct SHttpClient { @@ -62,6 +65,7 @@ typedef struct SHttpClient { uint16_t port; struct sockaddr_in dest; int64_t chanId; + int64_t seq; } SHttpClient; typedef struct SHttpConnList { @@ -215,14 +219,15 @@ static void* httpThread(void* arg) { static int32_t httpCreateMsg(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag, int64_t chanId, SHttpMsg** httpMsg) { + int64_t seqNum = atomic_fetch_add_64(&httpSeqNum, 1); if (server == NULL || uri == NULL) { - tError("http-report failed to report to invalid addr, chanId:%" PRId64 "", chanId); + tError("http-report failed to report to invalid addr, chanId:%" PRId64 ", seq:%" PRId64 "", chanId, seqNum); *httpMsg = NULL; return TSDB_CODE_INVALID_PARA; } if (pCont == NULL || contLen == 0) { - tError("http-report failed to report empty packet, chanId:%" PRId64 "", chanId); + tError("http-report failed to report empty packet, chanId:%" PRId64 ", seq:%" PRId64 "", chanId, seqNum); *httpMsg = NULL; return TSDB_CODE_INVALID_PARA; } @@ -233,6 +238,7 @@ static int32_t httpCreateMsg(const char* server, const char* uri, uint16_t port, return TSDB_CODE_OUT_OF_MEMORY; } + msg->seq = seqNum; msg->port = port; msg->server = taosStrdup(server); msg->uri = taosStrdup(uri); @@ -259,7 +265,11 @@ static void httpDestroyMsg(SHttpMsg* msg) { taosMemoryFree(msg->cont); taosMemoryFree(msg); } -static void httpDestroyMsgWrapper(void* cont, void* param) { httpDestroyMsg((SHttpMsg*)cont); } +static void httpDestroyMsgWrapper(void* cont, void* param) { + SHttpMsg* pMsg = cont; + tWarn("http-report destroy msg, seq: %" PRId64 "", pMsg->seq); + httpDestroyMsg(pMsg); +} static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { SHttpMsg *msg = NULL, *quitMsg = NULL; @@ -272,6 +282,8 @@ static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { QUEUE_REMOVE(h); msg = QUEUE_DATA(h, SHttpMsg, q); if (!msg->quit) { + tError("http-report failed to report chanId:%" PRId64 ",seq:%" PRId64 ", reason: %s", msg->chanId, msg->seq, + tstrerror(TSDB_CODE_HTTP_MODULE_QUIT)); httpDestroyMsg(msg); } else { quitMsg = msg; @@ -281,6 +293,25 @@ static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { QUEUE_PUSH(&item->qmsg, &quitMsg->q); } } +static void httpTrace(queue* q) { + if (!(rpcDebugFlag & DEBUG_DEBUG) || (QUEUE_IS_EMPTY(q))) { + return; + } + + int64_t startSeq = 0, endSeq = 0; + SHttpMsg* msg = NULL; + + queue* h = QUEUE_HEAD(q); + msg = QUEUE_DATA(h, SHttpMsg, q); + startSeq = msg->seq; + + h = QUEUE_TAIL(q); + msg = QUEUE_DATA(h, SHttpMsg, q); + endSeq = msg->seq; + + tDebug("http-report process msg, start_seq: %" PRId64 ", end_seq: %" PRId64 ", max_seq:%" PRId64 "", startSeq, endSeq, + atomic_load_64(&httpSeqNum) - 1); +} static void httpAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; @@ -290,7 +321,7 @@ static void httpAsyncCb(uv_async_t* handle) { queue wq; QUEUE_INIT(&wq); - static int32_t BATCH_SIZE = 5; + static int32_t BATCH_SIZE = 20; int32_t count = 0; (void)taosThreadMutexLock(&item->mtx); @@ -303,6 +334,8 @@ static void httpAsyncCb(uv_async_t* handle) { } (void)taosThreadMutexUnlock(&item->mtx); + httpTrace(&wq); + while (!QUEUE_IS_EMPTY(&wq)) { queue* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); @@ -347,9 +380,9 @@ static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { SHttpClient* cli = handle->data; if (nread < 0) { - tError("http-report recv error:%s", uv_strerror(nread)); + tError("http-report recv error:%s, seq: %" PRId64 "", uv_strerror(nread), cli->seq); } else { - tTrace("http-report succ to recv %d bytes", (int32_t)nread); + tTrace("http-report succ to recv %d bytes, seq: %" PRId64 "", (int32_t)nread, cli->seq); } if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -358,19 +391,19 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const static void clientSentCb(uv_write_t* req, int32_t status) { SHttpClient* cli = req->data; if (status != 0) { - tError("http-report failed to send data, reason: %s, dst:%s:%d, chanId:%" PRId64 "", uv_strerror(status), cli->addr, - cli->port, cli->chanId); + tError("http-report failed to send data, reason: %s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + uv_strerror(status), cli->addr, cli->port, cli->chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } return; } else { - tTrace("http-report succ to send data, chanId:%" PRId64 "", cli->chanId); + tTrace("http-report succ to send data, chanId:%" PRId64 ", seq: %" PRId64 "", cli->chanId, cli->seq); } status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); if (status != 0) { - tError("http-report failed to recv data,reason:%s, dst:%s:%d, chanId:%" PRId64 "", uv_strerror(status), cli->addr, - cli->port, cli->chanId); + tError("http-report failed to recv data,reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + uv_strerror(status), cli->addr, cli->port, cli->chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } @@ -383,9 +416,8 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { SHttpModule* http = taosAcquireRef(httpRefMgt, chanId); if (status != 0) { httpFailFastMayUpdate(http->connStatusTable, cli->addr, cli->port, 0); - - tError("http-report failed to conn to server, reason:%s, dst:%s:%d, chanId:%" PRId64 "", uv_strerror(status), - cli->addr, cli->port, chanId); + tError("http-report failed to conn to server, reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + uv_strerror(status), cli->addr, cli->port, chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } @@ -398,8 +430,8 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); if (0 != status) { - tError("http-report failed to send data,reason:%s, dst:%s:%d, chanId:%" PRId64 "", uv_strerror(status), cli->addr, - cli->port, chanId); + tError("http-report failed to send data,reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + uv_strerror(status), cli->addr, cli->port, chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } @@ -412,7 +444,7 @@ int32_t httpSendQuit(SHttpModule* http, int64_t chanId) { if (msg == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - + msg->seq = atomic_fetch_add_64(&httpSeqNum, 1); msg->quit = 1; msg->chanId = chanId; @@ -442,10 +474,11 @@ static void httpWalkCb(uv_handle_t* handle, void* arg) { return; } static void httpHandleQuit(SHttpMsg* msg) { + int64_t seq = msg->seq; int64_t chanId = msg->chanId; taosMemoryFree(msg); - tDebug("http-report receive quit, chanId:%" PRId64 "", chanId); + tDebug("http-report receive quit, chanId: %" PRId64 ", seq: %" PRId64 "", chanId, seq); SHttpModule* http = taosAcquireRef(httpRefMgt, chanId); if (http == NULL) return; uv_walk(http->loop, httpWalkCb, NULL); @@ -542,7 +575,7 @@ static void httpHandleReq(SHttpMsg* msg) { code = TSDB_CODE_OUT_OF_MEMORY; goto END; } - + cli->seq = msg->seq; cli->conn.data = cli; cli->tcp.data = cli; cli->req.data = cli; @@ -556,8 +589,8 @@ static void httpHandleReq(SHttpMsg* msg) { cli->wbuf = wb; cli->rbuf = taosMemoryCalloc(1, HTTP_RECV_BUF_SIZE); if (cli->rbuf == NULL) { - tError("http-report failed to alloc read buf, dst:%s:%d,chanId:%" PRId64 ", reason:%s", cli->addr, cli->port, - chanId, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + tError("http-report failed to alloc read buf, dst:%s:%d,chanId:%" PRId64 ", seq:%" PRId64 ",reason:%s", cli->addr, + cli->port, chanId, cli->seq, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); return; @@ -565,8 +598,8 @@ static void httpHandleReq(SHttpMsg* msg) { int err = uv_tcp_init(http->loop, &cli->tcp); if (err != 0) { - tError("http-report failed to init socket handle, dst:%s:%d,chanId:%" PRId64 ", reason:%s", cli->addr, cli->port, - chanId, uv_strerror(err)); + tError("http-report failed to init socket handle, dst:%s:%d,chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", + cli->addr, cli->port, chanId, cli->seq, uv_strerror(err)); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); return; @@ -575,8 +608,8 @@ static void httpHandleReq(SHttpMsg* msg) { // set up timeout to avoid stuck; int32_t fd = taosCreateSocketWithTimeout(5000); if (fd < 0) { - tError("http-report failed to open socket, dst:%s:%d, chanId:%" PRId64 ", reason:%s", cli->addr, cli->port, chanId, - tstrerror(TAOS_SYSTEM_ERROR(errno))); + tError("http-report failed to open socket, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", cli->addr, + cli->port, chanId, cli->seq, tstrerror(TAOS_SYSTEM_ERROR(errno))); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); return; @@ -584,8 +617,8 @@ static void httpHandleReq(SHttpMsg* msg) { int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); if (ret != 0) { - tError("http-report failed to open socket, reason:%s, dst:%s:%d, chanId:%" PRId64 ",reason:%s", uv_strerror(ret), - cli->addr, cli->port, chanId, uv_strerror(ret)); + tError("http-report failed to open socket, reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", + uv_strerror(ret), cli->addr, cli->port, chanId, cli->seq, uv_strerror(ret)); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); return; @@ -593,8 +626,8 @@ static void httpHandleReq(SHttpMsg* msg) { ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&cli->dest, clientConnCb); if (ret != 0) { - tError("http-report failed to connect to http-server,dst:%s:%d, chanId:%" PRId64 ",reson:%s", cli->addr, cli->port, - chanId, uv_strerror(ret)); + tError("http-report failed to connect to http-server,dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ",reson:%s", + cli->addr, cli->port, chanId, cli->seq, uv_strerror(ret)); httpFailFastMayUpdate(http->connStatusTable, cli->addr, cli->port, 0); destroyHttpClient(cli); } @@ -603,8 +636,8 @@ static void httpHandleReq(SHttpMsg* msg) { END: if (ignore == false) { - tError("http-report failed to report to addr: %s:%d, chanId:%" PRId64 ", reason:%s", msg->server, msg->port, chanId, - tstrerror(code)); + tError("http-report failed to report to addr: %s:%d, chanId:%" PRId64 ",seq:%" PRId64 " reason:%s", msg->server, + msg->port, chanId, msg->seq, tstrerror(code)); } httpDestroyMsg(msg); taosMemoryFree(header); @@ -661,6 +694,11 @@ static int32_t taosSendHttpReportImplByChan(const char* server, const char* uri, msg = NULL; _ERROR: + + if (code != 0) { + tError("http-report failed to report reason:%s, chanId:%" PRId64 ", seq:%" PRId64 "", tstrerror(code), chanId, + msg->seq); + } httpDestroyMsg(msg); if (load != NULL) taosReleaseRef(httpRefMgt, chanId); return code; @@ -684,6 +722,7 @@ int64_t transInitHttpChanImpl(); static void transHttpEnvInit() { httpRefMgt = taosOpenRef(64, transHttpDestroyHandle); httpDefaultChanId = transInitHttpChanImpl(); + httpSeqNum = 0; } void transHttpEnvDestroy() { @@ -772,4 +811,4 @@ void taosDestroyHttpChan(int64_t chanId) { (void)taosReleaseRef(httpRefMgt, chanId); (void)taosRemoveRef(httpRefMgt, chanId); -} \ No newline at end of file +} From feb23ba56ba9a44e84e5e2106605402a495d729b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 19 Aug 2024 17:27:01 +0800 Subject: [PATCH 03/59] add trace log --- source/libs/transport/src/thttp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6dee6aee09..41af0ab0c9 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -685,6 +685,7 @@ static int32_t taosSendHttpReportImplByChan(const char* server, const char* uri, code = TSDB_CODE_HTTP_MODULE_QUIT; goto _ERROR; } + tDebug("http-report start to report,chanId:%" PRId64 ", seq:%" PRId64 "", chanId, msg->seq); code = transAsyncSend(load->asyncPool, &(msg->q)); if (code != 0) { From bef98f4787da5a6c396aebec2b56bd073256c526 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 19 Aug 2024 17:39:53 +0800 Subject: [PATCH 04/59] add trace log --- source/libs/transport/src/thttp.c | 39 ++++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 41af0ab0c9..dbc3dbbb58 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -197,14 +197,14 @@ static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, uint32_t ip = 0; int32_t code = taosGetIpv4FromFqdn(server, &ip); if (code) { - tError("http-report failed to resolving domain names: %s", server); + tError("http-report failed to resolving domain names %s, reason: %s", server, tstrerror(code)); return TSDB_CODE_RPC_FQDN_ERROR; } char buf[256] = {0}; tinet_ntoa(buf, ip); int ret = uv_ip4_addr(buf, port, dest); if (ret != 0) { - tError("http-report failed to get addr %s", uv_err_name(ret)); + tError("http-report failed to get addr, reason:%s", uv_err_name(ret)); return TSDB_CODE_THIRDPARTY_ERROR; } return 0; @@ -267,7 +267,7 @@ static void httpDestroyMsg(SHttpMsg* msg) { } static void httpDestroyMsgWrapper(void* cont, void* param) { SHttpMsg* pMsg = cont; - tWarn("http-report destroy msg, seq: %" PRId64 "", pMsg->seq); + tWarn("http-report destroy msg, chanId:%" PRId64 ", seq:%" PRId64 "", pMsg->chanId, pMsg->seq); httpDestroyMsg(pMsg); } @@ -309,7 +309,7 @@ static void httpTrace(queue* q) { msg = QUEUE_DATA(h, SHttpMsg, q); endSeq = msg->seq; - tDebug("http-report process msg, start_seq: %" PRId64 ", end_seq: %" PRId64 ", max_seq:%" PRId64 "", startSeq, endSeq, + tDebug("http-report process msg, start_seq:%" PRId64 ", end_seq:%" PRId64 ", max_seq:%" PRId64 "", startSeq, endSeq, atomic_load_64(&httpSeqNum) - 1); } @@ -380,9 +380,9 @@ static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { SHttpClient* cli = handle->data; if (nread < 0) { - tError("http-report recv error:%s, seq: %" PRId64 "", uv_strerror(nread), cli->seq); + tError("http-report recv error:%s, seq:%" PRId64 "", uv_strerror(nread), cli->seq); } else { - tTrace("http-report succ to recv %d bytes, seq: %" PRId64 "", (int32_t)nread, cli->seq); + tTrace("http-report succ to recv %d bytes, seq:%" PRId64 "", (int32_t)nread, cli->seq); } if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -391,18 +391,18 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const static void clientSentCb(uv_write_t* req, int32_t status) { SHttpClient* cli = req->data; if (status != 0) { - tError("http-report failed to send data, reason: %s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + tError("http-report failed to send data, reason: %s, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 "", uv_strerror(status), cli->addr, cli->port, cli->chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } return; } else { - tTrace("http-report succ to send data, chanId:%" PRId64 ", seq: %" PRId64 "", cli->chanId, cli->seq); + tTrace("http-report succ to send data, chanId:%" PRId64 ", seq:%" PRId64 "", cli->chanId, cli->seq); } status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); if (status != 0) { - tError("http-report failed to recv data,reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + tError("http-report failed to recv data,reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 "", uv_strerror(status), cli->addr, cli->port, cli->chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -416,7 +416,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { SHttpModule* http = taosAcquireRef(httpRefMgt, chanId); if (status != 0) { httpFailFastMayUpdate(http->connStatusTable, cli->addr, cli->port, 0); - tError("http-report failed to conn to server, reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + tError("http-report failed to conn to server, reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 "", uv_strerror(status), cli->addr, cli->port, chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -430,7 +430,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); if (0 != status) { - tError("http-report failed to send data,reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq: %" PRId64 "", + tError("http-report failed to send data,reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 "", uv_strerror(status), cli->addr, cli->port, chanId, cli->seq); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -478,7 +478,7 @@ static void httpHandleQuit(SHttpMsg* msg) { int64_t chanId = msg->chanId; taosMemoryFree(msg); - tDebug("http-report receive quit, chanId: %" PRId64 ", seq: %" PRId64 "", chanId, seq); + tDebug("http-report receive quit, chanId:%" PRId64 ", seq:%" PRId64 "", chanId, seq); SHttpModule* http = taosAcquireRef(httpRefMgt, chanId); if (http == NULL) return; uv_walk(http->loop, httpWalkCb, NULL); @@ -589,7 +589,7 @@ static void httpHandleReq(SHttpMsg* msg) { cli->wbuf = wb; cli->rbuf = taosMemoryCalloc(1, HTTP_RECV_BUF_SIZE); if (cli->rbuf == NULL) { - tError("http-report failed to alloc read buf, dst:%s:%d,chanId:%" PRId64 ", seq:%" PRId64 ",reason:%s", cli->addr, + tError("http-report failed to alloc read buf, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ",reason:%s", cli->addr, cli->port, chanId, cli->seq, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); @@ -598,7 +598,7 @@ static void httpHandleReq(SHttpMsg* msg) { int err = uv_tcp_init(http->loop, &cli->tcp); if (err != 0) { - tError("http-report failed to init socket handle, dst:%s:%d,chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", + tError("http-report failed to init socket handle, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", cli->addr, cli->port, chanId, cli->seq, uv_strerror(err)); destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); @@ -617,8 +617,9 @@ static void httpHandleReq(SHttpMsg* msg) { int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); if (ret != 0) { - tError("http-report failed to open socket, reason:%s, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", - uv_strerror(ret), cli->addr, cli->port, chanId, cli->seq, uv_strerror(ret)); + tError("http-report failed to open socket, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", cli->addr, + cli->port, chanId, cli->seq, uv_strerror(ret)); + destroyHttpClient(cli); (void)taosReleaseRef(httpRefMgt, chanId); return; @@ -626,7 +627,7 @@ static void httpHandleReq(SHttpMsg* msg) { ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&cli->dest, clientConnCb); if (ret != 0) { - tError("http-report failed to connect to http-server,dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ",reson:%s", + tError("http-report failed to connect to http-server,dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reson:%s", cli->addr, cli->port, chanId, cli->seq, uv_strerror(ret)); httpFailFastMayUpdate(http->connStatusTable, cli->addr, cli->port, 0); destroyHttpClient(cli); @@ -685,7 +686,7 @@ static int32_t taosSendHttpReportImplByChan(const char* server, const char* uri, code = TSDB_CODE_HTTP_MODULE_QUIT; goto _ERROR; } - tDebug("http-report start to report,chanId:%" PRId64 ", seq:%" PRId64 "", chanId, msg->seq); + tDebug("http-report start to report, chanId:%" PRId64 ", seq:%" PRId64 "", chanId, msg->seq); code = transAsyncSend(load->asyncPool, &(msg->q)); if (code != 0) { @@ -788,7 +789,7 @@ int64_t taosInitHttpChan() { } void taosDestroyHttpChan(int64_t chanId) { - tDebug("http-report send quit, chanId:%" PRId64 "", chanId); + tDebug("http-report send quit, chanId: %" PRId64 "", chanId); int ret = 0; SHttpModule* load = taosAcquireRef(httpRefMgt, chanId); From 03a72ea6fbc4804d7992574b983bc12afe31c27b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 20 Aug 2024 17:25:12 +0800 Subject: [PATCH 05/59] remove ASSERT --- source/dnode/vnode/src/meta/metaTable.c | 64 +++++++++++--- source/libs/command/src/command.c | 72 +++++----------- source/libs/index/src/index.c | 27 +++++- source/libs/index/src/indexComm.c | 86 +------------------ source/libs/index/src/indexFilter.c | 12 ++- source/libs/index/src/indexFst.c | 54 ++++++------ source/libs/index/src/indexFstFile.c | 13 +-- source/libs/index/src/indexFstNode.c | 3 +- source/libs/index/src/indexFstRegister.c | 5 +- source/libs/index/src/indexTfile.c | 33 ++++--- source/libs/stream/src/streamBackendRocksdb.c | 7 +- source/libs/transport/src/transCli.c | 2 - source/libs/transport/src/transComm.c | 1 - source/libs/transport/src/transSvr.c | 43 ++++++---- 14 files changed, 200 insertions(+), 222 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 21602627e9..cbd4f298d2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -129,7 +129,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const } SIndexMultiTerm *terms = indexMultiTermCreate(); - int16_t nCols = taosArrayGetSize(pTagVals); + if (terms == NULL) { + return terrno; + } + + int16_t nCols = taosArrayGetSize(pTagVals); for (int i = 0; i < nCols; i++) { STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); char type = pTagVal->type; @@ -142,8 +146,14 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); } else if (type == TSDB_DATA_TYPE_NCHAR) { if (pTagVal->nData > 0) { - char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + if (val == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _except); + } int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + if (len < 0) { + TAOS_CHECK_GOTO(len, NULL, _except); + } memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len); @@ -160,8 +170,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const int len = sizeof(val); term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); } + if (term != NULL) { (void)indexMultiTermAdd(terms, term); + } else { + code = terrno; } } (void)indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); @@ -170,6 +183,10 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const taosArrayDestroy(pTagVals); #endif return 0; +_except: + indexMultiTermDestroy(terms); + taosArrayDestroy(pTagVals); + return code; } int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { #ifdef USE_INVERTED_INDEX @@ -191,7 +208,11 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche } SIndexMultiTerm *terms = indexMultiTermCreate(); - int16_t nCols = taosArrayGetSize(pTagVals); + if (terms == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int16_t nCols = taosArrayGetSize(pTagVals); for (int i = 0; i < nCols; i++) { STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); char type = pTagVal->type; @@ -204,8 +225,14 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); } else if (type == TSDB_DATA_TYPE_NCHAR) { if (pTagVal->nData > 0) { - char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + if (val == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _except); + } int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + if (len < 0) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _except); + } memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len); @@ -224,6 +251,8 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche } if (term != NULL) { (void)indexMultiTermAdd(terms, term); + } else { + code = terrno; } } (void)indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); @@ -231,6 +260,10 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche taosArrayDestroy(pTagVals); #endif return 0; +_except: + indexMultiTermDestroy(terms); + taosArrayDestroy(pTagVals); + return code; } static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { @@ -867,7 +900,6 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) goto _err; } - nStbEntry.stbEntry.schemaRow = *row; nStbEntry.stbEntry.schemaTag = *tag; nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; @@ -1580,7 +1612,12 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl : pAlterTbReq->compress; (void)updataTableColCmpr(&entry.colCmpr, pCol, 1, compress); freeColCmpr = true; - ASSERT(entry.colCmpr.nCols == pSchema->nCols); + if (entry.colCmpr.nCols != pSchema->nCols) { + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } break; case TSDB_ALTER_TABLE_DROP_COLUMN: if (pColumn == NULL) { @@ -1617,7 +1654,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl } (void)updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0); - ASSERT(entry.colCmpr.nCols == pSchema->nCols); + if (entry.colCmpr.nCols != pSchema->nCols) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: if (pColumn == NULL) { @@ -1698,7 +1738,7 @@ _err: (void)tdbTbcClose(pUidIdxc); tDecoderClear(&dc); - return TSDB_CODE_FAILED; + return terrno != 0 ? terrno : TSDB_CODE_FAILED; } static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { @@ -2514,8 +2554,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) { metaError("vgId:%d, failed to get stable suid for update. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - ret = -1; + ret = TSDB_CODE_TDB_INVALID_TABLE_ID; goto end; } tbDbKey.uid = pCtbEntry->ctbEntry.suid; @@ -2529,6 +2568,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { } if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { + ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -2573,7 +2613,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { } } end: - // metaDestroyTagIdxKey(pTagIdxKey); + if (terrno != 0) { + ret = terrno; + } tDecoderClear(&dc); tdbFree(pData); return ret; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 11ddc89d4c..26f8b00040 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -24,13 +24,13 @@ #include "tgrant.h" #define COL_DATA_SET_VAL_AND_CHECK(pCol, rows, buf, isNull) \ - do { \ - int _code = colDataSetVal(pCol, rows, buf, isNull);\ - if (TSDB_CODE_SUCCESS != _code) { \ - terrno = _code; \ - return _code; \ - } \ - } while(0) + do { \ + int _code = colDataSetVal(pCol, rows, buf, isNull); \ + if (TSDB_CODE_SUCCESS != _code) { \ + terrno = _code; \ + return _code; \ + } \ + } while (0) extern SConfig* tsCfg; @@ -77,7 +77,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) { QRY_OPTR_CHECK(pOutput); SSDataBlock* pBlock = NULL; - int32_t code = createDataBlock(&pBlock); + int32_t code = createDataBlock(&pBlock); if (code) { return code; } @@ -235,7 +235,7 @@ static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) { QRY_OPTR_CHECK(pOutput); SSDataBlock* pBlock = NULL; - int32_t code = createDataBlock(&pBlock); + int32_t code = createDataBlock(&pBlock); if (code) { return code; } @@ -280,15 +280,15 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { return v; } -static int32_t buildRetension(SArray* pRetension, char **ppRetentions ) { +static int32_t buildRetension(SArray* pRetension, char** ppRetentions) { size_t size = taosArrayGetSize(pRetension); if (size == 0) { *ppRetentions = NULL; return TSDB_CODE_SUCCESS; } - char* p1 = taosMemoryCalloc(1, 100); - if(NULL == p1) { + char* p1 = taosMemoryCalloc(1, 100); + if (NULL == p1) { return terrno; } int32_t len = 0; @@ -368,7 +368,7 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, break; } - char* pRetentions = NULL; + char* pRetentions = NULL; QRY_ERR_RET(buildRetension(pCfg->pRetensions, &pRetentions)); int32_t dbFNameLen = strlen(dbFName); int32_t hashPrefix = 0; @@ -427,7 +427,7 @@ static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) { QRY_OPTR_CHECK(pOutput); SSDataBlock* pBlock = NULL; - int32_t code = createDataBlock(&pBlock); + int32_t code = createDataBlock(&pBlock); if (code) { return code; } @@ -451,7 +451,7 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) { QRY_OPTR_CHECK(pOutput); SSDataBlock* pBlock = NULL; - int32_t code = createDataBlock(&pBlock); + int32_t code = createDataBlock(&pBlock); if (code) { return code; } @@ -578,36 +578,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { } else { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); } - - /* - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) { - if (pTagVal->nData > 0) { - if (num) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); - } - - memcpy(buf + VARSTR_HEADER_SIZE + *len, pTagVal->pData, pTagVal->nData); - *len += pTagVal->nData; - } - } else if (type == TSDB_DATA_TYPE_NCHAR) { - if (pTagVal->nData > 0) { - if (num) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); - } - int32_t tlen = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, buf + VARSTR_HEADER_SIZE + *len); - } - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double val = *(double *)(&pTagVal->i64); - int len = 0; - term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); - } else if (type == TSDB_DATA_TYPE_BOOL) { - int val = *(int *)(&pTagVal->i64); - int len = 0; - term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len); - } - */ } - _exit: taosArrayDestroy(pTagVals); @@ -864,26 +835,26 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN; - if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { + if (taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { goto _exit; } infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN; - if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { + if (taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { goto _exit; } infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN; - if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { + if (taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { goto _exit; } *pOutput = pBlock; _exit: - if(terrno != TSDB_CODE_SUCCESS) { + if (terrno != TSDB_CODE_SUCCESS) { taosMemoryFree(pBlock); taosArrayDestroy(pBlock->pDataBlock); } @@ -907,7 +878,7 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p QRY_OPTR_CHECK(pOutput); SSDataBlock* pBlock = NULL; - int32_t code = createDataBlock(&pBlock); + int32_t code = createDataBlock(&pBlock); if (code) { return code; } @@ -942,7 +913,8 @@ int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) { if (((SValueNode*)pProj)->isNull) { QRY_ERR_RET(colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true)); } else { - QRY_ERR_RET(colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false)); + QRY_ERR_RET(colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, + nodesGetValueFromNode((SValueNode*)pProj), false)); } } } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b881d2cac8..993b74ba94 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -237,8 +237,7 @@ int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { indexDebug("w suid:%" PRIu64 ", colName:%s, colType:%d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); - ASSERTS(*cache != NULL, "index-cache already release"); - if (*cache == NULL) return -1; + if (*cache == NULL) return TSDB_CODE_INVALID_PTR; int ret = idxCachePut(*cache, p, uid); if (ret != 0) { @@ -295,7 +294,9 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { }; int32_t indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { SIndexTermQuery q = {.qType = qType, .term = term}; - (void)taosArrayPush(pQuery->query, &q); + if (taosArrayPush(pQuery->query, &q) == NULL) { + return terrno; + } return 0; } @@ -303,6 +304,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (tm == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -311,6 +313,10 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->colType = colType; tm->colName = (char*)taosMemoryCalloc(1, nColName + 1); + if (tm->colName == NULL) { + taosMemoryFree(tm); + return NULL; + } memcpy(tm->colName, colName, nColName); tm->nColName = nColName; @@ -326,8 +332,23 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy buf = strndup(emptyStr, (int32_t)strlen(emptyStr)); len = (int32_t)strlen(emptyStr); } + tm->colVal = buf; + if (tm->colVal == NULL) { + taosMemoryFree(tm->colName); + taosMemoryFree(tm); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + tm->nColVal = len; + if (tm->nColVal < 0) { + taosMemoryFree(tm->colName); + taosMemoryFree(tm->colVal); + taosMemoryFree(tm); + terrno = len; + return NULL; + } return tm; } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 6337b2c556..ec572700ae 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -290,90 +290,6 @@ int idxUidCompare(const void* a, const void* b) { uint64_t r = *(uint64_t*)b; return l - r; } -#ifdef BUILD_NO_CALL -int32_t idxConvertData(void* src, int8_t type, void** dst) { - int tlen = -1; - switch (type) { - case TSDB_DATA_TYPE_TIMESTAMP: - tlen = taosEncodeFixedI64(NULL, *(int64_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedI64(dst, *(int64_t*)src); - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedU8(dst, *(uint8_t*)src); - break; - case TSDB_DATA_TYPE_TINYINT: - tlen = taosEncodeFixedI8(NULL, *(uint8_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedI8(dst, *(uint8_t*)src); - break; - case TSDB_DATA_TYPE_SMALLINT: - tlen = taosEncodeFixedI16(NULL, *(int16_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedI16(dst, *(int16_t*)src); - break; - case TSDB_DATA_TYPE_USMALLINT: - tlen = taosEncodeFixedU16(NULL, *(uint16_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedU16(dst, *(uint16_t*)src); - break; - case TSDB_DATA_TYPE_INT: - tlen = taosEncodeFixedI32(NULL, *(int32_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedI32(dst, *(int32_t*)src); - break; - case TSDB_DATA_TYPE_FLOAT: - tlen = taosEncodeBinary(NULL, src, sizeof(float)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, sizeof(float)); - break; - case TSDB_DATA_TYPE_UINT: - tlen = taosEncodeFixedU32(NULL, *(uint32_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedU32(dst, *(uint32_t*)src); - break; - case TSDB_DATA_TYPE_BIGINT: - tlen = taosEncodeFixedI64(NULL, *(int64_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedI64(dst, *(int64_t*)src); - break; - case TSDB_DATA_TYPE_DOUBLE: - tlen = taosEncodeBinary(NULL, src, sizeof(double)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, sizeof(double)); - break; - case TSDB_DATA_TYPE_UBIGINT: - tlen = taosEncodeFixedU64(NULL, *(uint64_t*)src); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedU64(dst, *(uint64_t*)src); - break; - case TSDB_DATA_TYPE_NCHAR: { - tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - - break; - } - case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_GEOMETRY: { - tlen = taosEncodeBinary(NULL, src, strlen(src)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, strlen(src)); - break; - } - default: - ASSERTS(0, "index invalid input type"); - break; - } - *dst = (char*)*dst - tlen; - // indexMayFillNumbericData(*dst, tlen); - return tlen; -} -#endif int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { if (src == NULL) { @@ -495,7 +411,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { break; } default: - ASSERTS(0, "index invalid input type"); + tlen = TSDB_CODE_INVALID_DATA_FMT; break; } return tlen; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index c362932da3..483920ba4f 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -646,7 +646,16 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP } SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); - (void)indexMultiTermQueryAdd(mtm, tm, qtype); + if (mtm == NULL) { + indexTermDestroy(tm); + return TSDB_CODE_OUT_OF_MEMORY; + } + + if ((ret = indexMultiTermQueryAdd(mtm, tm, qtype)) != 0) { + indexMultiTermQueryDestroy(mtm); + return ret; + } + ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); indexMultiTermQueryDestroy(mtm); } else { @@ -1104,7 +1113,6 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, *status = st; } - if (taosArrayAddAll(result, param.result) == NULL) { sifFreeParam(¶m); return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 54d28f4784..e57ba6267d 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -98,7 +98,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output } int32_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); - ASSERTS(un->last == NULL, "index-fst meet unexpected node"); + if (un->last != NULL) return; // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); @@ -318,7 +318,7 @@ void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode // set_comm_input void fstStateSetCommInput(FstState* s, uint8_t inp) { - ASSERT(s->state == OneTransNext || s->state == OneTrans); + // ASSERT(s->state == OneTransNext || s->state == OneTrans); uint8_t val; COMMON_INDEX(inp, 0b111111, val); @@ -327,7 +327,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) { // comm_input uint8_t fstStateCommInput(FstState* s, bool* null) { - ASSERT(s->state == OneTransNext || s->state == OneTrans); + // ASSERT(s->state == OneTransNext || s->state == OneTrans); uint8_t v = s->val & 0b00111111; if (v == 0) { *null = true; @@ -340,7 +340,7 @@ uint8_t fstStateCommInput(FstState* s, bool* null) { // input_len uint64_t fstStateInputLen(FstState* s) { - ASSERT(s->state == OneTransNext || s->state == OneTrans); + // ASSERT(s->state == OneTransNext || s->state == OneTrans); bool null = false; (void)fstStateCommInput(s, &null); return null ? 1 : 0; @@ -348,11 +348,11 @@ uint64_t fstStateInputLen(FstState* s) { // end_addr uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { - ASSERT(s->state == OneTransNext); + // ASSERT(s->state == OneTransNext); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); } uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { - ASSERT(s->state == OneTrans); + // ASSERT(s->state == OneTrans); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); } @@ -366,7 +366,7 @@ uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* } // input uint8_t fstStateInput(FstState* s, FstNode* node) { - ASSERT(s->state == OneTransNext || s->state == OneTrans); + // ASSERT(s->state == OneTransNext || s->state == OneTrans); FstSlice* slice = &node->data; bool null = false; uint8_t inp = fstStateCommInput(s, &null); @@ -374,7 +374,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) { return null == false ? inp : data[node->start - 1]; } uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -386,7 +386,7 @@ uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { // trans_addr CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { - ASSERT(s->state == OneTransNext || s->state == OneTrans); + // ASSERT(s->state == OneTransNext || s->state == OneTrans); FstSlice* slice = &node->data; if (s->state == OneTransNext) { return (CompiledAddr)(node->end) - 1; @@ -402,7 +402,7 @@ CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { } } CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); @@ -414,7 +414,7 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i // sizes PackSizes fstStateSizes(FstState* s, FstSlice* slice) { - ASSERT(s->state == OneTrans || s->state == AnyTrans); + /// ASSERT(s->state == OneTrans || s->state == AnyTrans); uint64_t i; if (s->state == OneTrans) { i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; @@ -427,7 +427,7 @@ PackSizes fstStateSizes(FstState* s, FstSlice* slice) { } // Output Output fstStateOutput(FstState* s, FstNode* node) { - ASSERT(s->state == OneTrans); + // ASSERT(s->state == OneTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { @@ -441,7 +441,7 @@ Output fstStateOutput(FstState* s, FstNode* node) { return unpackUint64(data + i, oSizes); } Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { @@ -458,19 +458,19 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { // anyTrans specify function void fstStateSetFinalState(FstState* s, bool yes) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); if (yes) { s->val |= 0b01000000; } return; } bool fstStateIsFinalState(FstState* s) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); return (s->val & 0b01000000) == 0b01000000; } void fstStateSetStateNtrans(FstState* s, uint8_t n) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } @@ -478,7 +478,7 @@ void fstStateSetStateNtrans(FstState* s, uint8_t n) { } // state_ntrans uint8_t fstStateStateNtrans(FstState* s, bool* null) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); *null = false; uint8_t n = s->val & 0b00111111; @@ -488,16 +488,16 @@ uint8_t fstStateStateNtrans(FstState* s, bool* null) { return n; } uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; } uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; } uint64_t fstStateNtransLen(FstState* s) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); bool null = false; (void)fstStateStateNtrans(s, &null); return null == true ? 1 : 0; @@ -526,7 +526,7 @@ Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackS return unpackUint64(data + at, (uint8_t)oSizes); } uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { - ASSERT(s->state == AnyTrans); + // ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -672,17 +672,17 @@ bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { bool s = true; FstState* st = &node->state; if (st->state == OneTransNext) { - ASSERT(i == 0); + /// ASSERT(i == 0); (void)fstStateTransAddr(st, node); } else if (st->state == OneTrans) { - ASSERT(i == 0); + // ASSERT(i == 0); (void)fstStateTransAddr(st, node); } else if (st->state == AnyTrans) { (void)fstStateTransAddrForAnyTrans(st, node, i); } else if (FST_STATE_EMPTY_FINAL(node)) { s = false; } else { - ASSERT(0); + // ASSERT(0); } return s; } @@ -718,7 +718,7 @@ bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { int32_t sz = taosArrayGetSize(builderNode->trans); - ASSERT(sz < 256); + // ASSERT(sz < 256); if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { return true; } else if (sz != 1 || builderNode->isFinal) { @@ -800,7 +800,7 @@ void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); if (prefixLen == FST_SLICE_LEN(s)) { - ASSERT(out == 0); + // ASSERT(out == 0); return; } @@ -844,7 +844,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { addr = fstBuilderCompile(b, bn); fstBuilderNodeDestroy(bn); - ASSERT(addr != NONE_ADDRESS); + // ASSERT(addr != NONE_ADDRESS); } fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 69fee8d29d..5c6a39d13c 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -104,7 +104,10 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of do { char key[1024] = {0}; - ASSERT(strlen(ctx->file.buf) + 1 + 64 < sizeof(key)); + if (strlen(ctx->file.buf) + 1 + 64 >= sizeof(key)) { + return TSDB_CODE_INDEX_INVALID_FILE; + } + idxGenLRUKey(key, ctx->file.buf, blkId); LRUHandle* h = taosLRUCacheLookup(ctx->lru, key, strlen(key)); @@ -118,8 +121,10 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of if (left < kBlockSize) { nread = TMIN(left, len); int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset); - ASSERTS(bytes == nread, "index read incomplete data"); - if (bytes != nread) break; + if (bytes != nread) { + total = TSDB_CODE_INDEX_INVALID_FILE; + break; + } total += bytes; return total; @@ -129,7 +134,6 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); - ASSERTS(blk->nread <= kBlockSize, "index read incomplete data"); if (blk->nread < kBlockSize && blk->nread < len) { taosMemoryFree(blk); break; @@ -288,7 +292,6 @@ int32_t idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) { // update checksum IFileCtx* ctx = write->wrt; int nWrite = ctx->write(ctx, buf, len); - ASSERTS(nWrite == len, "index write incomplete data"); if (nWrite != len) { code = TAOS_SYSTEM_ERROR(errno); return code; diff --git a/source/libs/index/src/indexFstNode.c b/source/libs/index/src/indexFstNode.c index b934cb0536..041444f1c9 100644 --- a/source/libs/index/src/indexFstNode.c +++ b/source/libs/index/src/indexFstNode.c @@ -97,9 +97,8 @@ void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { // bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile *wrt, CompiledAddr lastAddr, CompiledAddr // startAddr) { - // size_t sz = taosArrayGetSize(b->trans); -// assert(sz < 256); +// ASSERT(sz < 256); // if (FST_BUILDER_NODE_IS_FINAL(b) // && FST_BUILDER_NODE_TRANS_ISEMPTY(b) // && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { diff --git a/source/libs/index/src/indexFstRegister.c b/source/libs/index/src/indexFstRegister.c index e27ab939b3..a8c4365a44 100644 --- a/source/libs/index/src/indexFstRegister.c +++ b/source/libs/index/src/indexFstRegister.c @@ -57,8 +57,9 @@ static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { if (start >= sz && end >= sz) { return; } - ASSERTS(start >= end, "index-fst start lower than end"); - if (start < end) return; + if (start < end) { + return; + } int32_t s = (int32_t)start; int32_t e = (int32_t)end; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 99c7b6bc76..5b0734d3d0 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -898,17 +898,20 @@ static int tfileWriteFooter(TFileWriter* write) { char buf[sizeof(FILE_MAGIC_NUMBER) + 1] = {0}; void* pBuf = (void*)buf; (void)taosEncodeFixedU64((void**)(void*)&pBuf, FILE_MAGIC_NUMBER); - int nwrite = write->ctx->write(write->ctx, buf, (int32_t)strlen(buf)); + int nwrite = write->ctx->write(write->ctx, (uint8_t*)buf, (int32_t)strlen(buf)); indexInfo("tfile write footer size: %d", write->ctx->size(write->ctx)); - ASSERTS(nwrite == sizeof(FILE_MAGIC_NUMBER), "index write incomplete data"); - return nwrite; + if (nwrite != sizeof(FILE_MAGIC_NUMBER)) { + return TAOS_SYSTEM_ERROR(errno); + } else { + return nwrite; + } } static int tfileReaderLoadHeader(TFileReader* reader) { // TODO simple tfile header later char buf[TFILE_HEADER_SIZE] = {0}; - int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); + int64_t nread = reader->ctx->readFrom(reader->ctx, (uint8_t*)buf, sizeof(buf), 0); if (nread == -1) { indexError("actual Read: %d, to read: %d, code:0x%x, filename: %s", (int)(nread), (int)sizeof(buf), errno, @@ -932,14 +935,13 @@ static int tfileReaderLoadFst(TFileReader* reader) { } int64_t ts = taosGetTimestampUs(); - int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset); + int32_t nread = ctx->readFrom(ctx, (uint8_t*)buf, fstSize, reader->header.fstOffset); int64_t cost = taosGetTimestampUs() - ts; indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %d, time cost: %" PRId64 "us", nread, reader->header.fstOffset, fstSize, ctx->file.buf, size, cost); // we assuse fst size less than FST_MAX_SIZE - ASSERTS(nread > 0 && nread <= fstSize, "index read incomplete fst"); if (nread <= 0 || nread > fstSize) { - return -1; + return TSDB_CODE_INDEX_INVALID_FILE; } FstSlice st = fstSliceCreate((uint8_t*)buf, nread); @@ -954,8 +956,10 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* IFileCtx* ctx = reader->ctx; // add block cache char block[4096] = {0}; - int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset); - ASSERT(nread >= sizeof(uint32_t)); + int32_t nread = ctx->readFrom(ctx, (uint8_t*)block, sizeof(block), offset); + if (nread < sizeof(uint32_t)) { + return TSDB_CODE_INDEX_INVALID_FILE; + } char* p = block; int32_t nid = *(int32_t*)p; @@ -972,7 +976,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* memset(block, 0, sizeof(block)); offset += sizeof(block); - nread = ctx->readFrom(ctx, block, sizeof(block), offset); + nread = ctx->readFrom(ctx, (uint8_t*)block, sizeof(block), offset); memcpy(buf + left, block, sizeof(uint64_t) - left); (void)taosArrayPush(result, (uint64_t*)buf); @@ -991,13 +995,14 @@ static int tfileReaderVerify(TFileReader* reader) { int size = ctx->size(ctx); if (size < sizeof(tMagicNumber) || size <= sizeof(reader->header)) { - return -1; - } else if (ctx->readFrom(ctx, buf, sizeof(tMagicNumber), size - sizeof(tMagicNumber)) != sizeof(tMagicNumber)) { - return -1; + return TSDB_CODE_INDEX_INVALID_FILE; + } else if (ctx->readFrom(ctx, (uint8_t*)buf, sizeof(tMagicNumber), size - sizeof(tMagicNumber)) != + sizeof(tMagicNumber)) { + return TSDB_CODE_INDEX_INVALID_FILE; } (void)taosDecodeFixedU64(buf, &tMagicNumber); - return tMagicNumber == FILE_MAGIC_NUMBER ? 0 : -1; + return tMagicNumber == FILE_MAGIC_NUMBER ? 0 : TSDB_CODE_INDEX_INVALID_FILE; } void tfileReaderRef(TFileReader* rd) { diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 83e55791d2..29fb18ef07 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -420,7 +420,10 @@ int32_t remoteChkpGetDelFile(char* path, SArray* toDel) { } void cleanDir(const char* pPath, const char* id) { - ASSERT(pPath != NULL); + if (pPath == NULL) { + stError("%s try to clean dir, but path is NULL", id); + return; + } if (taosIsDir(pPath)) { taosRemoveDir(pPath); @@ -2603,7 +2606,7 @@ void taskDbDestroy(void* pDb, bool flush) { stDebug("succ to destroy stream backend:%p", wrapper); int8_t nCf = tListLen(ginitDict); - if (flush && wrapper->removeAllFiles == 0) { + if (flush && wrapper->removeAllFiles == 0) { if (wrapper->db && wrapper->pCf) { rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); rocksdb_flushoptions_set_wait(flushOpt, 1); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 862a74c72b..51780c484f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2193,7 +2193,6 @@ bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) { for (int i = 0; ahandle == 0 && i < transQueueSize(&conn->cliMsgs); i++) { SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, i); if (cliMsg->type == Release) { - ASSERTS(pMsg == NULL, "trans-cli recv invaid release-req"); tDebug("%s conn %p receive release request, refId:%" PRId64 ", ignore msg", CONN_GET_INST_LABEL(conn), conn, conn->refId); cliDestroyConn(conn, true); @@ -3015,7 +3014,6 @@ _exception: } int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { if (transpointId == NULL) { - ASSERT(0); return TSDB_CODE_INVALID_PARA; } int32_t code = 0; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5d82e157b3..f5820b80aa 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -183,7 +183,6 @@ int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { } } } else { - ASSERTS(0, "invalid read from sock buf"); return TSDB_CODE_INVALID_MSG; } return 0; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index c1b934c812..c3883830ad 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -468,8 +468,8 @@ static bool uvHandleReq(SSvrConn* pConn) { tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn, pConn->refId); - ASSERTS(transMsg.info.handle != NULL, "trans-svr failed to alloc handle to msg"); if (transMsg.info.handle == NULL) { + tError("%s handle %p conn:%p handle failed to init" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn); return false; } @@ -960,15 +960,19 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { return; } // free memory allocated by - ASSERTS(nread == strlen(notify), "trans-svr mem corrupted"); - ASSERTS(buf->base[0] == notify[0], "trans-svr mem corrupted"); + if (nread != strlen(notify) || strncmp(buf->base, notify, strlen(notify)) != 0) { + tError("failed to read pip "); + taosMemoryFree(buf->base); + uv_close((uv_handle_t*)q, NULL); + } + taosMemoryFree(buf->base); SWorkThrd* pThrd = q->data; uv_pipe_t* pipe = (uv_pipe_t*)q; if (!uv_pipe_pending_count(pipe)) { - tError("No pending count"); + tError("no pending count, unexpected error"); uv_close((uv_handle_t*)q, NULL); return; } @@ -1344,25 +1348,32 @@ static void uvPipeListenCb(uv_stream_t* handle, int status) { uv_pipe_t* pipe = &(srv->pipe[srv->numOfWorkerReady][0]); int ret = uv_pipe_init(srv->loop, pipe, 1); - ASSERTS(ret == 0, "trans-svr failed to init pipe"); - if (ret != 0) return; + if (ret != 0) { + tError("trans-svr failed to init pipe, errmsg: %s", uv_err_name(ret)); + } ret = uv_accept((uv_stream_t*)&srv->pipeListen, (uv_stream_t*)pipe); - ASSERTS(ret == 0, "trans-svr failed to accept pipe msg"); - if (ret != 0) return; + if (ret != 0) { + tError("trans-svr failed to accept pipe, errmsg: %s", uv_err_name(ret)); + return; + } ret = uv_is_readable((uv_stream_t*)pipe); - ASSERTS(ret == 1, "trans-svr pipe status corrupted"); - if (ret != 1) return; - + if (ret != 1) { + tError("trans-svr failed to check pipe, pip not readable"); + return; + } ret = uv_is_writable((uv_stream_t*)pipe); - ASSERTS(ret == 1, "trans-svr pipe status corrupted"); - if (ret != 1) return; + if (ret != 1) { + tError("trans-svr failed to check pipe, pip not writable"); + return; + } ret = uv_is_closing((uv_handle_t*)pipe); - ASSERTS(ret == 0, "trans-svr pipe status corrupted"); - if (ret != 0) return; - + if (ret != 0) { + tError("trans-svr failed to check pipe, pip is closing"); + return; + } srv->numOfWorkerReady++; } From b7d08e49bb532bb9d14b751723613920af7d74a5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 20 Aug 2024 19:35:35 +0800 Subject: [PATCH 06/59] remove ASSERT --- source/dnode/vnode/src/vnd/vnodeQuery.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 904b29bf43..7ed7a78e44 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -37,7 +37,10 @@ int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol int8_t tblType = reader->me.type; if (useCompress(tblType)) { SColCmprWrapper *p = &(reader->me.colCmpr); - ASSERT(numOfCol == p->nCols); + if (numOfCol != p->nCols) { + return TSDB_CODE_INVALID_MSG; + } + for (int i = 0; i < p->nCols; i++) { SColCmpr *pCmpr = &p->pColCmpr[i]; pExt[i].colId = pCmpr->id; @@ -541,7 +544,7 @@ int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) { } int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1); if (NULL == pCur) { qError("vnode get all table list failed"); @@ -571,7 +574,7 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo } int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SVnode *pVnodeObj = pVnode; SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1); if (NULL == pCur) { @@ -585,7 +588,7 @@ int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { break; } - if (NULL == taosArrayPush(list, &id)) { + if (NULL == taosArrayPush(list, &id)) { qError("taosArrayPush failed"); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -598,7 +601,7 @@ _exit: } int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); if (!pCur) { return TSDB_CODE_OUT_OF_MEMORY; @@ -610,7 +613,7 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { break; } - if (NULL == taosArrayPush(list, &id)) { + if (NULL == taosArrayPush(list, &id)) { qError("taosArrayPush failed"); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -624,7 +627,7 @@ _exit: int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1), void *arg) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); if (!pCur) { return terrno; @@ -640,7 +643,7 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo continue; } - if (NULL == taosArrayPush(list, &id)) { + if (NULL == taosArrayPush(list, &id)) { qError("taosArrayPush failed"); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; From b7dddef9c0bc133c6d61fcda5cd3c68c4b0d714c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 21 Aug 2024 09:15:21 +0800 Subject: [PATCH 07/59] remove assert --- source/dnode/vnode/src/meta/metaTable.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cbd4f298d2..0334990365 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -148,11 +148,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const if (pTagVal->nData > 0) { char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); if (val == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _except); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); } int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); if (len < 0) { - TAOS_CHECK_GOTO(len, NULL, _except); + TAOS_CHECK_GOTO(len, NULL, _exception); } memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; @@ -175,15 +175,16 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const (void)indexMultiTermAdd(terms, term); } else { code = terrno; + goto _exception; } } - (void)indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + code = indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); #endif - return 0; -_except: + return code; +_exception: indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); return code; @@ -227,11 +228,11 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche if (pTagVal->nData > 0) { char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); if (val == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _except); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); } int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); if (len < 0) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _except); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); } memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; @@ -253,14 +254,15 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche (void)indexMultiTermAdd(terms, term); } else { code = terrno; + goto _exception; } } - (void)indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + code = indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); #endif - return 0; -_except: + return code; +_exception: indexMultiTermDestroy(terms); taosArrayDestroy(pTagVals); return code; From 9ae951621bd1151b09efc42a777fb53ada68721d Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 22 Aug 2024 19:09:25 +0800 Subject: [PATCH 08/59] fix error not reported in sort operator --- source/common/src/tdatablock.c | 3 +++ source/libs/executor/src/sortoperator.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 8f97aaf154..8687c98310 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -973,6 +973,9 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { int32_t numOfRows = *(int32_t*)buf; + if (numOfRows == 0) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } int32_t code = blockDataEnsureCapacity(pBlock, numOfRows); if (code) { return code; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 36f9ac0954..af8ec299b1 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -295,6 +295,9 @@ static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, break; } } + if (TSDB_CODE_SUCCESS != code) { + return code; + } if (p->info.rows > 0) { code = blockDataEnsureCapacity(pDataBlock, capacity); From e6a35dd34dec467c466efff8da8f6f14d82cb982 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 20 Aug 2024 14:33:38 +0800 Subject: [PATCH 09/59] remove asserts of nodes/planner/parser/tsdbread --- include/util/tlosertree.h | 4 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 74 ++++++++++++----- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 15 +++- source/libs/executor/src/operator.c | 7 +- source/libs/executor/src/projectoperator.c | 28 +++++-- source/libs/executor/src/scanoperator.c | 4 +- source/libs/executor/src/tlinearhash.c | 38 +++++++-- source/libs/executor/src/tsort.c | 94 +++++++++++++++------- source/libs/parser/src/parTranslater.c | 10 ++- source/libs/parser/src/parUtil.c | 6 +- source/libs/planner/src/planOptimizer.c | 11 +-- source/util/src/tlosertree.c | 48 ++++++++--- source/util/src/tpagedbuf.c | 1 - 13 files changed, 245 insertions(+), 95 deletions(-) diff --git a/include/util/tlosertree.h b/include/util/tlosertree.h index b3aa37a537..c82eff4bd7 100644 --- a/include/util/tlosertree.h +++ b/include/util/tlosertree.h @@ -45,9 +45,9 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void tMergeTreeDestroy(SMultiwayMergeTreeInfo **pTree); -void tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx); +int32_t tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx); -void tMergeTreeRebuild(SMultiwayMergeTreeInfo *pTree); +int32_t tMergeTreeRebuild(SMultiwayMergeTreeInfo *pTree); void tMergeTreePrint(const SMultiwayMergeTreeInfo *pTree); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 9be2c3b3f6..ffd2427577 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -137,7 +137,6 @@ static int32_t tGetPrimaryKeyIndex(uint8_t* p, SPrimaryKeyIndex* index) { static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { SPrimaryKeyIndex indices[TD_MAX_PK_COLS]; - ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS); uint8_t* data = pRow->data; for (int32_t i = 0; i < pRow->numOfPKs; i++) { @@ -673,7 +672,10 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead break; } - ASSERT(pBrinBlk->minTbid.suid <= pReader->info.suid && pBrinBlk->maxTbid.suid >= pReader->info.suid); + if (!(pBrinBlk->minTbid.suid <= pReader->info.suid && pBrinBlk->maxTbid.suid >= pReader->info.suid)) { + tsdbError("tsdb failed at: %s %d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[0]) { i += 1; continue; @@ -753,7 +755,10 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S continue; } - ASSERT(pRecord->suid == pReader->info.suid && uid == pRecord->uid); + if (!(pRecord->suid == pReader->info.suid && uid == pRecord->uid)) { + tsdbError("tsdb failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } STableBlockScanInfo* pScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); @@ -924,7 +929,7 @@ static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInf size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { - ASSERT(pBlockIter->numOfBlocks == num); + tsdbError("tsdb read failed at: %s:%d", __func__, __LINE__); return TSDB_CODE_FAILED; } @@ -932,13 +937,15 @@ static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInf return (*pInfo) != NULL? TSDB_CODE_SUCCESS:TSDB_CODE_FAILED; } -static int doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) { +static int32_t doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) { // start end position int s, e; s = pos; // check - ASSERT(pos >= 0 && pos < num && num > 0); + if (!(pos >= 0 && pos < num && num > 0)) { + return -1; + } if (order == TSDB_ORDER_ASC) { // find the first position which is smaller than the key e = num - 1; @@ -1242,7 +1249,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro } pDumpInfo->rowIndex = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, pDumpInfo->rowIndex, (!asc)); - ASSERT(pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer); + if (!(pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer)) { + tsdbError("tsdb failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_INVALID_PARA; + } // find the appropriate start position that satisfies the version requirement. if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) || @@ -1382,7 +1392,12 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro } static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) { - ASSERT(pReader->info.pSchema == NULL); + if (pReader->info.pSchema != NULL) { + terrno = TSDB_CODE_INVALID_PARA; + tsdbError("tsdb invalid input param at: %s:%d", __func__, __LINE__); + return NULL; + } + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema); if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) { terrno = code; @@ -1663,7 +1678,6 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, order); // todo handle the primary key overlap case - ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts; pInfo->overlapWithSttBlock = !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt); @@ -1921,7 +1935,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized if (pMerger->pArray == NULL) { - ASSERT(pReader->info.pSchema == NULL); + if (pReader->info.pSchema != NULL) { + tsdbError("tsdb failed at %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { return terrno; @@ -2014,7 +2031,10 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized if (pMerger->pArray == NULL) { - ASSERT(pReader->info.pSchema == NULL); + if (pReader->info.pSchema) { + tsdbError("tsdb failed at %s %d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { return terrno; @@ -2147,7 +2167,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized if (pMerger->pArray == NULL) { - ASSERT(pReader->info.pSchema == NULL); + if (pReader->info.pSchema != NULL) { + tsdbError("tsdb read failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { return terrno; @@ -2548,7 +2571,10 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized if (pMerger->pArray == NULL) { - ASSERT(pReader->info.pSchema == NULL); + if (pReader->info.pSchema != NULL) { + tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { return terrno; @@ -3267,8 +3293,6 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { // current active data block not overlap with the stt-files/stt-blocks static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) { - ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); - if ((!hasDataInSttBlock(pScanInfo)) || (pScanInfo->cleanSttBlocks == true)) { return true; } else { @@ -3336,7 +3360,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // data block, so the overlap check is invalid actually. buildCleanBlockFromDataFiles(pReader, pScanInfo, pBlockInfo, pBlockIter->index); } else { // clean stt block - ASSERT(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL); + if (!(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL)) { + tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } code = buildCleanBlockFromSttFiles(pReader, pScanInfo); return code; } @@ -3357,7 +3384,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // no data in stt block, no need to proceed. while (hasDataInSttBlock(pScanInfo)) { - ASSERT(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); + if (pScanInfo->sttKeyInfo.status != STT_FILE_HAS_DATA) { + tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader); if (code != TSDB_CODE_SUCCESS) { @@ -3580,7 +3610,11 @@ static ERetrieveType doReadDataFromSttFiles(STsdbReader* pReader) { } // all data blocks are checked in this stt file, now let's try the next file set - ASSERT(pReader->status.pTableIter == NULL); + if (pReader->status.pTableIter != NULL) { + terrno = TSDB_CODE_FAILED; + tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); + return TSDB_READ_RETURN; + } code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked @@ -3794,7 +3828,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t return false; } - ASSERT(key >= last->ts); + // ASSERT(key >= last->ts); if (key > last->ts) { return false; } else if (key == last->ts) { @@ -3857,7 +3891,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t } else if (key == pFirst->ts) { return pFirst->version >= ver; } else { - ASSERT(0); + // ASSERT(0); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 57405c0e48..fe258093b4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -741,7 +741,10 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 } numOfTotal += 1; - tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + code = tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } for (int32_t i = 0; i < numOfTables; ++i) { @@ -843,7 +846,10 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ continue; } - ASSERT(record.suid == pReader->info.suid && uid == record.uid); + if (!(record.suid == pReader->info.suid && uid == record.uid)) { + tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } if (record.version <= pReader->info.verRange.maxVer) { SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; @@ -877,7 +883,10 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs break; } - ASSERT(pTombBlk->minTbid.suid <= pReader->info.suid && pTombBlk->maxTbid.suid >= pReader->info.suid); + if (!(pTombBlk->minTbid.suid <= pReader->info.suid && pTombBlk->maxTbid.suid >= pReader->info.suid)) { + tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) { i += 1; continue; diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index af52c31364..0ff6870405 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -240,7 +240,12 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan *order = info.order; *scanFlag = info.scanFlag; - ASSERT(*order == TSDB_ORDER_ASC || *order == TSDB_ORDER_DESC); + if (p.code == TSDB_CODE_SUCCESS) { + if (!(*order == TSDB_ORDER_ASC || *order == TSDB_ORDER_DESC)) { + qError("operator failed at: %s:%d", __func__, __LINE__); + p.code = TSDB_CODE_INVALID_PARA; + } + } return p.code; } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index dd0acc42ed..1a43fe47a3 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -218,7 +218,10 @@ static int32_t discardGroupDataBlock(SSDataBlock* pBlock, SLimitInfo* pLimitInfo static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, SOperatorInfo* pOperator) { // remainGroupOffset == 0 // here check for a new group data, we need to handle the data of the previous group. - ASSERT(pLimitInfo->remainGroupOffset == 0 || pLimitInfo->remainGroupOffset == -1); + if (!(pLimitInfo->remainGroupOffset == 0 || pLimitInfo->remainGroupOffset == -1)) { + qError("project failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_INVALID_PARA; + } bool newGroup = false; if (0 == pBlock->info.id.groupId) { @@ -818,7 +821,10 @@ int32_t doGenerateSourceData(SOperatorInfo* pOperator) { } int32_t startOffset = pRes->info.rows; - ASSERT(pRes->info.capacity > 0); + if (pRes->info.capacity <= 0) { + qError("project failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } code = colDataAssign(pResColData, &idata, dest.numOfRows, &pRes->info); if (code) { return code; @@ -875,7 +881,11 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc for (int32_t k = 0; k < numOfOutput; ++k) { int32_t outputSlotId = pExpr[k].base.resSchema.slotId; - ASSERT(pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE); + if (pExpr[k].pExpr->nodeType != QUERY_NODE_VALUE) { + qError("project failed at: %s:%d", __func__, __LINE__); + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _exit); + } SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); if (pColInfoData == NULL) { code = terrno; @@ -1019,7 +1029,11 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - ASSERT(pResult->info.capacity > 0); + if (pResult->info.capacity <= 0) { + qError("project failed at: %s:%d", __func__, __LINE__); + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _exit); + } int32_t ret = colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows); if (ret < 0) { @@ -1146,7 +1160,11 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - ASSERT(pResult->info.capacity > 0); + if (pResult->info.capacity <= 0) { + qError("project failed at: %s:%d", __func__, __LINE__); + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _exit); + } int32_t ret = colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows); if (ret < 0) { code = ret; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b7b5e4fff9..9d2dd68f7d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -5118,9 +5118,7 @@ static int32_t adjustSubTableForNextRow(SOperatorInfo* pOperatorInfo, STmsSubTab } } - tMergeTreeAdjust(pSubTblsInfo->pTree, tMergeTreeGetAdjustIndex(pSubTblsInfo->pTree)); - - return TSDB_CODE_SUCCESS; + return tMergeTreeAdjust(pSubTblsInfo->pTree, tMergeTreeGetAdjustIndex(pSubTblsInfo->pTree)); } static int32_t appendChosenRowToDataBlock(STmsSubTablesMergeInfo* pSubTblsInfo, SSDataBlock* pBlock) { diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 0c70428e78..9487e9d163 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -14,6 +14,7 @@ */ #include "tlinearhash.h" +#include "query.h" #include "taoserror.h" #include "tdef.h" #include "tpagedbuf.h" @@ -59,7 +60,11 @@ static int32_t doGetBucketIdFromHashVal(int32_t hashv, int32_t bits) { return ha static int32_t doGetAlternativeBucketId(int32_t bucketId, int32_t bits, int32_t numOfBuckets) { int32_t v = bucketId - (1ul << (bits - 1)); - ASSERT(v < numOfBuckets); + if (v >= numOfBuckets) { + qError("tlinearhash failed at: %s:%d", __func__, __LINE__); + terrno = TSDB_CODE_FAILED; + return -1; + } return v; } @@ -85,11 +90,15 @@ static int32_t doAddToBucket(SLHashObj* pHashObj, SLHashBucket* pBucket, int32_t int32_t pageId = *(int32_t*)taosArrayGetLast(pBucket->pPageIdList); SFilePage* pPage = getBufPage(pHashObj->pBuf, pageId); - ASSERT(pPage != NULL); + if (pPage == NULL) { + return TSDB_CODE_INVALID_PARA; + } // put to current buf page size_t nodeSize = sizeof(SLHashNode) + keyLen + size; - ASSERT(nodeSize + sizeof(SFilePage) <= getBufPageSize(pHashObj->pBuf)); + if (nodeSize + sizeof(SFilePage) > getBufPageSize(pHashObj->pBuf)) { + return TSDB_CODE_INVALID_PARA; + } if (pPage->num + nodeSize > getBufPageSize(pHashObj->pBuf)) { releaseBufPage(pHashObj->pBuf, pPage); @@ -174,7 +183,7 @@ static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) { setBufPageDirty(pFirst, true); setBufPageDirty(pLast, true); - ASSERT(pLast->num >= nodeSize + sizeof(SFilePage)); + // ASSERT(pLast->num >= nodeSize + sizeof(SFilePage)); pFirst->num += nodeSize; pLast->num -= nodeSize; @@ -320,6 +329,9 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data // printf("bucketId: 0x%x not exists, put it into 0x%x instead\n", v, newBucketId); v = newBucketId; } + if (v < 0) { + return terrno; + } SLHashBucket* pBucket = pHashObj->pBucket[v]; code = doAddToBucket(pHashObj, pBucket, v, key, keyLen, data, size); @@ -343,7 +355,10 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data int32_t numOfBits = ceil(log(pHashObj->numOfBuckets) / log(2)); if (numOfBits > pHashObj->bits) { // printf("extend the bits from %d to %d, new bucket:%d\n", pHashObj->bits, numOfBits, newBucketId); - ASSERT(numOfBits == pHashObj->bits + 1); + if (numOfBits != pHashObj->bits + 1) { + qError("linear hash faield at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } pHashObj->bits = numOfBits; } @@ -360,14 +375,20 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data char* pStart = p->data; while (pStart - ((char*)p) < p->num) { SLHashNode* pNode = (SLHashNode*)pStart; - ASSERT(pNode->keyLen > 0); + if (pNode->keyLen <= 0) { + qError("linear hash faield at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } char* k = GET_LHASH_NODE_KEY(pNode); int32_t hashv = pHashObj->hashFn(k, pNode->keyLen); int32_t v1 = doGetBucketIdFromHashVal(hashv, pHashObj->bits); if (v1 != splitBucketId) { // place it into the new bucket - ASSERT(v1 == newBucketId); + if (v1 != newBucketId) { + qError("linear hash failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } // printf("move key:%d to 0x%x bucket, remain items:%d\n", *(int32_t*)k, v1, pBucket->size - 1); SLHashBucket* pNewBucket = pHashObj->pBucket[newBucketId]; code = doAddToBucket(pHashObj, pNewBucket, newBucketId, (void*)GET_LHASH_NODE_KEY(pNode), pNode->keyLen, @@ -395,6 +416,9 @@ char* tHashGet(SLHashObj* pHashObj, const void* key, size_t keyLen) { if (bucketId >= pHashObj->numOfBuckets) { bucketId = doGetAlternativeBucketId(bucketId, pHashObj->bits, pHashObj->numOfBuckets); } + if (bucketId < 0) { + return NULL; + } SLHashBucket* pBucket = pHashObj->pBucket[bucketId]; int32_t num = taosArrayGetSize(pBucket->pPageIdList); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 37cf131a95..74fd48d85a 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -468,7 +468,11 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock))) / rowSize; - ASSERT(numOfRows > 0); + if (numOfRows <= 0) { + qError("sort failed at: %s:%d", __func__, __LINE__); + taosArrayDestroy(pPageIdList); + return TSDB_CODE_FAILED; + } return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); } @@ -528,7 +532,10 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { } int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); - ASSERT(size <= getBufPageSize(pHandle->pBuf)); + if (size > getBufPageSize(pHandle->pBuf)) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } code = blockDataToBuf(pPage, p); if (code) { @@ -733,7 +740,10 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT tMergeTreePrint(pTree); #endif - tMergeTreeAdjust(pTree, leafNodeIndex); + int32_t code = tMergeTreeAdjust(pTree, leafNodeIndex); + if (TSDB_CODE_SUCCESS != code) { + return code; + } #ifdef _DEBUG_VIEW printf("\nafter adjust:\t"); @@ -1040,7 +1050,10 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); - ASSERT(size <= getBufPageSize(pHandle->pBuf)); + if (size > getBufPageSize(pHandle->pBuf)) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } code= blockDataToBuf(pPage, pDataBlock); if (code) { @@ -1310,7 +1323,10 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i } pRegion->bufLen = readBytes; } - ASSERT(pRegion->bufRegOffset <= tupleOffset); + if (pRegion->bufRegOffset > tupleOffset) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } if (pRegion->bufRegOffset + pRegion->bufLen >= tupleOffset + rowLen) { *pFreeRow = false; *ppRow = pRegion->buf + tupleOffset - pRegion->bufRegOffset; @@ -1487,7 +1503,10 @@ static int32_t tsortCloseRegion(SSortHandle* pHandle) { static int32_t tsortFinalizeRegions(SSortHandle* pHandle) { SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; size_t numRegions = taosArrayGetSize(pMemFile->aFileRegions); - ASSERT(numRegions == (pMemFile->currRegionId + 1)); + if (numRegions != (pMemFile->currRegionId + 1)) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } if (numRegions == 0) { return TSDB_CODE_SUCCESS; } @@ -1812,7 +1831,10 @@ static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, } int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); - ASSERT(size <= getBufPageSize(pHandle->pBuf)); + if (size > getBufPageSize(pHandle->pBuf)) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } int32_t code = blockDataToBuf(pPage, blk); @@ -1863,12 +1885,18 @@ static int32_t getPageBufIncForRowIdSort(SSDataBlock* pDstBlock, int32_t srcRowI int32_t numOfCols = blockDataGetNumOfCols(pDstBlock); if (pPkCol == NULL) { // no var column - ASSERT((numOfCols == 4) && (!pDstBlock->info.hasVarCol)); + if (!((numOfCols == 4) && (!pDstBlock->info.hasVarCol))) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } size += numOfCols * ((dstRowIndex & 0x7) == 0 ? 1: 0); size += blockDataGetRowSize(pDstBlock); } else { - ASSERT(numOfCols == 5); + if (numOfCols != 5) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } size += (numOfCols - 1) * (((dstRowIndex & 0x7) == 0)? 1:0); for(int32_t i = 0; i < numOfCols - 1; ++i) { @@ -2018,7 +2046,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); if (aPgId == NULL) { - return terrno; + goto _error; } int32_t nRows = 0; @@ -2039,10 +2067,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pTree); - taosArrayDestroy(aPgId); - cleanupMergeSup(&sup); - return code; + goto _error; } nMergedRows += pHandle->pDataBlock->info.rows; @@ -2064,7 +2089,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* code = blockDataEnsureCapacity(pHandle->pDataBlock, pHandle->pDataBlock->info.rows + 1); if (code) { - return code; + goto _error; } if (pHandle->bSortByRowId) { @@ -2074,11 +2099,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* } if (code) { - return code; + goto _error; } blkPgSz += bufInc; - ASSERT(blkPgSz == blockDataGetSize(pHandle->pDataBlock) + pageHeaderSize); + if (blkPgSz != blockDataGetSize(pHandle->pDataBlock) + pageHeaderSize) { + qError("sort failed at: %s:%d", __func__, __LINE__); + goto _error; + } ++nRows; @@ -2087,7 +2115,10 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* } else { ++sup.aRowIdx[minIdx]; } - tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + code = tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + if (TSDB_CODE_SUCCESS != code) { + goto _error; + } } if (pHandle->pDataBlock->info.rows > 0) { @@ -2096,10 +2127,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(aPgId); - taosMemoryFree(pTree); - cleanupMergeSup(&sup); - return code; + goto _error; } nMergedRows += pHandle->pDataBlock->info.rows; if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { @@ -2115,11 +2143,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* SSDataBlock* pMemSrcBlk = NULL; code = createOneDataBlock(pHandle->pDataBlock, false, &pMemSrcBlk); - if (code) { - cleanupMergeSup(&sup); - tMergeTreeDestroy(&pTree); - return code; - } + if (code) goto _error; code = doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId); @@ -2127,6 +2151,12 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* tMergeTreeDestroy(&pTree); return code; + +_error: + tMergeTreeDestroy(&pTree); + cleanupMergeSup(&sup); + if (aPgId) taosArrayDestroy(aPgId); + return code; } static int32_t getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSHashObj* mTableNumRows, SSDataBlock* pOrigBlk, @@ -2502,7 +2532,10 @@ static int32_t tsortOpenForBufMergeSort(SSortHandle* pHandle) { int32_t numOfSources = taosArrayGetSize(pHandle->pOrderedSource); if (pHandle->pBuf != NULL) { - ASSERT(numOfSources <= getNumOfInMemBufPages(pHandle->pBuf)); + if (numOfSources > getNumOfInMemBufPages(pHandle->pBuf)) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } } if (numOfSources == 0) { @@ -2596,7 +2629,10 @@ static int32_t tsortBufMergeSortNextTuple(SSortHandle* pHandle, STupleHandle** p index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); pSource = pHandle->cmpParam.pSources[index]; - ASSERT(pSource->src.pBlock != NULL); + if (pSource->src.pBlock == NULL) { + qError("sort failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_FAILED; + } pHandle->tupleHandle.rowIndex = pSource->src.rowIndex; pHandle->tupleHandle.pBlock = pSource->src.pBlock; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 05cceb656e..ae4dc280b4 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4014,7 +4014,6 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo if (TSDB_CODE_SUCCESS == code) code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetTbName, &vgInfo, &exists); if (TSDB_CODE_SUCCESS == code) { - ASSERT(exists); if (!pRealTable->tsmaTargetTbVgInfo) { pRealTable->tsmaTargetTbVgInfo = taosArrayInit(pRealTable->pTsmas->size, POINTER_BYTES); if (!pRealTable->tsmaTargetTbVgInfo) { @@ -7320,7 +7319,10 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete SValueNode* pFreq = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 0); SValueNode* pKeep = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 1); - ASSERTS(IS_DURATION_VAL(pFreq->flag) && IS_DURATION_VAL(pKeep->flag), "Retentions freq/keep should have unit"); + if (!IS_DURATION_VAL(pFreq->flag) || !IS_DURATION_VAL(pKeep->flag)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Retentions freq/keep should have unit"); + } // check unit if (IS_DURATION_VAL(pFreq->flag) && TIME_UNIT_SECOND != pFreq->unit && TIME_UNIT_MINUTE != pFreq->unit && @@ -9427,7 +9429,9 @@ static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* (void)tNameGetFullDbName(&name, pReq->dbFName); SNode* pNode = NULL; - ASSERT(LIST_LENGTH(pStmt->pCols) == 1); + if (LIST_LENGTH(pStmt->pCols) != 1) { + return TSDB_CODE_PAR_INVALID_TAGS_NUM; + } FOREACH(pNode, pStmt->pCols) { SColumnNode* p = (SColumnNode*)pNode; memcpy(pReq->colName, p->colName, sizeof(p->colName)); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 6a36ea7b85..250e9385b4 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -1281,8 +1281,10 @@ int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, ST } STableTSMAInfoRsp* pTsmaRsp = NULL; code = getMetaDataFromHash(tsmaFName, strlen(tsmaFName), pMetaCache->pTSMAs, (void**)&pTsmaRsp); - if (TSDB_CODE_SUCCESS == code && pTsmaRsp) { - ASSERT(pTsmaRsp->pTsmas->size == 1); + if (TSDB_CODE_SUCCESS == code) { + if (!pTsmaRsp || pTsmaRsp->pTsmas->size != 1) { + return TSDB_CODE_PAR_INTERNAL_ERROR; + } *pTsma = taosArrayGetP(pTsmaRsp->pTsmas, 0); } else if (code == TSDB_CODE_PAR_INTERNAL_ERROR){ code = TSDB_CODE_MND_SMA_NOT_EXIST; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c7ce9aed76..3634e05247 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -6641,7 +6641,6 @@ static int32_t fillTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pScan) { pTsmaOptCtx->pAggFuncs = pWindow->pFuncs; pTsmaOptCtx->ppParentTsmaSubplans = &pWindow->pTsmaSubplans; } else { - ASSERT(nodeType(pTsmaOptCtx->pParent) == QUERY_NODE_LOGIC_PLAN_AGG); SAggLogicNode* pAgg = (SAggLogicNode*)pTsmaOptCtx->pParent; pTsmaOptCtx->pAggFuncs = pAgg->pAggFuncs; pTsmaOptCtx->ppParentTsmaSubplans = &pAgg->pTsmaSubplans; @@ -6939,8 +6938,9 @@ static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* } int32_t tsmaOptCreateTsmaScanCols(const STSMAOptUsefulTsma* pTsma, const SNodeList* pAggFuncs, SNodeList** ppList) { - ASSERT(pTsma->pTsma); - ASSERT(pTsma->pTsmaScanCols); + if (!pTsma->pTsma || !pTsma->pTsmaScanCols) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } int32_t code; SNode* pNode; SNodeList* pScanCols = NULL; @@ -6996,8 +6996,7 @@ static int32_t tsmaOptRewriteTag(const STSMAOptCtx* pTsmaOptCtx, const STSMAOptU break; } } - ASSERT(found); - return 0; + return found ? TSDB_CODE_SUCCESS : TSDB_CODE_PLAN_INTERNAL_ERROR; } static int32_t tsmaOptRewriteTbname(const STSMAOptCtx* pTsmaOptCtx, SNode** pTbNameNode, @@ -7116,7 +7115,6 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew SColumnNode* pPkTsCol = NULL; FOREACH(pNode, pNewScan->pScanCols) { SColumnNode* pCol = (SColumnNode*)pNode; - ASSERT(pTsma->pTsmaScanCols); if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { pPkTsCol = NULL; code = nodesCloneNode((SNode*)pCol, (SNode**)&pPkTsCol); @@ -7302,7 +7300,6 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) { for (int32_t j = 0; j < pTsmaOptCtx->pScan->pTsmas->size; ++j) { if (taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, j) == pTsma->pTsma) { const STsmaTargetTbInfo* ptbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetTbInfo, j); - ASSERT(ptbInfo->uid != 0); strcpy(pTsma->targetTbName, ptbInfo->tableName); pTsma->targetTbUid = ptbInfo->uid; } diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 7973a84593..06ed9b7690 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -19,8 +19,11 @@ #include "tlog.h" // Set the initial value of the multiway merge tree. -static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { - ASSERT((pTree->totalSources & 0x01) == 0 && (pTree->numOfSources << 1 == pTree->totalSources)); +static int32_t tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { + if (!((pTree->totalSources & 0x01) == 0 && (pTree->numOfSources << 1 == pTree->totalSources))) { + uError("losertree failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_INVALID_PARA; + } for (int32_t i = 0; i < pTree->totalSources; ++i) { if (i < pTree->numOfSources) { @@ -29,6 +32,7 @@ static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { pTree->pNode[i].index = i - pTree->numOfSources; } } + return TSDB_CODE_SUCCESS; } int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, @@ -50,7 +54,11 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, pTreeInfo->comparFn = compareFn; // set initial value for loser tree - tMergeTreeInit(pTreeInfo); + int32_t code = tMergeTreeInit(pTreeInfo); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(pTreeInfo); + return code; + } #ifdef _DEBUG_VIEW printf("the initial value of loser tree:\n"); @@ -58,7 +66,11 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, #endif for (int32_t i = totalEntries - 1; i >= numOfSources; i--) { - tMergeTreeAdjust(pTreeInfo, i); + code = tMergeTreeAdjust(pTreeInfo, i); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(pTreeInfo); + return code; + } } #if defined(_DEBUG_VIEW) @@ -79,13 +91,17 @@ void tMergeTreeDestroy(SMultiwayMergeTreeInfo** pTree) { taosMemoryFreeClear(*pTree); } -void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { - ASSERT(idx <= pTree->totalSources - 1 && idx >= pTree->numOfSources && pTree->totalSources >= 2); +int32_t tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { + int32_t code = 0; + if (!(idx <= pTree->totalSources - 1 && idx >= pTree->numOfSources && pTree->totalSources >= 2)) { + uError("losertree failed at: %s:%d", __func__, __LINE__); + return TSDB_CODE_INVALID_PARA; + } if (pTree->totalSources == 2) { pTree->pNode[0].index = 0; pTree->pNode[1].index = 0; - return; + return code; } int32_t parentId = idx >> 1; @@ -95,7 +111,7 @@ void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { STreeNode* pCur = &pTree->pNode[parentId]; if (pCur->index == -1) { pTree->pNode[parentId] = kLeaf; - return; + return code; } int32_t ret = pTree->comparFn(pCur, &kLeaf, pTree->param); @@ -112,13 +128,21 @@ void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { // winner cannot be identical to the loser, which is pTreeNode[1] pTree->pNode[0] = kLeaf; } + return code; } -void tMergeTreeRebuild(SMultiwayMergeTreeInfo* pTree) { - tMergeTreeInit(pTree); - for (int32_t i = pTree->totalSources - 1; i >= pTree->numOfSources; i--) { - tMergeTreeAdjust(pTree, i); +int32_t tMergeTreeRebuild(SMultiwayMergeTreeInfo* pTree) { + int32_t code = tMergeTreeInit(pTree); + if (TSDB_CODE_SUCCESS != code) { + return code; } + for (int32_t i = pTree->totalSources - 1; i >= pTree->numOfSources; i--) { + code = tMergeTreeAdjust(pTree, i); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + return TSDB_CODE_SUCCESS; } /* diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index ef0a45233a..32b9b6e18d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -313,7 +313,6 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) { static char* evictBufPage(SDiskbasedBuf* pBuf) { SListNode* pn = getEldestUnrefedPage(pBuf); if (pn == NULL) { // no available buffer pages now, return. - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } From 043a8a8cfc2961d48ed9e4d138ff65dfb6e04fa9 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 22 Aug 2024 18:34:15 +0800 Subject: [PATCH 10/59] use internal error --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 20 ++++++++++---------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 4 ++-- source/libs/executor/src/projectoperator.c | 2 +- source/libs/executor/src/tlinearhash.c | 8 ++++---- source/libs/executor/src/tsort.c | 20 ++++++++++---------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index ffd2427577..391b7f636d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -674,7 +674,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead if (!(pBrinBlk->minTbid.suid <= pReader->info.suid && pBrinBlk->maxTbid.suid >= pReader->info.suid)) { tsdbError("tsdb failed at: %s %d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[0]) { i += 1; @@ -757,7 +757,7 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S if (!(pRecord->suid == pReader->info.suid && uid == pRecord->uid)) { tsdbError("tsdb failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } STableBlockScanInfo* pScanInfo = NULL; @@ -930,7 +930,7 @@ static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInf size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { tsdbError("tsdb read failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); @@ -1937,7 +1937,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pMerger->pArray == NULL) { if (pReader->info.pSchema != NULL) { tsdbError("tsdb failed at %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { @@ -2033,7 +2033,7 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* if (pMerger->pArray == NULL) { if (pReader->info.pSchema) { tsdbError("tsdb failed at %s %d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { @@ -2169,7 +2169,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (pMerger->pArray == NULL) { if (pReader->info.pSchema != NULL) { tsdbError("tsdb read failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { @@ -2573,7 +2573,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc if (pMerger->pArray == NULL) { if (pReader->info.pSchema != NULL) { tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); if (ps == NULL) { @@ -3362,7 +3362,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } else { // clean stt block if (!(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL)) { tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } code = buildCleanBlockFromSttFiles(pReader, pScanInfo); return code; @@ -3386,7 +3386,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { while (hasDataInSttBlock(pScanInfo)) { if (pScanInfo->sttKeyInfo.status != STT_FILE_HAS_DATA) { tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader); @@ -3611,7 +3611,7 @@ static ERetrieveType doReadDataFromSttFiles(STsdbReader* pReader) { // all data blocks are checked in this stt file, now let's try the next file set if (pReader->status.pTableIter != NULL) { - terrno = TSDB_CODE_FAILED; + terrno = TSDB_CODE_INTERNAL_ERROR; tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); return TSDB_READ_RETURN; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index fe258093b4..5d4dc94431 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -848,7 +848,7 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ if (!(record.suid == pReader->info.suid && uid == record.uid)) { tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } if (record.version <= pReader->info.verRange.maxVer) { @@ -885,7 +885,7 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs if (!(pTombBlk->minTbid.suid <= pReader->info.suid && pTombBlk->maxTbid.suid >= pReader->info.suid)) { tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) { i += 1; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 1a43fe47a3..bf523af918 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -823,7 +823,7 @@ int32_t doGenerateSourceData(SOperatorInfo* pOperator) { int32_t startOffset = pRes->info.rows; if (pRes->info.capacity <= 0) { qError("project failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } code = colDataAssign(pResColData, &idata, dest.numOfRows, &pRes->info); if (code) { diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 9487e9d163..55628c18f9 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -62,7 +62,7 @@ static int32_t doGetAlternativeBucketId(int32_t bucketId, int32_t bits, int32_t int32_t v = bucketId - (1ul << (bits - 1)); if (v >= numOfBuckets) { qError("tlinearhash failed at: %s:%d", __func__, __LINE__); - terrno = TSDB_CODE_FAILED; + terrno = TSDB_CODE_INTERNAL_ERROR; return -1; } return v; @@ -357,7 +357,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data // printf("extend the bits from %d to %d, new bucket:%d\n", pHashObj->bits, numOfBits, newBucketId); if (numOfBits != pHashObj->bits + 1) { qError("linear hash faield at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } pHashObj->bits = numOfBits; } @@ -377,7 +377,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data SLHashNode* pNode = (SLHashNode*)pStart; if (pNode->keyLen <= 0) { qError("linear hash faield at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } char* k = GET_LHASH_NODE_KEY(pNode); @@ -387,7 +387,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data if (v1 != splitBucketId) { // place it into the new bucket if (v1 != newBucketId) { qError("linear hash failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_INTERNAL_ERROR; } // printf("move key:%d to 0x%x bucket, remain items:%d\n", *(int32_t*)k, v1, pBucket->size - 1); SLHashBucket* pNewBucket = pHashObj->pBucket[newBucketId]; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 74fd48d85a..c1edf700d1 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -471,7 +471,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource if (numOfRows <= 0) { qError("sort failed at: %s:%d", __func__, __LINE__); taosArrayDestroy(pPageIdList); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); @@ -534,7 +534,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); if (size > getBufPageSize(pHandle->pBuf)) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } code = blockDataToBuf(pPage, p); @@ -1052,7 +1052,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); if (size > getBufPageSize(pHandle->pBuf)) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } code= blockDataToBuf(pPage, pDataBlock); @@ -1325,7 +1325,7 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i } if (pRegion->bufRegOffset > tupleOffset) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } if (pRegion->bufRegOffset + pRegion->bufLen >= tupleOffset + rowLen) { *pFreeRow = false; @@ -1505,7 +1505,7 @@ static int32_t tsortFinalizeRegions(SSortHandle* pHandle) { size_t numRegions = taosArrayGetSize(pMemFile->aFileRegions); if (numRegions != (pMemFile->currRegionId + 1)) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } if (numRegions == 0) { return TSDB_CODE_SUCCESS; @@ -1833,7 +1833,7 @@ static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); if (size > getBufPageSize(pHandle->pBuf)) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } int32_t code = blockDataToBuf(pPage, blk); @@ -1887,7 +1887,7 @@ static int32_t getPageBufIncForRowIdSort(SSDataBlock* pDstBlock, int32_t srcRowI if (pPkCol == NULL) { // no var column if (!((numOfCols == 4) && (!pDstBlock->info.hasVarCol))) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } size += numOfCols * ((dstRowIndex & 0x7) == 0 ? 1: 0); @@ -1895,7 +1895,7 @@ static int32_t getPageBufIncForRowIdSort(SSDataBlock* pDstBlock, int32_t srcRowI } else { if (numOfCols != 5) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } size += (numOfCols - 1) * (((dstRowIndex & 0x7) == 0)? 1:0); @@ -2534,7 +2534,7 @@ static int32_t tsortOpenForBufMergeSort(SSortHandle* pHandle) { if (pHandle->pBuf != NULL) { if (numOfSources > getNumOfInMemBufPages(pHandle->pBuf)) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } } @@ -2631,7 +2631,7 @@ static int32_t tsortBufMergeSortNextTuple(SSortHandle* pHandle, STupleHandle** p if (pSource->src.pBlock == NULL) { qError("sort failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } pHandle->tupleHandle.rowIndex = pSource->src.rowIndex; From a4827ad086be85aea1dfdc95d6fea5a2a01825f1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 21 Aug 2024 15:54:46 +0800 Subject: [PATCH 11/59] add network random error --- include/os/osFile.h | 26 ++++++++++++++++++++------ include/util/tdef.h | 9 ++------- source/libs/transport/src/thttp.c | 4 ++++ source/libs/transport/src/transCli.c | 9 ++++++++- source/libs/transport/src/transComm.c | 5 +++++ source/libs/transport/src/transSvr.c | 9 ++++++++- source/os/src/osFile.c | 4 ++-- source/os/src/osMemory.c | 4 ++-- 8 files changed, 51 insertions(+), 19 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 8bacb1bf7c..a56c54b086 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -120,15 +120,29 @@ int32_t taosSetFileHandlesLimit(); int32_t taosLinkFile(char *src, char *dst); -FILE* taosOpenCFile(const char* filename, const char* mode); -int taosSeekCFile(FILE* file, int64_t offset, int whence); -size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ); -size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream); -int taosCloseCFile(FILE *); -int taosSetAutoDelFile(char* path); +FILE *taosOpenCFile(const char *filename, const char *mode); +int taosSeekCFile(FILE *file, int64_t offset, int whence); +size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream); +size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream); +int taosCloseCFile(FILE *); +int taosSetAutoDelFile(char *path); bool lastErrorIsFileNotExist(); +#ifdef BUILD_WITH_RAND_ERR +#define STUB_RAND_NETWORK_ERR(status) \ + do { \ + if (tsEnableRandErr && (tsRandErrScope & RAND_ERR_NETWORK)) { \ + uint32_t r = taosRand() % tsRandErrDivisor; \ + if ((r + 1) <= tsRandErrChance) { \ + status = TSDB_CODE_RPC_NETWORK_UNAVAIL; \ + } \ + } \ + while (0) +#else +#define STUB_RAND_NETWORK_ERR(status) +#endif + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index 35c4adab50..f087c28684 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -294,7 +294,7 @@ typedef enum ELogicConditionType { #define TSDB_SHOW_SUBQUERY_LEN 1000 #define TSDB_LOG_VAR_LEN 32 -#define TSDB_MAX_EP_NUM 10 +#define TSDB_MAX_EP_NUM 10 #define TSDB_ARB_GROUP_MEMBER_NUM 2 #define TSDB_ARB_TOKEN_SIZE 32 @@ -568,12 +568,7 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -enum { - RAND_ERR_MEMORY = 1, - RAND_ERR_FILE = 2, - // RAND_ERR_SCOPE_XXX... = 4, - // ... -}; +enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; #define DEFAULT_HANDLE 0 #define MNODE_HANDLE 1 diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 908468f094..e517b8a0bc 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -345,6 +345,7 @@ static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested } static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { + STUB_RAND_NETWORK_ERR(nread); SHttpClient* cli = handle->data; if (nread < 0) { tError("http-report recv error:%s", uv_strerror(nread)); @@ -356,6 +357,7 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const } } static void clientSentCb(uv_write_t* req, int32_t status) { + STUB_RAND_NETWORK_ERR(status); SHttpClient* cli = req->data; if (status != 0) { tError("http-report failed to send data, reason: %s, dst:%s:%d, chanId:%" PRId64 "", uv_strerror(status), cli->addr, @@ -367,6 +369,7 @@ static void clientSentCb(uv_write_t* req, int32_t status) { } else { tTrace("http-report succ to send data, chanId:%" PRId64 "", cli->chanId); } + status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); if (status != 0) { tError("http-report failed to recv data,reason:%s, dst:%s:%d, chanId:%" PRId64 "", uv_strerror(status), cli->addr, @@ -377,6 +380,7 @@ static void clientSentCb(uv_write_t* req, int32_t status) { } } static void clientConnCb(uv_connect_t* req, int32_t status) { + STUB_RAND_NETWORK_ERR(status); SHttpClient* cli = req->data; int64_t chanId = cli->chanId; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 862a74c72b..a5cc492b2b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -923,10 +923,12 @@ static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_ } } static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { - // impl later + STUB_RAND_NETWORK_ERR(nread); + if (handle->data == NULL) { return; } + SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { @@ -1117,6 +1119,8 @@ static bool cliHandleNoResp(SCliConn* conn) { return res; } static void cliSendCb(uv_write_t* req, int status) { + STUB_RAND_NETWORK_ERR(status); + SCliConn* pConn = transReqQueueRemove(req); if (pConn == NULL) return; @@ -1434,6 +1438,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { cliSendBatch(conn); } static void cliSendBatchCb(uv_write_t* req, int status) { + STUB_RAND_NETWORK_ERR(status); SCliConn* conn = req->data; SCliThrd* thrd = conn->hostThrd; SCliBatch* p = conn->pBatch; @@ -1523,6 +1528,8 @@ void cliConnCb(uv_connect_t* req, int status) { pConn->timer = NULL; } + STUB_RAND_NETWORK_ERR(status); + if (status != 0) { cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, pConn->dstAddr); if (timeout == false) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5d82e157b3..aea8282d44 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -869,3 +869,8 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { *ppBuf = pBuf; return len; } + +// int32_t transGenRandomError(int32_t status) { +// STUB_RAND_NETWORK_ERR(status) +// return status; +// } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index c1b934c812..84938126c3 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -493,6 +493,8 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { SSvrConn* conn = cli->data; SWorkThrd* pThrd = conn->hostThrd; + STUB_RAND_NETWORK_ERR(nread); + if (true == pThrd->quit) { tInfo("work thread received quit msg, destroy conn"); destroyConn(conn, true); @@ -553,6 +555,7 @@ void uvOnTimeoutCb(uv_timer_t* handle) { } void uvOnSendCb(uv_write_t* req, int status) { + STUB_RAND_NETWORK_ERR(status); SSvrConn* conn = transReqQueueRemove(req); if (conn == NULL) return; @@ -602,6 +605,7 @@ void uvOnSendCb(uv_write_t* req, int status) { } } static void uvOnPipeWriteCb(uv_write_t* req, int status) { + STUB_RAND_NETWORK_ERR(status); if (status == 0) { tTrace("success to dispatch conn to work thread"); } else { @@ -949,6 +953,7 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { } } void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + STUB_RAND_NETWORK_ERR(nread); if (nread < 0) { if (nread != UV_EOF) { tError("read error %s", uv_err_name(nread)); @@ -1041,9 +1046,11 @@ void* transAcceptThread(void* arg) { return NULL; } void uvOnPipeConnectionCb(uv_connect_t* connect, int status) { + STUB_RAND_NETWORK_ERR(status); if (status != 0) { return; - } + }; + SWorkThrd* pThrd = container_of(connect, SWorkThrd, connect_req); (void)uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c274c7fbab..144baeb148 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -904,7 +904,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { int32_t code = _fstat64(pFile->fd, &fileStat); #else struct stat fileStat; - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); #endif if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1611,7 +1611,7 @@ int taosSeekCFile(FILE *file, int64_t offset, int whence) { #ifdef WINDOWS return _fseeki64(file, offset, whence); #else - int code = fseeko(file, offset, whence); + int code = fseeko(file, offset, whence); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 91eb7763bc..49a5c2a2a2 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -24,7 +24,7 @@ int32_t tsRandErrChance = 1; int64_t tsRandErrDivisor = 10001; -int64_t tsRandErrScope = (RAND_ERR_MEMORY | RAND_ERR_FILE); +int64_t tsRandErrScope = (RAND_ERR_MEMORY | RAND_ERR_FILE | RAND_ERR_NETWORK); threadlocal bool tsEnableRandErr = 0; #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE) @@ -385,7 +385,7 @@ char *taosStrdup(const char *ptr) { } #endif - return tstrdup(ptr); + return tstrdup(ptr); #endif } From f464aec00af90173fa720aa7154367c62109318c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 22 Aug 2024 20:01:10 +0800 Subject: [PATCH 12/59] add network error --- include/util/taoserror.h | 2 ++ source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 15 +++++++++------ source/util/src/terror.c | 1 + 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a0898fa94b..a7bcfbba64 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -468,6 +468,8 @@ int32_t taosGetErrSize(); #define TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR TAOS_DEF_ERROR_CODE(0, 0x0427) #define TSDB_CODE_DNODE_INVALID_EN_WHITELIST TAOS_DEF_ERROR_CODE(0, 0x0428) #define TSDB_CODE_DNODE_INVALID_MONITOR_PARAS TAOS_DEF_ERROR_CODE(0, 0x0429) +#define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A) + // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index fc070d0d05..e8e7a75889 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -22,7 +22,7 @@ static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) { int32_t code = 0; (void)taosThreadRwlockRdlock(&pMgmt->lock); if (pMgmt->stopped) { - code = -1; + code = TSDB_CODE_MNODE_STOPPED; } else { (void)atomic_add_fetch_32(&pMgmt->refCount, 1); } @@ -134,16 +134,19 @@ int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + int32_t code = 0; if (NULL == pMgmt->pMnode) { const STraceId *trace = &pMsg->info.traceId; - dGError("msg:%p, stop to pre-process in mnode since mnode is NULL, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - return -1; + code = TSDB_CODE_MNODE_NOT_FOUND; + dGError("msg:%p, stop to pre-process in mnode since %s, type:%s", pMsg, tstrerror(code), TMSG_INFO(pMsg->msgType)); + return code; } pMsg->info.node = pMgmt->pMnode; - if (mndPreProcessQueryMsg(pMsg) != 0) { + if ((code = mndPreProcessQueryMsg(pMsg)) != 0) { const STraceId *trace = &pMsg->info.traceId; - dGError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); - return -1; + dGError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, tstrerror(code), + TMSG_INFO(pMsg->msgType)); + return code; } return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a58baf5883..60d7e335d7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -389,6 +389,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_CHARSET, "charset not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_LOCALE, "locale not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR, "ttlChangeOnWrite not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_EN_WHITELIST, "enableWhiteList not match") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "enableWhiteList not match") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") From 3a20e9c24123ee9d1a2bc4474bedbccf4247b53f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 22 Aug 2024 20:04:18 +0800 Subject: [PATCH 13/59] add network error --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 60d7e335d7..87876a1450 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -389,7 +389,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_CHARSET, "charset not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_LOCALE, "locale not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR, "ttlChangeOnWrite not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_EN_WHITELIST, "enableWhiteList not match") -TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "enableWhiteList not match") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "mnode stopped") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") From 08a51e21dc1ceb28cfafa88ac4512b6ed27cfa8d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 22 Aug 2024 20:05:14 +0800 Subject: [PATCH 14/59] add network error --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 87876a1450..0423850320 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -389,7 +389,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_CHARSET, "charset not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_LOCALE, "locale not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR, "ttlChangeOnWrite not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_EN_WHITELIST, "enableWhiteList not match") -TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "mnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "Mnode stopped") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") From 64660c5abf57ee84add5afb213512e9e0a9a8a17 Mon Sep 17 00:00:00 2001 From: Leo Xu <381899826@qq.com> Date: Thu, 22 Aug 2024 20:29:08 +0800 Subject: [PATCH 15/59] add 2 case --- tests/system-test/2-query/concat.py | 2 ++ tests/system-test/2-query/concat_ws.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/concat.py b/tests/system-test/2-query/concat.py index 326f6940f6..feb9bcd647 100644 --- a/tests/system-test/2-query/concat.py +++ b/tests/system-test/2-query/concat.py @@ -144,6 +144,8 @@ class TDTestCase: f"{dbname}.ct2", f"{dbname}.ct4", ] + tdSql.query("select concat(null,null)") # TD-31571 + tdSql.checkRows(1) for tb in tbname: for i in range(2,8): self.__concat_check(tb,i) diff --git a/tests/system-test/2-query/concat_ws.py b/tests/system-test/2-query/concat_ws.py index 26731715c1..c148d734c5 100644 --- a/tests/system-test/2-query/concat_ws.py +++ b/tests/system-test/2-query/concat_ws.py @@ -156,7 +156,8 @@ class TDTestCase: f"{dbname}.ct2", f"{dbname}.ct4", ] - + tdSql.query("select concat_ws(null,null,null);") # TD-31572 + tdSql.checkRows(1) for tb in tbname: for errsql in self.__concat_ws_err_check(tb): tdSql.error(sql=errsql) From 6eb2ee6f1b964667ae193c4d28afdc1e0cc43372 Mon Sep 17 00:00:00 2001 From: Leo Xu <381899826@qq.com> Date: Thu, 22 Aug 2024 20:40:14 +0800 Subject: [PATCH 16/59] add check data --- tests/system-test/2-query/concat.py | 1 + tests/system-test/2-query/concat_ws.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/system-test/2-query/concat.py b/tests/system-test/2-query/concat.py index feb9bcd647..f2f231a858 100644 --- a/tests/system-test/2-query/concat.py +++ b/tests/system-test/2-query/concat.py @@ -146,6 +146,7 @@ class TDTestCase: ] tdSql.query("select concat(null,null)") # TD-31571 tdSql.checkRows(1) + tdSql.getData(0, 0) == "NULL" for tb in tbname: for i in range(2,8): self.__concat_check(tb,i) diff --git a/tests/system-test/2-query/concat_ws.py b/tests/system-test/2-query/concat_ws.py index c148d734c5..e0fc260527 100644 --- a/tests/system-test/2-query/concat_ws.py +++ b/tests/system-test/2-query/concat_ws.py @@ -158,6 +158,7 @@ class TDTestCase: ] tdSql.query("select concat_ws(null,null,null);") # TD-31572 tdSql.checkRows(1) + tdSql.getData(0, 0) == "NULL" for tb in tbname: for errsql in self.__concat_ws_err_check(tb): tdSql.error(sql=errsql) From c648f708e26acba54804ab2db09d258e1620f6d0 Mon Sep 17 00:00:00 2001 From: Leo Xu <381899826@qq.com> Date: Thu, 22 Aug 2024 22:04:34 +0800 Subject: [PATCH 17/59] add assert --- tests/system-test/2-query/concat.py | 2 +- tests/system-test/2-query/concat_ws.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/concat.py b/tests/system-test/2-query/concat.py index f2f231a858..2c3667959e 100644 --- a/tests/system-test/2-query/concat.py +++ b/tests/system-test/2-query/concat.py @@ -146,7 +146,7 @@ class TDTestCase: ] tdSql.query("select concat(null,null)") # TD-31571 tdSql.checkRows(1) - tdSql.getData(0, 0) == "NULL" + assert tdSql.getData(0, 0) == None for tb in tbname: for i in range(2,8): self.__concat_check(tb,i) diff --git a/tests/system-test/2-query/concat_ws.py b/tests/system-test/2-query/concat_ws.py index e0fc260527..29df526ff8 100644 --- a/tests/system-test/2-query/concat_ws.py +++ b/tests/system-test/2-query/concat_ws.py @@ -158,7 +158,7 @@ class TDTestCase: ] tdSql.query("select concat_ws(null,null,null);") # TD-31572 tdSql.checkRows(1) - tdSql.getData(0, 0) == "NULL" + assert tdSql.getData(0, 0) == None for tb in tbname: for errsql in self.__concat_ws_err_check(tb): tdSql.error(sql=errsql) From 4228ecf0bfc00f9f94ea51992759eef5e85a3a52 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 23 Aug 2024 01:22:52 +0000 Subject: [PATCH 18/59] fix/TD-28935-add-sdbfilelock --- source/dnode/mnode/sdb/src/sdbHash.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 03b45c25b4..c488416287 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -148,11 +148,13 @@ static int32_t sdbGetkeySize(SSdb *pSdb, ESdbType type, const void *pKey) { static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { int32_t type = pRow->type; + (void)taosThreadMutexLock(&pSdb->filelock); sdbWriteLock(pSdb, type); SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); if (pOldRow != NULL) { sdbUnLock(pSdb, type); + (void)taosThreadMutexUnlock(&pSdb->filelock); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; @@ -164,6 +166,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { sdbUnLock(pSdb, type); + (void)taosThreadMutexUnlock(&pSdb->filelock); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; @@ -180,11 +183,13 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * sdbFreeRow(pSdb, pRow, false); terrno = code; sdbUnLock(pSdb, type); + (void)taosThreadMutexUnlock(&pSdb->filelock); return terrno; } } sdbUnLock(pSdb, type); + (void)taosThreadMutexUnlock(&pSdb->filelock); if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); From 61baff01c9e830ea58c1bf9dda0ca0ee2b823fcb Mon Sep 17 00:00:00 2001 From: Leo Xu <381899826@qq.com> Date: Fri, 23 Aug 2024 09:52:07 +0800 Subject: [PATCH 19/59] change assert to checkdata --- tests/system-test/2-query/concat.py | 2 +- tests/system-test/2-query/concat_ws.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/concat.py b/tests/system-test/2-query/concat.py index 2c3667959e..b43d4fafa8 100644 --- a/tests/system-test/2-query/concat.py +++ b/tests/system-test/2-query/concat.py @@ -146,7 +146,7 @@ class TDTestCase: ] tdSql.query("select concat(null,null)") # TD-31571 tdSql.checkRows(1) - assert tdSql.getData(0, 0) == None + tdSql.checkData(0, 0, None) for tb in tbname: for i in range(2,8): self.__concat_check(tb,i) diff --git a/tests/system-test/2-query/concat_ws.py b/tests/system-test/2-query/concat_ws.py index 29df526ff8..338166c61e 100644 --- a/tests/system-test/2-query/concat_ws.py +++ b/tests/system-test/2-query/concat_ws.py @@ -158,7 +158,7 @@ class TDTestCase: ] tdSql.query("select concat_ws(null,null,null);") # TD-31572 tdSql.checkRows(1) - assert tdSql.getData(0, 0) == None + tdSql.checkData(0, 0, None) for tb in tbname: for errsql in self.__concat_ws_err_check(tb): tdSql.error(sql=errsql) From 526443a2292bbdba0a1525ede92c7086bf6a5765 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 23 Aug 2024 10:15:50 +0800 Subject: [PATCH 20/59] Merge remote-tracking branch 'origin/3.0' into fix/TD-31540 --- source/libs/transport/src/transComm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f5820b80aa..d3fd4c12dc 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -183,6 +183,7 @@ int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { } } } else { + tError("failed to reset buffer, total:%d, len:%d, reason:%s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG)); return TSDB_CODE_INVALID_MSG; } return 0; From de4f09b8fdfc6c6ec81f1f990dcad8d98e3feb73 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 23 Aug 2024 11:01:45 +0800 Subject: [PATCH 21/59] ehn(query): remove query executor assert --- source/libs/executor/inc/executil.h | 2 +- source/libs/executor/src/cachescanoperator.c | 10 +++- source/libs/executor/src/exchangeoperator.c | 4 +- source/libs/executor/src/executil.c | 33 ++++++++---- source/libs/executor/src/executor.c | 21 ++++++-- source/libs/executor/src/executorInt.c | 9 +++- source/libs/executor/src/groupoperator.c | 16 +++--- source/libs/executor/src/scanoperator.c | 55 ++++++++++++++++---- source/libs/executor/src/sysscanoperator.c | 5 +- source/libs/executor/src/tfill.c | 44 ++++++++-------- source/util/src/tbloomfilter.c | 1 + source/util/src/tscalablebf.c | 2 +- 12 files changed, 142 insertions(+), 60 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index b68ea5a781..e35b26627b 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -126,7 +126,7 @@ uint64_t tableListGetTableGroupId(const STableListInfo* pTableList, uint6 int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t gid); int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo, int32_t* num); -uint64_t tableListGetSize(const STableListInfo* pTableList); +int32_t tableListGetSize(const STableListInfo* pTableList, int32_t* pRes); uint64_t tableListGetSuid(const STableListInfo* pTableList); STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index); int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 6b5fadbaa1..a219a5b5f0 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -176,7 +176,10 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl code = extractCacheScanSlotId(pInfo->matchInfo.pList, pTaskInfo, &pInfo->pSlotIds, &pInfo->pDstSlotIds); QUERY_CHECK_CODE(code, lino, _error); - int32_t totalTables = tableListGetSize(pTableListInfo); + int32_t totalTables = 0; + code = tableListGetSize(pTableListInfo, &totalTables); + QUERY_CHECK_CODE(code, lino, _error); + int32_t capacity = 0; pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); @@ -271,7 +274,10 @@ int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { SSDataBlock* pBufRes = pInfo->pBufferedRes; uint64_t suid = tableListGetSuid(pTableList); - int32_t size = tableListGetSize(pTableList); + int32_t size = 0; + code = tableListGetSize(pTableList, &size); + QUERY_CHECK_CODE(code, lino, _end); + if (size == 0) { setOperatorCompleted(pOperator); (*ppRes) = NULL; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index d90f59047d..f354c9a1cc 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -894,12 +894,12 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa int32_t rawLen = *(int32_t*)pStart; pStart += sizeof(int32_t); - ASSERT(compLen <= rawLen && compLen != 0); + QUERY_CHECK_CONDITION((compLen <= rawLen && compLen != 0), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); pNextStart = pStart + compLen; if (pRetrieveRsp->compressed && (compLen < rawLen)) { int32_t t = tsDecompressString(pStart, compLen, 1, pDataInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0); - ASSERT(t == rawLen); + QUERY_CHECK_CONDITION((t == rawLen), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); pStart = pDataInfo->decompBuf; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 24eafe7d57..04706982d1 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -197,7 +197,6 @@ void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL pGroupResInfo->freeItem = true; pGroupResInfo->pRows = pArrayList; pGroupResInfo->index = 0; - ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } bool hasRemainResults(SGroupResInfo* pGroupResInfo) { @@ -1560,7 +1559,11 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, return code; } - ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); + if (nodeType(pNew) != QUERY_NODE_VALUE) { + nodesDestroyList(groupNew); + pAPI->metaReaderFn.clearReader(&mr); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } SValueNode* pValue = (SValueNode*)pNew; if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) { @@ -1879,7 +1882,8 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pCond->node.aliasName); pExp->pExpr->_optrRoot.pRootNode = pNode; } else { - ASSERT(0); + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); } _end: @@ -2149,7 +2153,7 @@ int32_t relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SAr } else if (p->info.colId < pmInfo->colId) { i++; } else { - ASSERT(0); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } } return code; @@ -2383,9 +2387,13 @@ void resetLimitInfoForNextGroup(SLimitInfo* pLimitInfo) { pLimitInfo->remainOffset = pLimitInfo->limit.offset; } -uint64_t tableListGetSize(const STableListInfo* pTableList) { - ASSERT(taosArrayGetSize(pTableList->pTableList) == taosHashGetSize(pTableList->map)); - return taosArrayGetSize(pTableList->pTableList); +int32_t tableListGetSize(const STableListInfo* pTableList, int32_t* pRes) { + if (taosArrayGetSize(pTableList->pTableList) != taosHashGetSize(pTableList->map)) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + (*pRes) = taosArrayGetSize(pTableList->pTableList); + return TSDB_CODE_SUCCESS; } uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->idInfo.suid; } @@ -2430,7 +2438,6 @@ uint64_t tableListGetTableGroupId(const STableListInfo* pTableList, uint64_t tab } STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, *slot); - ASSERT(pKeyInfo->uid == tableUid); return pKeyInfo->groupId; } @@ -2457,7 +2464,8 @@ int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t int32_t slot = (int32_t)taosArrayGetSize(pTableList->pTableList) - 1; code = taosHashPut(pTableList->map, &uid, sizeof(uid), &slot, sizeof(slot)); if (code != TSDB_CODE_SUCCESS) { - ASSERT(code != TSDB_CODE_DUP_KEY); // we have checked the existence of uid in hash map above + // we have checked the existence of uid in hash map above + QUERY_CHECK_CONDITION((code != TSDB_CODE_DUP_KEY), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); taosArrayPopTailBatch(pTableList->pTableList, 1); // let's pop the last element in the array list } @@ -2474,7 +2482,12 @@ _end: int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo, int32_t* size) { int32_t totalGroups = tableListGetOutputGroups(pTableList); - int32_t numOfTables = tableListGetSize(pTableList); + int32_t numOfTables = 0; + int32_t code = tableListGetSize(pTableList, &numOfTables); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + return code; + } if (ordinalGroupIndex < 0 || ordinalGroupIndex >= totalGroups) { return TSDB_CODE_INVALID_PARA; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b2cbef8919..570de21e0f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1295,7 +1295,13 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // this value may be changed if new tables are created taosRLockLatch(&pTaskInfo->lock); - int32_t numOfTables = tableListGetSize(pTableListInfo); + int32_t numOfTables = 0; + code = tableListGetSize(pTableListInfo, &numOfTables); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + taosRUnLockLatch(&pTaskInfo->lock); + return code; + } if (uid == 0) { if (numOfTables != 0) { @@ -1439,7 +1445,13 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSchemaWrapper(mtInfo.schema); return code; } - int32_t size = tableListGetSize(pTableListInfo); + int32_t size = 0; + code = tableListGetSize(pTableListInfo, &size); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + tDeleteSchemaWrapper(mtInfo.schema); + return code; + } code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**)&pInfo->dataReader, NULL, NULL); @@ -1520,7 +1532,10 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { SArray* pUidList = taosArrayInit(10, sizeof(uint64_t)); QUERY_CHECK_NULL(pUidList, code, lino, _end, terrno); - int32_t numOfTables = tableListGetSize(pTableListInfo); + int32_t numOfTables = 0; + code = tableListGetSize(pTableListInfo, &numOfTables); + QUERY_CHECK_CODE(code, lino, _end); + for (int32_t i = 0; i < numOfTables; ++i) { STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); QUERY_CHECK_NULL(pKeyInfo, code, lino, _end, terrno); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 031ffbb50e..966bda382b 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -1115,7 +1115,14 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pDeleterParam->suid = tableListGetSuid(pTableListInfo); // TODO extract uid list - int32_t numOfTables = tableListGetSize(pTableListInfo); + int32_t numOfTables = 0; + code = tableListGetSize(pTableListInfo, &numOfTables); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + taosMemoryFree(pDeleterParam); + return code; + } + pDeleterParam->pUidList = taosArrayInit(numOfTables, sizeof(uint64_t)); if (NULL == pDeleterParam->pUidList) { taosMemoryFree(pDeleterParam); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index e5289fa216..125a149339 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -213,7 +213,6 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData memcpy(pkey->pData, val, dataLen); } else if (IS_VAR_DATA_TYPE(pkey->type)) { memcpy(pkey->pData, val, varDataTLen(val)); - ASSERT(varDataTLen(val) <= pkey->bytes); } else { memcpy(pkey->pData, val, pkey->bytes); } @@ -241,7 +240,6 @@ static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { } else if (IS_VAR_DATA_TYPE(pkey->type)) { varDataCopy(pStart, pkey->pData); pStart += varDataTLen(pkey->pData); - ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); } else { memcpy(pStart, pkey->pData, pkey->bytes); pStart += pkey->bytes; @@ -740,7 +738,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { memcpy(data + (*columnLen), src, dataLen); int32_t v = (data + (*columnLen) + dataLen - (char*)pPage); - ASSERT(v > 0); + QUERY_CHECK_CONDITION((v > 0), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); contentLen = dataLen; } else { @@ -748,7 +746,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { char* src = colDataGetData(pColInfoData, j); memcpy(data + (*columnLen), src, varDataTLen(src)); int32_t v = (data + (*columnLen) + varDataTLen(src) - (char*)pPage); - ASSERT(v > 0); + QUERY_CHECK_CONDITION((v > 0), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); contentLen = varDataTLen(src); } @@ -762,7 +760,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { colDataSetNull_f(bitmap, (*rows)); } else { memcpy(data + (*columnLen), colDataGetData(pColInfoData, j), bytes); - ASSERT((data + (*columnLen) + bytes - (char*)pPage) <= getBufPageSize(pInfo->pBuf)); + QUERY_CHECK_CONDITION(((data + (*columnLen) + bytes - (char*)pPage) <= getBufPageSize(pInfo->pBuf)), code, + lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } contentLen = bytes; } @@ -1299,7 +1298,7 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { SStreamPartitionOperatorInfo* pInfo = pOperator->info; SSDataBlock* pDest = pInfo->binfo.pRes; - ASSERT(hasRemainPartion(pInfo)); + QUERY_CHECK_CONDITION((hasRemainPartion(pInfo)), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte; blockDataCleanup(pDest); int32_t rows = taosArrayGetSize(pParInfo->rowIds); @@ -1343,7 +1342,7 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { pDest->info.id.groupId = pParInfo->groupId; pOperator->resultInfo.totalRows += pDest->info.rows; pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte); - ASSERT(pDest->info.rows > 0); + QUERY_CHECK_CONDITION((pDest->info.rows > 0), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); _end: if (code != TSDB_CODE_SUCCESS) { @@ -1549,7 +1548,8 @@ static int32_t doStreamHashPartitionNext(SOperatorInfo* pOperator, SSDataBlock** return code; } default: - ASSERTS(0, "invalid SSDataBlock type"); + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); } // there is an scalar expression that needs to be calculated right before apply the group aggregation. diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b7b5e4fff9..66862c1484 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1082,7 +1082,10 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { return getBlockForEmptyTable(pOperator, pStart); } } else { // group by tag + no sort - int32_t numOfTables = tableListGetSize(pTableListInfo); + int32_t numOfTables = 0; + code = tableListGetSize(pTableListInfo, &numOfTables); + QUERY_CHECK_CODE(code, lino, _end); + if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) { // get empty group, mark processed & rm from hash void* pIte = taosHashIterate(pTableListInfo->remainGroups, NULL); @@ -1171,7 +1174,10 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + int32_t numOfTables = 0; + code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables); + QUERY_CHECK_CODE(code, lino, _end); + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { setOperatorCompleted(pOperator); if (pOperator->dynamicTask) { @@ -1304,7 +1310,7 @@ static int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { // scan table one by one sequentially if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { - int32_t numOfTables = 0; // tableListGetSize(pTaskInfo->pTableListInfo); + int32_t numOfTables = 0; STableKeyInfo tInfo = {0}; pInfo->countState = TABLE_COUNT_STATE_END; @@ -1319,7 +1325,13 @@ static int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pInfo->currentTable++; taosRLockLatch(&pTaskInfo->lock); - numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + numOfTables = 0; + code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables); + if (code != TSDB_CODE_SUCCESS) { + taosRUnLockLatch(&pTaskInfo->lock); + lino = __LINE__; + goto _end; + } if (pInfo->currentTable >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); @@ -3612,7 +3624,9 @@ static int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** QUERY_CHECK_NULL(tableIdList, code, lino, _end, terrno); // Transfer the Array of STableKeyInfo into uid list. - size_t size = tableListGetSize(pTableListInfo); + int32_t size = 0; + code = tableListGetSize(pTableListInfo, &size); + QUERY_CHECK_CODE(code, lino, _end); for (int32_t i = 0; i < size; ++i) { STableKeyInfo* pkeyInfo = tableListGetInfo(pTableListInfo, i); QUERY_CHECK_NULL(pkeyInfo, code, lino, _end, terrno); @@ -4628,7 +4642,13 @@ static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock* SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); - int32_t size = tableListGetSize(pInfo->pTableListInfo); + int32_t size = 0; + code = tableListGetSize(pInfo->pTableListInfo, &size); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + return code; + } + if (size == 0) { setTaskStatus(pTaskInfo, TASK_COMPLETED); (*ppRes) = NULL; @@ -4926,7 +4946,13 @@ static int32_t fetchNextSubTableBlockFromReader(SOperatorInfo* pOperator, STmsSu static int32_t setGroupStartEndIndex(STableMergeScanInfo* pInfo) { pInfo->bGroupProcessed = false; - size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + int32_t numOfTables = 0; + int32_t code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + return code; + } + int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i); @@ -5267,7 +5293,10 @@ int32_t doTableMergeScanParaSubTablesNext(SOperatorInfo* pOperator, SSDataBlock* int64_t st = taosGetTimestampUs(); - size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo); + int32_t tableListSize = 0; + code = tableListGetSize(pInfo->base.pTableListInfo, &tableListSize); + QUERY_CHECK_CODE(code, lino, _end); + if (!pInfo->hasGroupId) { pInfo->hasGroupId = true; @@ -5644,7 +5673,10 @@ void startGroupTableMergeScan(SOperatorInfo* pOperator) { qDebug("%s table merge scan start group %" PRIu64, GET_TASKID(pTaskInfo), pInfo->groupId); { - size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + int32_t numOfTables = 0; + code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables); + QUERY_CHECK_CODE(code, lino, _end); + int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i); @@ -5764,7 +5796,10 @@ int32_t doTableMergeScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { int64_t st = taosGetTimestampUs(); - size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo); + int32_t tableListSize = 0; + code = tableListGetSize(pInfo->base.pTableListInfo, &tableListSize); + QUERY_CHECK_CODE(code, lino, _end); + if (!pInfo->hasGroupId) { pInfo->hasGroupId = true; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 6559ba0e68..9eb9d60226 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2843,7 +2843,10 @@ int32_t createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanP QUERY_CHECK_CODE(code, lino, _error); pInfo->pTableListInfo = pTableListInfo; - size_t num = tableListGetSize(pTableListInfo); + int32_t num = 0; + code = tableListGetSize(pTableListInfo, &num); + QUERY_CHECK_CODE(code, lino, _error); + void* pList = tableListGetInfo(pTableListInfo, 0); code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 59c19a706c..85417bdc42 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -348,7 +348,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa saveColData(pRowVal->pRowVal, i, p, reset ? true : isNull); } else { - ASSERT(0); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); } } } @@ -362,10 +362,6 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); bool ascFill = FILL_IS_ASC_FILL(pFillInfo); -#if 0 - ASSERT(ascFill && (pFillInfo->currentKey >= pFillInfo->start) || (!ascFill && (pFillInfo->currentKey <= pFillInfo->start))); -#endif - while (pFillInfo->numOfCurrent < outputRows) { int64_t ts = ((int64_t*)pTsCol->pData)[pFillInfo->index]; @@ -392,7 +388,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t goto _end; } } else { - ASSERT(pFillInfo->currentKey == ts); + QUERY_CHECK_CONDITION((pFillInfo->currentKey == ts), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); int32_t index = pBlock->info.rows; int32_t nextRowIndex = pFillInfo->index + 1; @@ -500,7 +496,9 @@ static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bo } } -static void appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int64_t resultCapacity) { +static int32_t appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int64_t resultCapacity) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; /* * These data are generated according to fill strategy, since the current timestamp is out of the time window of * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. @@ -512,7 +510,14 @@ static void appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int64_ pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - ASSERT(pFillInfo->numOfCurrent == resultCapacity); + QUERY_CHECK_CONDITION((pFillInfo->numOfCurrent == resultCapacity), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { @@ -635,15 +640,6 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) // the endKey is now the aligned time window value. truncate time window isn't correct. pFillInfo->end = endKey; - -#if 0 - if (pFillInfo->order == TSDB_ORDER_ASC) { - ASSERT(pFillInfo->start <= pFillInfo->end); - } else { - ASSERT(pFillInfo->start >= pFillInfo->end); - } -#endif - pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; } @@ -687,7 +683,6 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma numOfRes = taosTimeCountIntervalForFill(lastKey, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision, pFillInfo->order); - ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { @@ -719,23 +714,30 @@ void taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* po int32_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t remain = taosNumOfRemainRows(pFillInfo); int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); - ASSERT(numOfRes <= capacity); + QUERY_CHECK_CONDITION((numOfRes <= capacity), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); // no data existed for fill operation now, append result according to the fill strategy if (remain == 0) { - appendFilledResult(pFillInfo, p, numOfRes); + code = appendFilledResult(pFillInfo, p, numOfRes); + QUERY_CHECK_CODE(code, lino, _end); } else { code = fillResultImpl(pFillInfo, p, (int32_t)numOfRes); - ASSERT(numOfRes == pFillInfo->numOfCurrent); + QUERY_CHECK_CODE(code, lino, _end); + QUERY_CHECK_CONDITION((numOfRes == pFillInfo->numOfCurrent), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%" PRId64 "-%" PRId64 ", currentKey:%" PRId64 ", current : % d, total : % d, %s", pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent, pFillInfo->numOfTotal, pFillInfo->id); +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 2108389aec..c87c482167 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -79,6 +79,7 @@ _error: int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { if (tBloomFilterIsFull(pBF)) { + uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_INVALID_PARA)); return TSDB_CODE_FAILED; } bool hasChange = false; diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 407223e937..4e2eb99d54 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -118,7 +118,7 @@ int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int3 } SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); - ASSERT(pNormalBf); + QUERY_CHECK_NULL(pNormalBf, code, lino, _end, terrno); if (tBloomFilterIsFull(pNormalBf)) { code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); From ee26b61eb634a40f94839d4c4507fcae593b5324 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 23 Aug 2024 11:13:36 +0800 Subject: [PATCH 22/59] add log --- source/libs/executor/src/executil.c | 2 ++ source/libs/executor/src/tfill.c | 31 ++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 04706982d1..29ffd900f2 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1562,6 +1562,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, if (nodeType(pNew) != QUERY_NODE_VALUE) { nodesDestroyList(groupNew); pAPI->metaReaderFn.clearReader(&mr); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } SValueNode* pValue = (SValueNode*)pNew; @@ -2153,6 +2154,7 @@ int32_t relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SAr } else if (p->info.colId < pmInfo->colId) { i++; } else { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 85417bdc42..cdfbd7a850 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -324,8 +324,11 @@ _end: static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal, bool reset) { +static int32_t copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal, bool reset) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId); + QUERY_CHECK_NULL(pTsCol, code, lino, _end, terrno); pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex]; for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { @@ -342,15 +345,24 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa } SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); + QUERY_CHECK_NULL(pSrcCol, code, lino, _end, terrno); bool isNull = colDataIsNull_s(pSrcCol, rowIndex); char* p = colDataGetData(pSrcCol, rowIndex); saveColData(pRowVal->pRowVal, i, p, reset ? true : isNull); } else { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + QUERY_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t outputRows) { @@ -368,10 +380,12 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // set the next value for interpolation if (pFillInfo->currentKey < ts && ascFill) { SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->next : &pFillInfo->prev; - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); + code = copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); + QUERY_CHECK_CODE(code, lino, _end); } else if (pFillInfo->currentKey > ts && !ascFill) { SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->prev : &pFillInfo->next; - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); + code = copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); + QUERY_CHECK_CODE(code, lino, _end); } if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && @@ -394,15 +408,18 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t int32_t nextRowIndex = pFillInfo->index + 1; if (pFillInfo->type == TSDB_FILL_NEXT) { if ((pFillInfo->index + 1) < pFillInfo->numOfRows) { - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, false); + code = copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, false); + QUERY_CHECK_CODE(code, lino, _end); } else { // reset to null after last row - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); + code = copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); + QUERY_CHECK_CODE(code, lino, _end); } } if (pFillInfo->type == TSDB_FILL_PREV) { if (nextRowIndex + 1 >= pFillInfo->numOfRows && !FILL_IS_ASC_FILL(pFillInfo)) { - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); + code = copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); + QUERY_CHECK_CODE(code, lino, _end); } } From 32cfc2cea5417b79944e06d14faa940e89ddd2c5 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 23 Aug 2024 03:26:44 +0000 Subject: [PATCH 23/59] fix/TD-28935-add-sdbfilelock --- source/dnode/mnode/impl/src/mndSync.c | 3 +++ source/dnode/mnode/sdb/src/sdbHash.c | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3c5724dde3..da5873039b 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -189,13 +189,16 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { goto _OUT; } + (void)taosThreadMutexLock(&pMnode->pSdb->filelock); code = sdbWriteWithoutFree(pMnode->pSdb, pRaw); if (code != 0) { mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); // code = 0; + (void)taosThreadMutexUnlock(&pMnode->pSdb->filelock); pMeta->code = code; goto _OUT; } + (void)taosThreadMutexUnlock(&pMnode->pSdb->filelock); pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index c488416287..03b45c25b4 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -148,13 +148,11 @@ static int32_t sdbGetkeySize(SSdb *pSdb, ESdbType type, const void *pKey) { static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { int32_t type = pRow->type; - (void)taosThreadMutexLock(&pSdb->filelock); sdbWriteLock(pSdb, type); SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); if (pOldRow != NULL) { sdbUnLock(pSdb, type); - (void)taosThreadMutexUnlock(&pSdb->filelock); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; @@ -166,7 +164,6 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { sdbUnLock(pSdb, type); - (void)taosThreadMutexUnlock(&pSdb->filelock); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; @@ -183,13 +180,11 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * sdbFreeRow(pSdb, pRow, false); terrno = code; sdbUnLock(pSdb, type); - (void)taosThreadMutexUnlock(&pSdb->filelock); return terrno; } } sdbUnLock(pSdb, type); - (void)taosThreadMutexUnlock(&pSdb->filelock); if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); From 61c85f43bbfa8847b6a85047dd2ff399c33ccbec Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 23 Aug 2024 04:09:14 +0000 Subject: [PATCH 24/59] fix/TD-31622-set-pointor-nulll-after-free --- source/dnode/mnode/impl/src/mndShow.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6989e1e4f1..99b7876755 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -180,7 +180,11 @@ static void mndFreeShowObj(SShowObj *pShow) { ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type]; if (freeFp != NULL) { if (pShow->pIter != NULL) { + mTrace("show:0x%" PRIx64 ", is destroying, data:%p, pIter:%p, ", pShow->id, pShow, pShow->pIter); + (*freeFp)(pMnode, pShow->pIter); + + pShow->pIter = NULL; } } From 18cb94bd1e2c03481bf946e45e2ba3253e43a7a0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Aug 2024 14:22:24 +0800 Subject: [PATCH 25/59] enh: refact to remove assert --- include/common/trow.h | 2 +- source/common/src/trow.c | 37 +++++++++++++++-------- source/dnode/vnode/src/sma/smaRollup.c | 4 +-- source/dnode/vnode/src/sma/smaTimeRange.c | 3 +- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 8d30384c61..6b836add21 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -310,7 +310,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx); int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx); -void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal); +int32_t tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal); typedef struct { STSchema *pSchema; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 942255cd7c..3e0e52a860 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -232,7 +232,6 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { if (!tdSTSRowIterNext(&iter, &sVal)) { break; } - ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); tdSCellValPrint(&sVal, cols[iter.colIdx - 1].type); } printf("\n"); @@ -389,7 +388,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } else if (TD_IS_KV_ROW(pIter->pRow)) { (void)tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { - ASSERT(0); + return false; } ++pIter->colIdx; @@ -409,7 +408,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r void *varBuf = NULL; bool isAlloc = false; - ASSERT(nColVal > 1); + if(nColVal <= 1) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) { pTColumn = &pTSchema->columns[iColumn]; @@ -423,9 +424,10 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r } if (iColumn == 0) { - ASSERT(pColVal && pColVal->cid == pTColumn->colId); - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); + if ((pColVal && pColVal->cid != pTColumn->colId) || (pTColumn->type != TSDB_DATA_TYPE_TIMESTAMP) || + (pTColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID)) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } } else { if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (pColVal && COL_VAL_IS_VALUE(pColVal)) { @@ -598,7 +600,10 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP - ASSERT(colIdx < tdRowGetNCols(pRow) - 1); + if (!(colIdx < tdRowGetNCols(pRow) - 1)) { + output->valType = TD_VTYPE_NONE; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } int32_t code = 0; if ((code = tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0)) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; @@ -699,7 +704,6 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { if (colIdx < 1) { - ASSERTS(0, "colIdx is %" PRIi64, colIdx); TAOS_RETURN(TSDB_CODE_INVALID_PARA); } --colIdx; @@ -775,7 +779,9 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); - ASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + if(!(pBuilder->nBitmaps > 0 && pBuilder->flen > 0)) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } uint32_t len = 0; switch (pBuilder->rowType) { @@ -814,7 +820,9 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - ASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + if(!(pBuilder->nBitmaps > 0 && pBuilder->flen > 0)) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } uint32_t len = 0; switch (pBuilder->rowType) { @@ -930,18 +938,20 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int TAOS_RETURN(TSDB_CODE_SUCCESS); } -void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { +int32_t tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { STColumn *pTColumn = &pTSchema->columns[iCol]; SCellVal cv = {0}; - ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0)); + if (!((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0))) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } if (TD_IS_TP_ROW(pRow)) { (void)tdSTpRowGetVal(pRow, pTColumn->colId, pTColumn->type, pTSchema->flen, pTColumn->offset, iCol - 1, &cv); } else if (TD_IS_KV_ROW(pRow)) { (void)tdSKvRowGetVal(pRow, pTColumn->colId, iCol - 1, &cv); } else { - ASSERT(0); + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } if (tdValTypeIsNone(cv.valType)) { @@ -960,4 +970,5 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV (void)memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); } } + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ebff03ff99..4acdc3e92c 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -928,7 +928,7 @@ static int32_t tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid, SRSmaInfo **ppR tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { + if (pRSmaInfo->suid != suid) { TAOS_RETURN(TSDB_CODE_APP_ERROR); } *ppRSmaInfo = pRSmaInfo; @@ -1645,7 +1645,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { ((oldStat == 2) && atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)) < TASK_TRIGGER_STAT_PAUSED)) { int32_t oldVal = atomic_fetch_add_32(&pRSmaStat->nFetchAll, 1); - if (ASSERTS(oldVal >= 0, "oldVal of nFetchAll: %d < 0", oldVal)) { + if (oldVal < 0) { code = TSDB_CODE_APP_ERROR; taosHashCancelIterate(infoHash, pIter); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 96010728c2..595bec44dd 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -350,8 +350,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } } - if (ASSERTS(pTsmaStat->pTSma->indexUid == indexUid, "indexUid:%" PRIi64 " != %" PRIi64, pTsmaStat->pTSma->indexUid, - indexUid)) { + if (pTsmaStat->pTSma->indexUid == indexUid) { code = TSDB_CODE_APP_ERROR; TSDB_CHECK_CODE(code, lino, _exit); } From cfd722e5a8dba0f4b776448329f47c8d5d1ca35b Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Aug 2024 14:26:51 +0800 Subject: [PATCH 26/59] enh: refact to remove assert --- source/dnode/vnode/src/sma/smaTimeRange.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 595bec44dd..9498e8c5ff 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -350,7 +350,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } } - if (pTsmaStat->pTSma->indexUid == indexUid) { + if (pTsmaStat->pTSma->indexUid != indexUid) { code = TSDB_CODE_APP_ERROR; TSDB_CHECK_CODE(code, lino, _exit); } From bb25893d0e0daf3e49b134d5fa42aafb44541bc6 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 23 Aug 2024 14:55:58 +0800 Subject: [PATCH 27/59] enh: rm ASSERT in mndArbGroup.c/geosWrapper.c/geomFuncTestUtil.cpp --- source/dnode/mnode/impl/inc/mndArbGroup.h | 2 +- source/dnode/mnode/impl/src/mndArbGroup.c | 8 +++- source/dnode/mnode/impl/src/mndDb.c | 46 ++++++++----------- source/libs/geometry/src/geosWrapper.c | 20 ++++++-- .../libs/geometry/test/geomFuncTestUtil.cpp | 2 +- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndArbGroup.h b/source/dnode/mnode/impl/inc/mndArbGroup.h index fcd11310e7..779d64c7e2 100644 --- a/source/dnode/mnode/impl/inc/mndArbGroup.h +++ b/source/dnode/mnode/impl/inc/mndArbGroup.h @@ -29,7 +29,7 @@ void mndReleaseArbGroup(SMnode *pMnode, SArbGroup *pObj); SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup); SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw); -void mndArbGroupInitFromVgObj(SVgObj *pVgObj, SArbGroup *outGroup); +int32_t mndArbGroupInitFromVgObj(SVgObj *pVgObj, SArbGroup *outGroup); int32_t mndSetCreateArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup); int32_t mndSetCreateArbGroupUndoLogs(STrans *pTrans, SArbGroup *pGroup); diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 32b62422e9..23434e5ee3 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -102,8 +102,10 @@ void mndReleaseArbGroup(SMnode *pMnode, SArbGroup *pGroup) { sdbRelease(pSdb, pGroup); } -void mndArbGroupInitFromVgObj(SVgObj *pVgObj, SArbGroup *outGroup) { - ASSERT(pVgObj->replica == 2); +int32_t mndArbGroupInitFromVgObj(SVgObj *pVgObj, SArbGroup *outGroup) { + if (pVgObj->replica != 2) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } (void)memset(outGroup, 0, sizeof(SArbGroup)); outGroup->dbUid = pVgObj->dbUid; outGroup->vgId = pVgObj->vgId; @@ -111,6 +113,8 @@ void mndArbGroupInitFromVgObj(SVgObj *pVgObj, SArbGroup *outGroup) { SArbGroupMember *pMember = &outGroup->members[i]; pMember->info.dnodeId = pVgObj->vnodeGid[i].dnodeId; } + + TAOS_RETURN(TSDB_CODE_SUCCESS); } SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index d0eed37f99..9b83fa8a1d 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -629,7 +629,7 @@ static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SVgObj *pVgObj = pVgroups + v; SArbGroup arbGroup = {0}; - mndArbGroupInitFromVgObj(pVgObj, &arbGroup); + TAOS_CHECK_RETURN(mndArbGroupInitFromVgObj(pVgObj, &arbGroup)); TAOS_CHECK_RETURN(mndSetCreateArbGroupRedoLogs(pTrans, &arbGroup)); } } @@ -663,7 +663,7 @@ static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SVgObj *pVgObj = pVgroups + v; SArbGroup arbGroup = {0}; - mndArbGroupInitFromVgObj(pVgObj, &arbGroup); + TAOS_CHECK_RETURN(mndArbGroupInitFromVgObj(pVgObj, &arbGroup)); TAOS_CHECK_RETURN(mndSetCreateArbGroupUndoLogs(pTrans, &arbGroup)); } } @@ -698,7 +698,7 @@ static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj * for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SVgObj *pVgObj = pVgroups + v; SArbGroup arbGroup = {0}; - mndArbGroupInitFromVgObj(pVgObj, &arbGroup); + TAOS_CHECK_RETURN(mndArbGroupInitFromVgObj(pVgObj, &arbGroup)); TAOS_CHECK_RETURN(mndSetCreateArbGroupCommitLogs(pTrans, &arbGroup)); } } @@ -1156,44 +1156,30 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p } static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { - int32_t code = 0; + int32_t code = 0, lino = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; + SVgObj *pVgroup = NULL; SArray *pArray = mndBuildDnodesArray(pMnode, 0); while (1) { - SVgObj *pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; if (mndVgroupInDb(pVgroup, pNewDb->uid)) { SVgObj newVgroup = {0}; - if ((code = mndBuildAlterVgroupAction(pMnode, pTrans, pOldDb, pNewDb, pVgroup, pArray, &newVgroup)) != 0) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - taosArrayDestroy(pArray); - TAOS_RETURN(code); - } + TAOS_CHECK_GOTO(mndBuildAlterVgroupAction(pMnode, pTrans, pOldDb, pNewDb, pVgroup, pArray, &newVgroup), &lino, + _err); + if (pNewDb->cfg.withArbitrator != pOldDb->cfg.withArbitrator) { if (pNewDb->cfg.withArbitrator) { SArbGroup arbGroup = {0}; - mndArbGroupInitFromVgObj(&newVgroup, &arbGroup); - if ((code = mndSetCreateArbGroupCommitLogs(pTrans, &arbGroup)) != 0) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - taosArrayDestroy(pArray); - TAOS_RETURN(code); - } - + TAOS_CHECK_GOTO(mndArbGroupInitFromVgObj(&newVgroup, &arbGroup), &lino, _err); + TAOS_CHECK_GOTO(mndSetCreateArbGroupCommitLogs(pTrans, &arbGroup), &lino, _err); } else { SArbGroup arbGroup = {0}; - mndArbGroupInitFromVgObj(pVgroup, &arbGroup); - if ((code = mndSetDropArbGroupCommitLogs(pTrans, &arbGroup)) != 0) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - taosArrayDestroy(pArray); - TAOS_RETURN(code); - } + TAOS_CHECK_GOTO(mndArbGroupInitFromVgObj(pVgroup, &arbGroup), &lino, _err); + TAOS_CHECK_GOTO(mndSetDropArbGroupCommitLogs(pTrans, &arbGroup), &lino, _err); } } } @@ -1203,6 +1189,14 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * taosArrayDestroy(pArray); TAOS_RETURN(code); + +_err: + mError("db:%s, %s failed at %d since %s", pNewDb->name, __func__, lino, tstrerror(code)); + + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + taosArrayDestroy(pArray); + TAOS_RETURN(code); } static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *pNew) { diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index dde34edc91..b349ab5cbe 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -331,18 +331,26 @@ int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *pr if (!preparedGeom1) { if (!swapped) { - ASSERT(relationFn); + if (!relationFn) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } *res = relationFn(geosCtx->handle, geom1, geom2); } else { - ASSERT(swappedRelationFn); + if (!swappedRelationFn) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } *res = swappedRelationFn(geosCtx->handle, geom1, geom2); } } else { if (!swapped) { - ASSERT(preparedRelationFn); + if (!preparedRelationFn) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } *res = preparedRelationFn(geosCtx->handle, preparedGeom1, geom2); } else { - ASSERT(swappedPreparedRelationFn); + if (!swappedPreparedRelationFn) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } *res = swappedPreparedRelationFn(geosCtx->handle, preparedGeom1, geom2); } } @@ -391,7 +399,9 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom) { SGeosContext *geosCtx = getThreadLocalGeosCtx(); - ASSERT(outputGeom); // it is not allowed if outputGeom is NULL + if (!outputGeom) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } *outputGeom = NULL; if (outputPreparedGeom) { // it means not to generate PreparedGeometry if outputPreparedGeom is NULL diff --git a/source/libs/geometry/test/geomFuncTestUtil.cpp b/source/libs/geometry/test/geomFuncTestUtil.cpp index 0918781499..25b2dd7c8d 100644 --- a/source/libs/geometry/test/geomFuncTestUtil.cpp +++ b/source/libs/geometry/test/geomFuncTestUtil.cpp @@ -66,7 +66,7 @@ void setScalarParam(SScalarParam *sclParam, int32_t type, void *valueArray, TDRo break; } default: { - ASSERT(0); + ASSERT_TRUE(false); break; } } From bc0ad2d461bebe89a03cf5cb485e88ce40e39954 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 23 Aug 2024 15:51:18 +0800 Subject: [PATCH 28/59] fix: check blockEncode result --- source/client/src/clientMsgHandler.c | 44 ++++++++++++++++++--- source/common/src/tdatablock.c | 6 +-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 6 +++ source/dnode/mnode/impl/src/mndShow.c | 18 +++++++-- source/dnode/vnode/src/tq/tqScan.c | 4 ++ source/libs/command/src/command.c | 4 ++ source/libs/command/src/explain.c | 4 ++ source/libs/executor/src/dataDispatcher.c | 8 ++++ source/libs/executor/src/groupoperator.c | 2 +- source/libs/stream/src/streamDispatch.c | 10 ++++- 10 files changed, 91 insertions(+), 15 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4dea9c17b0..771a22b7e3 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -576,8 +576,8 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { - blockDataDestroy(pBlock); - return TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _exit; } (*pRsp)->useconds = 0; @@ -589,6 +589,11 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, SHOW_VARIABLES_RESULT_COLS); + if(len < 0) { + uError("buildShowVariablesRsp error, len:%d", len); + code = terrno; + goto _exit; + } blockDataDestroy(pBlock); SET_PAYLOAD_LEN((*pRsp)->data, len, len); @@ -600,10 +605,21 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) { uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t)(rspSize - sizeof(SRetrieveTableRsp))); - return TSDB_CODE_TSC_INVALID_INPUT; + code = TSDB_CODE_TSC_INVALID_INPUT; + goto _exit; } return TSDB_CODE_SUCCESS; +_exit: + if(*pRsp) { + taosMemoryFree(*pRsp); + *pRsp = NULL; + } + if(pBlock) { + blockDataDestroy(pBlock); + pBlock = NULL; + } + return code; } int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { @@ -711,8 +727,8 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { - blockDataDestroy(pBlock); - return TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _exit; } (*pRsp)->useconds = 0; @@ -725,6 +741,11 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, COMPACT_DB_RESULT_COLS); + if(len < 0) { + uError("buildRetriveTableRspForCompactDb error, len:%d", len); + code = terrno; + goto _exit; + } blockDataDestroy(pBlock); SET_PAYLOAD_LEN((*pRsp)->data, len, len); @@ -736,10 +757,21 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) { uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t)(rspSize - sizeof(SRetrieveTableRsp))); - return TSDB_CODE_TSC_INVALID_INPUT; + code = TSDB_CODE_TSC_INVALID_INPUT; + goto _exit; } return TSDB_CODE_SUCCESS; +_exit: + if(*pRsp) { + taosMemoryFree(*pRsp); + *pRsp = NULL; + } + if(pBlock) { + blockDataDestroy(pBlock); + pBlock = NULL; + } + return code; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b1e1dcadc2..c9314318ee 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2888,6 +2888,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha return code; } +// return length of encoded data, return -1 if failed int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { int32_t dataLen = 0; @@ -2921,7 +2922,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); if (pColInfoData == NULL) { - return terrno; + return -1; } *((int8_t*)data) = pColInfoData->info.type; @@ -2940,7 +2941,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { for (int32_t col = 0; col < numOfCols; ++col) { SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col); if (pColRes == NULL) { - return terrno; + return -1; } // copy the null bitmap @@ -2991,7 +2992,6 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { *actualLen = dataLen; *groupId = pBlock->info.id.groupId; - ASSERT(dataLen > 0); return dataLen; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 70f258a362..0de0a34c25 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -479,6 +479,12 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t len = blockEncode(pBlock, pStart, numOfCols); + if(len < 0) { + dError("failed to retrieve data since %s", tstrerror(code)); + blockDataDestroy(pBlock); + rpcFreeCont(pRsp); + return terrno; + } pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6989e1e4f1..7cabeed0e4 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -330,11 +330,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { - mndReleaseShowObj(pShow, false); - code = TSDB_CODE_OUT_OF_MEMORY; mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code)); - blockDataDestroy(pBlock); - TAOS_RETURN(code); + code = terrno; + goto _exit; } pRsp->handle = htobe64(pShow->id); @@ -356,6 +354,11 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); + if(len < 0){ + mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code)); + code = terrno; + return code; + } } pRsp->numOfRows = htonl(rowsRead); @@ -374,6 +377,13 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { blockDataDestroy(pBlock); return TSDB_CODE_SUCCESS; +_exit: + mndReleaseShowObj(pShow, false); + blockDataDestroy(pBlock); + if(pRsp) { + rpcFreeCont(pRsp); + } + return code; } static bool mndCheckRetrieveFinished(SShowObj *pShow) { diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 4357456790..8ca50f9de7 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -29,6 +29,10 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOf pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + if(actualLen < 0){ + taosMemoryFree(buf); + return terrno; + } actualLen += sizeof(SRetrieveTableRspForTmq); if (taosArrayPush(((SMqDataRspCommon*)pRsp)->blockDataLen, &actualLen) == NULL){ taosMemoryFree(buf); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 11ddc89d4c..552fa29f5f 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -50,6 +50,10 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, numOfCols); + if(len < 0) { + taosMemoryFree(*pRsp); + return terrno; + } SET_PAYLOAD_LEN((*pRsp)->data, len, len); int32_t payloadLen = len + PAYLOAD_PREFIX_LEN; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 3a73c05de2..b9f79d1e00 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1978,6 +1978,10 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data + PAYLOAD_PREFIX_LEN, taosArrayGetSize(pBlock->pDataBlock)); + if(len < 0) { + qError("qExplainGetRspFromCtx: blockEncode failed"); + QRY_ERR_JRET(terrno); + } rsp->compLen = htonl(len); rsp->payloadLen = htonl(len); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 6eb1f9cd18..fb3ed06224 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -106,6 +106,10 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* } int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, numOfCols); + if(dataLen < 0) { + qError("failed to encode data block, code: %d", dataLen); + return terrno; + } int32_t len = tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); if (len < dataLen) { @@ -120,6 +124,10 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* } } else { pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); + if(pEntry->dataLen < 0) { + qError("failed to encode data block, code: %d", pEntry->dataLen); + return terrno; + } pEntry->rawLen = pEntry->dataLen; } } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index e5289fa216..959ace1228 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -548,7 +548,7 @@ int32_t createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNo SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _error; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index de67d55703..8928138950 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -158,6 +158,10 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl pRetrieve->version = htobe64(pBlock->info.version); int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); + if(actualLen < 0) { + taosMemoryFree(pRetrieve); + return terrno; + } SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; @@ -1064,7 +1068,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { - return -1; + return terrno; } SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; @@ -1084,6 +1088,10 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); + if(actualLen < 0) { + taosMemoryFree(buf); + return terrno; + } SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; From 734e59d5a7b7f77edc37ddeff016f747b7f46a86 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Fri, 23 Aug 2024 15:52:22 +0800 Subject: [PATCH 29/59] fix:[TD-31654] Free memory at the end of substrIdxFunction to avoid memory leak. --- source/libs/scalar/src/sclfunc.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 477a768ccf..ed55bde663 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1544,9 +1544,15 @@ int32_t replaceFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pO needFreeFrom = true; } if (GET_PARAM_TYPE(&pInput[2]) != GET_PARAM_TYPE(&pInput[0])) { - SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[2], colIdx3)), &toStr, - varDataLen(colDataGetData(pInputData[2], colIdx3)), &toLen, - GET_PARAM_TYPE(&pInput[0]))); + code = convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[2], colIdx3)), &toStr, + varDataLen(colDataGetData(pInputData[2], colIdx3)), &toLen, + GET_PARAM_TYPE(&pInput[0])); + if (TSDB_CODE_SUCCESS != code) { + if (needFreeFrom) { + taosMemoryFree(fromStr); + } + goto _return; + } needFreeTo = true; } @@ -1660,9 +1666,9 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * SCL_ERR_JRET(colDataSetVal(pOutputData, k, output, false)); } - -_return: pOutput->numOfRows = numOfRows; +_return: + taosMemoryFree(outputBuf); return code; } From 7c15c54f4b778126c1ddb014d8106d4954fec0df Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 23 Aug 2024 16:15:14 +0800 Subject: [PATCH 30/59] fix: partition node memory leak --- source/libs/planner/src/planOptimizer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c7ce9aed76..6bb314befd 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -6493,6 +6493,7 @@ static int32_t partitionColsOpt(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub pSort->calcGroupId = true; code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pNode, (SLogicNode*)pSort); if (code == TSDB_CODE_SUCCESS) { + nodesDestroyNode((SNode*)pNode); pCxt->optimized = true; } else { nodesDestroyNode((SNode*)pSort); From 23365f1cf61c5f5303a719de9282b77cd2493ef9 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 23 Aug 2024 16:58:05 +0800 Subject: [PATCH 31/59] fix: (last) memleak in mergeLastCid --- source/dnode/vnode/src/tsdb/tsdbCache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 20d4363c73..ae57576279 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3162,6 +3162,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC SLastCol lastCol = {.rowKey = rowKey.key, .colVal = *pColVal, .cacheStatus = TSDB_LAST_CACHE_VALID}; TAOS_CHECK_GOTO(tsdbCacheReallocSLastCol(&lastCol, NULL), &lino, _err); + tsdbCacheFreeSLastColItem(lastColVal); taosArraySet(pColArray, iCol, &lastCol); int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); if (aColIndex >= 0) { From 9aaa1c66b309b3045ef3322c1bab20fb42e81fde Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 23 Aug 2024 18:40:32 +0800 Subject: [PATCH 32/59] fix: check arraypush result --- source/libs/parser/src/parTranslater.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 05cceb656e..a99cf6423d 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -11068,7 +11068,10 @@ static int32_t createStreamReqVersionInfo(SSDataBlock* pBlock, SArray** pArray, for (int32_t i = 0; i < pBlock->info.rows; ++i) { SVgroupVer v = {.vgId = *(int32_t*)colDataGetData(pCol1, i), .ver = *(int64_t*)colDataGetData(pCol2, i)}; - (void)taosArrayPush(*pArray, &v); + if((taosArrayPush(*pArray, &v)) == NULL) { + taosArrayDestroy(*pArray); + return terrno; + } } } else { int32_t precision = (pInterval->interval > 0) ? pInterval->precision : TSDB_TIME_PRECISION_MILLI; From 4b6193a07e30e826d9b129d6a9ee498facbb0d72 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Fri, 23 Aug 2024 23:18:04 +0800 Subject: [PATCH 33/59] feat:Optimising the duration and keep of the show create database command --- source/libs/command/src/command.c | 39 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 7344f7d521..fbf7be3053 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -344,6 +344,26 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { return TSDB_CACHE_MODEL_NONE_STR; } +static int formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { + int len = 0; + if (timeInMinutes % 1440 == 0) { + int days = timeInMinutes / 1440; + len = snprintf(NULL, 0, "%dd", days); + *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); + sprintf(*buffer, "%dd", days); + } else if (timeInMinutes % 60 == 0) { + int hours = timeInMinutes / 60; + len = snprintf(NULL, 0, "%dh", hours); + *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); + sprintf(*buffer, "%dh", hours); + } else { + len = snprintf(NULL, 0, "%dm", timeInMinutes); + *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); + sprintf(*buffer, "%dm", timeInMinutes); + } + return len; +} + static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1)); pBlock->info.rows = 1; @@ -381,20 +401,29 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, } else if (pCfg->hashPrefix < 0) { hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; } - + char* durationStr = NULL; + formatDurationOrKeep(&durationStr, pCfg->daysPerFile); + char* keep0Str = NULL; + formatDurationOrKeep(&keep0Str, pCfg->daysToKeep0); + char* keep1Str = NULL; + formatDurationOrKeep(&keep1Str, pCfg->daysToKeep1); + char* keep2Str = NULL; + formatDurationOrKeep(&keep2Str, pCfg->daysToKeep2); if (IS_SYS_DBNAME(dbName)) { len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); } else { len += sprintf(buf2 + VARSTR_HEADER_SIZE, - "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " - "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d " + "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %s " + "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %s,%s,%s PAGES %d PAGESIZE %d " "PRECISION '%s' REPLICA %d " "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d", dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, - pCfg->daysPerFile, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, - pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages, pCfg->pageSize, prec, + durationStr, + pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, + keep0Str, keep1Str, keep2Str, + pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize, From 7c207dcf8e2967b6fbc66254879cc0fa4038e754 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Fri, 23 Aug 2024 23:39:14 +0800 Subject: [PATCH 34/59] update pr --- source/libs/command/src/command.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index fbf7be3053..1882db1dda 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -344,7 +344,7 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { return TSDB_CACHE_MODEL_NONE_STR; } -static int formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { +static void formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { int len = 0; if (timeInMinutes % 1440 == 0) { int days = timeInMinutes / 1440; @@ -361,7 +361,6 @@ static int formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); sprintf(*buffer, "%dm", timeInMinutes); } - return len; } static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { @@ -402,13 +401,13 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; } char* durationStr = NULL; - formatDurationOrKeep(&durationStr, pCfg->daysPerFile); + (void)formatDurationOrKeep(&durationStr, pCfg->daysPerFile); char* keep0Str = NULL; - formatDurationOrKeep(&keep0Str, pCfg->daysToKeep0); + (void)formatDurationOrKeep(&keep0Str, pCfg->daysToKeep0); char* keep1Str = NULL; - formatDurationOrKeep(&keep1Str, pCfg->daysToKeep1); + (void)formatDurationOrKeep(&keep1Str, pCfg->daysToKeep1); char* keep2Str = NULL; - formatDurationOrKeep(&keep2Str, pCfg->daysToKeep2); + (void)formatDurationOrKeep(&keep2Str, pCfg->daysToKeep2); if (IS_SYS_DBNAME(dbName)) { len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); } else { @@ -435,6 +434,10 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, } taosMemoryFree(pRetentions); + taosMemoryFree(durationStr); + taosMemoryFree(keep0Str); + taosMemoryFree(keep1Str); + taosMemoryFree(keep2Str); (varDataLen(buf2)) = len; From e1998d151b7d221a86d4028b40438a3c3c8d1f54 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Sat, 24 Aug 2024 10:19:25 +0000 Subject: [PATCH 35/59] update qid log --- docs/examples/c/with_reqid_demo.c | 2 +- source/client/src/clientEnv.c | 6 +- source/client/src/clientImpl.c | 38 ++-- source/client/src/clientMain.c | 16 +- source/client/src/clientRawBlockWrite.c | 2 +- source/client/src/clientTmq.c | 29 +-- source/dnode/mgmt/node_util/inc/dmUtil.h | 12 +- source/dnode/mnode/impl/inc/mndInt.h | 12 +- source/dnode/vnode/src/inc/vnd.h | 12 +- source/dnode/vnode/src/tq/tq.c | 18 +- source/dnode/vnode/src/tq/tqRead.c | 6 +- source/dnode/vnode/src/tq/tqUtil.c | 18 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/libs/catalog/inc/catalogInt.h | 243 +++++++++++---------- source/libs/catalog/src/ctgAsync.c | 62 +++--- source/libs/catalog/src/ctgRemote.c | 10 +- source/libs/catalog/src/ctgUtil.c | 205 ++++++++--------- source/libs/executor/src/executor.c | 16 +- source/libs/executor/src/querytask.c | 9 +- source/libs/planner/src/planner.c | 13 +- source/libs/qworker/inc/qwInt.h | 181 +++++++-------- source/libs/scheduler/inc/schInt.h | 152 ++++++------- source/libs/scheduler/src/schJob.c | 87 ++++---- source/libs/scheduler/src/schRemote.c | 4 +- source/libs/scheduler/src/schTask.c | 63 +++--- source/libs/stream/src/streamCheckStatus.c | 16 +- source/libs/stream/src/streamDispatch.c | 6 +- source/libs/stream/src/streamExec.c | 2 +- source/libs/stream/src/streamTask.c | 2 +- source/libs/transport/inc/transLog.h | 12 +- 30 files changed, 633 insertions(+), 623 deletions(-) diff --git a/docs/examples/c/with_reqid_demo.c b/docs/examples/c/with_reqid_demo.c index 21de28ffe8..0cfedf3b3d 100644 --- a/docs/examples/c/with_reqid_demo.c +++ b/docs/examples/c/with_reqid_demo.c @@ -45,7 +45,7 @@ static int DemoWithReqId() { TAOS_RES *result = taos_query_with_reqid(taos, sql, reqid); code = taos_errno(result); if (code != 0) { - fprintf(stderr, "Failed to execute sql with QID: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, + fprintf(stderr, "Failed to execute sql with qid: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, taos_errstr(result)); taos_close(taos); taos_cleanup(); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 491a49778c..d5287d5062 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -94,7 +94,7 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 - ", current:%d, app current:%d, total:%d, QID:0x%" PRIx64, + ", current:%d, app current:%d, total:%d, qid:0x%" PRIx64, pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } @@ -249,7 +249,7 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t reqType = SLOW_LOG_TYPE_OTHERS; int64_t duration = taosGetTimestampUs() - pRequest->metric.start; - tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", QID:0x%" PRIx64 + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", qid:0x%" PRIx64 " elapsed:%.2f ms, " "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); @@ -294,7 +294,7 @@ static void deregisterRequest(SRequestObj *pRequest) { checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) { (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) { - taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s", + taosPrintSlowLog("PID:%d, Conn:%u, qid:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 039e17fd87..6642e1077a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -232,7 +232,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self)); if (err) { - tscError("%" PRId64 " failed to add to request container, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s", + tscError("%" PRId64 " failed to add to request container, qid:0x%" PRIx64 ", conn:%" PRId64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; @@ -243,7 +243,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { if (TSDB_CODE_SUCCESS != nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { - tscError("%" PRId64 " failed to create node allocator, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s", + tscError("%" PRId64 " failed to create node allocator, qid:0x%" PRIx64 ", conn:%" PRId64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; @@ -251,7 +251,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, } } - tscDebugL("0x%" PRIx64 " SQL: %s, QID:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); + tscDebugL("0x%" PRIx64 " SQL: %s, qid:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); return TSDB_CODE_SUCCESS; } @@ -365,10 +365,10 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code), + tscError("0x%" PRIx64 " fetch results failed, code:%s, qid:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, qid:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); } @@ -974,7 +974,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { break; } default: - tscError("0x%" PRIx64 ", invalid exec result for request type %d, QID:0x%" PRIx64, pRequest->self, pRequest->type, + tscError("0x%" PRIx64 ", invalid exec result for request type %d, qid:0x%" PRIx64, pRequest->self, pRequest->type, pRequest->requestId); code = TSDB_CODE_APP_ERROR; } @@ -1019,7 +1019,7 @@ void returnToUser(SRequestObj* pRequest) { (void)releaseRequest(pRequest->relation.userRefId); return; } else { - tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self, + tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, qid:0x%" PRIx64, pRequest->self, pRequest->relation.userRefId, pRequest->requestId); } } @@ -1084,7 +1084,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) { SSDataBlock* pBlock = NULL; if (TSDB_CODE_SUCCESS != createResultBlock(res, rowNum, &pBlock)) { - tscError("0x%" PRIx64 ", create result block failed, QID:0x%" PRIx64, pRequest->self, pRequest->requestId); + tscError("0x%" PRIx64 ", create result block failed, qid:0x%" PRIx64, pRequest->self, pRequest->requestId); return; } @@ -1093,7 +1093,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) { continuePostSubQuery(pNextReq, pBlock); (void)releaseRequest(pRequest->relation.nextRefId); } else { - tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self, + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, qid:0x%" PRIx64, pRequest->self, pRequest->relation.nextRefId, pRequest->requestId); } @@ -1112,7 +1112,7 @@ void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) { continuePostSubQuery(pNextReq, NULL); (void)releaseRequest(pRequest->relation.nextRefId); } else { - tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self, + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, qid:0x%" PRIx64, pRequest->self, pRequest->relation.nextRefId, pRequest->requestId); } } @@ -1143,11 +1143,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { } taosMemoryFree(pResult); - tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code), + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, qid:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, QID:0x%" PRIx64, pRequest->self, + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, qid:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId); (void)removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)); restartAsyncQuery(pRequest, code); @@ -1580,7 +1580,7 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta *pTscObj = NULL; return terrno; } else { - tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, QID:0x%" PRIx64, (*pTscObj)->id, + tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, qid:0x%" PRIx64, (*pTscObj)->id, (*pTscObj)->connId, (*pTscObj)->pAppInfo->pTransporter, pRequest->requestId); destroyRequest(pRequest); } @@ -1711,7 +1711,7 @@ int32_t doProcessMsgFromServer(void* param) { char tbuf[40] = {0}; TRACE_TO_STR(trace, tbuf); - tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, QID:%s", pMsg->info.handle, + tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, qid:%s", pMsg->info.handle, TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); if (pSendInfo->requestObjRefId != 0) { @@ -1909,7 +1909,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) return NULL; } - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, qid:0x%" PRIx64, pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; @@ -2859,7 +2859,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; - tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code, + tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, qid:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); pResultInfo->pData = pResult; @@ -2882,10 +2882,10 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code), + tscError("0x%" PRIx64 " fetch results failed, code:%s, qid:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, qid:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); @@ -2937,7 +2937,7 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param int32_t code = schedulerFetchRows(pRequest->body.queryJob, &req); if (TSDB_CODE_SUCCESS != code) { tscError("0x%" PRIx64 " failed to schedule fetch rows", pRequest->requestId); - //pRequest->body.fetchFp(param, pRequest, code); + // pRequest->body.fetchFp(param, pRequest, code); } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 55401d7eb2..5732df8013 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -941,7 +941,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t SRequestObj *pRequest = pWrapper->pRequest; SQuery *pQuery = pRequest->pQuery; - qDebug("0x%" PRIx64 " start to semantic analysis, QID:0x%" PRIx64, pRequest->self, pRequest->requestId); + qDebug("0x%" PRIx64 " start to semantic analysis, qid:0x%" PRIx64, pRequest->self, pRequest->requestId); int64_t analyseStart = taosGetTimestampUs(); pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart; @@ -1060,14 +1060,14 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64, + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, qid:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); restartAsyncQuery(pRequest, code); return; } // return to app directly - tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, QID:0x%" PRIx64, pRequest->self, tstrerror(code), + tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, qid:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); pRequest->code = code; returnToUser(pRequest); @@ -1117,7 +1117,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c SQuery *pQuery = pRequest->pQuery; pRequest->metric.ctgCostUs += taosGetTimestampUs() - pRequest->metric.ctgStart; - qDebug("0x%" PRIx64 " start to continue parse, QID:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId, + qDebug("0x%" PRIx64 " start to continue parse, qid:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId, tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { @@ -1130,7 +1130,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c } if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code, + tscError("0x%" PRIx64 " error happens, code:%d - %s, qid:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); pRequest->pWrapper = NULL; @@ -1147,7 +1147,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) } if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code, + tscError("0x%" PRIx64 " error happens, code:%d - %s, qid:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); pRequest->pWrapper = NULL; @@ -1265,7 +1265,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), + tscError("0x%" PRIx64 " error happens, code:%d - %s, qid:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); pRequest->pWrapper = NULL; @@ -1273,7 +1273,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64, + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, qid:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); (void)refreshMeta(pRequest->pTscObj, pRequest); // ignore return code,try again pRequest->prevCode = code; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 7bd2b726e9..8eb2f46dfb 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -47,7 +47,7 @@ } \ } while (0) -#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64 +#define LOG_ID_TAG "connId:0x%" PRIx64 ",qid:0x%" PRIx64 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId #define TMQ_META_VERSION "1.0" diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 1db1df5d2e..d67f3565b9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1570,14 +1570,15 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (msgEpoch < clientEpoch) { // do not write into queue since updating epoch reset tscWarn("consumer:0x%" PRIx64 - " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, QID:0x%" PRIx64, + " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, qid:0x%" PRIx64, tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; goto END; } - if(msgEpoch != clientEpoch) { - tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, + if (msgEpoch != clientEpoch) { + tscError("consumer:0x%" PRIx64 + " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; goto END; @@ -1602,7 +1603,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); - tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, QID:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, qid:0x%" PRIx64, tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder = {0}; @@ -1634,7 +1635,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } tDecoderClear(&decoder); (void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); - tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, QID:0x%" PRIx64, tmq->consumerId, vgId, + tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, qid:0x%" PRIx64, tmq->consumerId, vgId, requestId); } else { // invalid rspType tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); @@ -1651,7 +1652,7 @@ END: } } int32_t total = taosQueueItemSize(tmq->mqueue); - tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, QID:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, qid:0x%" PRIx64, tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId); if (tmq) (void)tsem2_post(&tmq->rspSem); @@ -1879,7 +1880,7 @@ void changeByteEndian(char* pData) { // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | version: int32_t blockVersion = *(int32_t*)p; - if(blockVersion != BLOCK_VERSION_1) { + if (blockVersion != BLOCK_VERSION_1) { tscError("invalid block version:%d", blockVersion); return; } @@ -2049,7 +2050,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, QID:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, qid:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); if (code != 0) { return code; @@ -2222,7 +2223,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 - ", total:%" PRId64 ", QID:0x%" PRIx64, + ", total:%" PRId64 ", qid:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); tmqFreeRspWrapper(pRspWrapper); @@ -2245,7 +2246,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } } tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64, + ", vg total:%" PRId64 ", total:%" PRId64 ", qid:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); taosFreeQitem(pRspWrapper); @@ -2356,7 +2357,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tmq->consumerId, pDataRsp->blockNum != 0); if (pDataRsp->blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", QID:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", qid:0x%" PRIx64, tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); tmqFreeRspWrapper(pRspWrapper); @@ -2378,7 +2379,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64, + ", vg total:%" PRId64 ", total:%" PRId64 ", qid:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); @@ -2991,7 +2992,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); + tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, qid:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); } @@ -3498,7 +3499,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); - tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, QID:0x%" PRIx64, tmq->consumerId, + tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, qid:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); if (code != 0) { diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 83bc02ed1c..95556bba86 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -83,12 +83,12 @@ extern "C" { }\ } -#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", QID:%s", __VA_ARGS__, buf);}} -#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", QID:%s", __VA_ARGS__, buf);}} -#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", QID:%s", __VA_ARGS__, buf);}} -#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", QID:%s", __VA_ARGS__, buf);}} -#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", QID:%s", __VA_ARGS__, buf);}} -#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", QID:%s", __VA_ARGS__, buf);}} +#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", qid:%s", __VA_ARGS__, buf);}} +#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", qid:%s", __VA_ARGS__, buf);}} +#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", qid:%s", __VA_ARGS__, buf);}} +#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", qid:%s", __VA_ARGS__, buf);}} +#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", qid:%s", __VA_ARGS__, buf);}} +#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", qid:%s", __VA_ARGS__, buf);}} // clang-format on diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8681373198..5a4062daf6 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -41,12 +41,12 @@ extern "C" { #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} -#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", QID:%s", __VA_ARGS__, buf);}} -#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", QID:%s", __VA_ARGS__, buf);}} -#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", QID:%s", __VA_ARGS__, buf);}} -#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", QID:%s", __VA_ARGS__, buf);}} -#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", QID:%s", __VA_ARGS__, buf);}} -#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", QID:%s", __VA_ARGS__, buf);}} +#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", qid:%s", __VA_ARGS__, buf);}} +#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", qid:%s", __VA_ARGS__, buf);}} +#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", qid:%s", __VA_ARGS__, buf);}} +#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", qid:%s", __VA_ARGS__, buf);}} +#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", qid:%s", __VA_ARGS__, buf);}} +#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", qid:%s", __VA_ARGS__, buf);}} // clang-format on #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 3c1f529ec4..dbcda45ef5 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -32,12 +32,12 @@ extern "C" { #define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }} while(0) #define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }} while(0) -#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0) -#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", QID:%s", __VA_ARGS__, buf);}} while(0) -#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", QID:%s", __VA_ARGS__, buf);}} while(0) -#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", QID:%s", __VA_ARGS__, buf);}} while(0) -#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", QID:%s", __VA_ARGS__, buf);}} while(0) -#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", QID:%s", __VA_ARGS__, buf);}} while(0) +#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", qid:%s", __VA_ARGS__, buf);}} while(0) +#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", qid:%s", __VA_ARGS__, buf);}} while(0) +#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", qid:%s", __VA_ARGS__, buf);}} while(0) +#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", qid:%s", __VA_ARGS__, buf);}} while(0) +#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", qid:%s", __VA_ARGS__, buf);}} while(0) +#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", qid:%s", __VA_ARGS__, buf);}} while(0) // clang-format on diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3df859d6cb..b280d62b3a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -173,7 +173,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { dataRsp.common.blockNum = 0; char buf[TSDB_OFFSET_LEN] = {0}; (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset); - tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, QID:0x%" PRIx64, req.consumerId, vgId, buf, + tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, qid:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId); code = tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -193,7 +193,7 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset); (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset); - tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, QID:0x%" PRIx64, + tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, qid:0x%" PRIx64, vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId); return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); @@ -421,7 +421,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { char buf[TSDB_OFFSET_LEN] = {0}; (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, QID:0x%" PRIx64, + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, qid:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); @@ -777,9 +777,9 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory, (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam, nextProcessVer); - if(pChkInfo->checkpointVer > pChkInfo->nextProcessVer) { - tqError("vgId:%d build stream task, s-task:%s, checkpointVer:%" PRId64 " > nextProcessVer:%" PRId64, - vgId, pTask->id.idStr, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); + if (pChkInfo->checkpointVer > pChkInfo->nextProcessVer) { + tqError("vgId:%d build stream task, s-task:%s, checkpointVer:%" PRId64 " > nextProcessVer:%" PRId64, vgId, + pTask->id.idStr, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); return TSDB_CODE_STREAM_INTERNAL_ERROR; } } @@ -956,9 +956,9 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // the following procedure should be executed, no matter status is stop/pause or not tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El); - if(pTask->info.fillHistory != 1) { + if (pTask->info.fillHistory != 1) { tqError("s-task:%s fill-history is disabled, unexpected", id); - return TSDB_CODE_STREAM_INTERNAL_ERROR; + return TSDB_CODE_STREAM_INTERNAL_ERROR; } // 1. get the related stream task @@ -976,7 +976,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { return code; // todo: handle failure } - if(pStreamTask->info.taskLevel != TASK_LEVEL__SOURCE) { + if (pStreamTask->info.taskLevel != TASK_LEVEL__SOURCE) { tqError("s-task:%s fill-history task related stream task level:%d, unexpected", id, pStreamTask->info.taskLevel); return TSDB_CODE_STREAM_INTERNAL_ERROR; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index e9385bc9da..98cfc6888c 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -214,12 +214,12 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t while (offset <= appliedVer) { if (walFetchHead(pHandle->pWalReader, offset) < 0) { tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 - ", no more log to return, QID:0x%" PRIx64 " 0x%" PRIx64, + ", no more log to return, qid:0x%" PRIx64 " 0x%" PRIx64, pHandle->consumerId, pHandle->epoch, vgId, offset, reqId, id); goto END; } - tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, QID:0x%" PRIx64 " 0x%" PRIx64, + tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, qid:0x%" PRIx64 " 0x%" PRIx64, vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id); if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) { @@ -676,7 +676,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* pReader->cachedSchemaSuid = suid; pReader->cachedSchemaVer = sversion; - if(pReader->cachedSchemaVer != pReader->pSchemaWrapper->version) { + if (pReader->cachedSchemaVer != pReader->pSchemaWrapper->version) { tqError("vgId:%d, schema version mismatch, suid:%" PRId64 ", uid:%" PRId64 ", version:%d, cached version:%d", vgId, suid, uid, sversion, pReader->pSchemaWrapper->version); return TSDB_CODE_TQ_INTERNAL_ERROR; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 1b48ad9e18..fc97a542a7 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -92,7 +92,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand char formatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal); tqDebug("tmq poll: consumer:0x%" PRIx64 - ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. QID:0x%" PRIx64, + ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. qid:0x%" PRIx64, consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); return 0; } else { @@ -176,7 +176,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, end : { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, QID:0x%" PRIx64 + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, qid:0x%" PRIx64 " code:%d", consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code); tDeleteMqDataRsp(&dataRsp); @@ -245,7 +245,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int32_t totalMetaRows = 0; while (1) { int32_t savedEpoch = atomic_load_32(&pHandle->epoch); - if(savedEpoch > pRequest->epoch) { + if (savedEpoch > pRequest->epoch) { tqError("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, savedEpoch error, vgId:%d offset %" PRId64, pRequest->consumerId, pRequest->epoch, vgId, fetchVer); code = TSDB_CODE_TQ_INTERNAL_ERROR; @@ -256,7 +256,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (totalMetaRows > 0) { tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); - if(totalRows != 0) { + if (totalRows != 0) { tqError("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, totalRows error, vgId:%d offset %" PRId64, pRequest->consumerId, pRequest->epoch, vgId, fetchVer); code = code == 0 ? TSDB_CODE_TQ_INTERNAL_ERROR : code; @@ -332,9 +332,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); continue; } - int32_t tLen = sizeof(SMqRspHead) + len; - void* tBuf = taosMemoryCalloc(1, tLen); - if (tBuf == NULL){ + int32_t tLen = sizeof(SMqRspHead) + len; + void* tBuf = taosMemoryCalloc(1, tLen); + if (tBuf == NULL) { code = TAOS_GET_TERRNO(terrno); goto END; } @@ -348,11 +348,11 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); continue; } - if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL){ + if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL) { code = TAOS_GET_TERRNO(terrno); goto END; } - if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL){ + if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL) { code = TAOS_GET_TERRNO(terrno); goto END; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 020bc0aaae..a63c15edfb 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -482,7 +482,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe } tDecoderClear(&decoder); - tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(qid:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); if (!isLeader) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5580b0cadc..380e5224e1 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -21,11 +21,11 @@ extern "C" { #endif #include "catalog.h" +#include "os.h" #include "query.h" #include "tcommon.h" -#include "ttimer.h" #include "tglobal.h" -#include "os.h" +#include "ttimer.h" #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 @@ -40,7 +40,7 @@ extern "C" { #define CTG_DEFAULT_FETCH_NUM 8 #define CTG_MAX_COMMAND_LEN 512 #define CTG_DEFAULT_CACHE_MON_MSEC 5000 -#define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 +#define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 #define CTG_RENT_SLOT_SECOND 1.5 @@ -292,7 +292,7 @@ typedef struct SCtgTSMAFetch { typedef struct SCtgTbTSMACtx { int32_t fetchNum; - SArray* pNames; // SArray + SArray* pNames; // SArray SArray* pResList; SArray* pFetches; } SCtgTbTSMACtx; @@ -301,10 +301,10 @@ typedef STableIndexRsp STableIndex; typedef STableTSMAInfo STSMACache; typedef struct SCtgTbCache { - SRWLatch metaLock; - SRWLatch indexLock; - STableMeta* pMeta; - STableIndex* pIndex; + SRWLatch metaLock; + SRWLatch indexLock; + STableMeta* pMeta; + STableIndex* pIndex; } SCtgTbCache; typedef struct SCtgVgCache { @@ -318,13 +318,13 @@ typedef struct SCtgCfgCache { } SCtgCfgCache; typedef struct SCtgViewCache { - SRWLatch viewLock; - SViewMeta* pMeta; + SRWLatch viewLock; + SViewMeta* pMeta; } SCtgViewCache; typedef struct SCtgTSMACache { SRWLatch tsmaLock; - SArray* pTsmas; // SArray + SArray* pTsmas; // SArray } SCtgTSMACache; typedef struct SCtgDBCache { @@ -333,9 +333,9 @@ typedef struct SCtgDBCache { int8_t deleted; SCtgVgCache vgCache; SCtgCfgCache cfgCache; - SHashObj* viewCache; // key:viewname, value:SCtgViewCache - SHashObj* tbCache; // key:tbname, value:SCtgTbCache - SHashObj* stbCache; // key:suid, value:char* + SHashObj* viewCache; // key:viewname, value:SCtgViewCache + SHashObj* tbCache; // key:tbname, value:SCtgTbCache + SHashObj* stbCache; // key:suid, value:char* SHashObj* tsmaCache; // key:tbname, value: SCtgTSMACache int32_t tsmaVersion; uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; @@ -365,7 +365,7 @@ typedef struct SCtgUserAuth { } SCtgUserAuth; typedef struct SCatalog { - int64_t clusterId; + int64_t clusterId; bool stopUpdate; SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth @@ -420,7 +420,7 @@ typedef struct SCtgJob { int32_t svrVerNum; int32_t viewNum; int32_t tbTsmaNum; - int32_t tsmaNum; // currently, only 1 is possible + int32_t tsmaNum; // currently, only 1 is possible } SCtgJob; typedef struct SCtgMsgCtx { @@ -587,8 +587,8 @@ typedef struct SCtgUpdateEpsetMsg { } SCtgUpdateEpsetMsg; typedef struct SCtgUpdateViewMetaMsg { - SCatalog* pCtg; - SViewMetaRsp* pRsp; + SCatalog* pCtg; + SViewMetaRsp* pRsp; } SCtgUpdateViewMetaMsg; typedef struct SCtgDropViewMetaMsg { @@ -618,7 +618,6 @@ typedef struct SCtgDropTbTSMAMsg { bool dropAllForTb; } SCtgDropTbTSMAMsg; - typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -647,7 +646,7 @@ typedef struct SCatalogMgmt { int32_t jobPool; SRWLatch lock; SCtgQueue queue; - void *timer; + void* timer; tmr_h cacheTimer; TdThread updateThread; SHashObj* pCluster; // key: clusterId, value: SCatalog* @@ -665,8 +664,8 @@ typedef struct SCtgOperation { } SCtgOperation; typedef struct SCtgCacheItemInfo { - char* name; - int32_t flag; + char* name; + int32_t flag; } SCtgCacheItemInfo; #define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) @@ -800,8 +799,8 @@ typedef struct SCtgCacheItemInfo { ((CTG_TASK_GET_TB_META_BATCH == (_taskType)) || (CTG_TASK_GET_TB_HASH_BATCH == (_taskType)) || \ (CTG_TASK_GET_VIEW == (_taskType)) || (CTG_TASK_GET_TB_TSMA == (_taskType))) -#define CTG_GET_TASK_MSGCTX(_task, _id) \ - (CTG_IS_BATCH_TASK((_task)->type) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx) +#define CTG_GET_TASK_MSGCTX(_task, _id) \ + (CTG_IS_BATCH_TASK((_task)->type) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx) #define CTG_META_SIZE(pMeta) \ (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) @@ -810,9 +809,8 @@ typedef struct SCtgCacheItemInfo { #define CTG_DB_NOT_EXIST(code) \ (code == TSDB_CODE_MND_DB_NOT_EXIST || code == TSDB_CODE_MND_DB_IN_CREATING || code == TSDB_CODE_MND_DB_IN_DROPPING) -#define CTG_CACHE_OVERFLOW(_csize, _maxsize) ((_maxsize >= 0) ? ((_csize) >= (_maxsize) * 1048576L * 0.9) : false) -#define CTG_CACHE_LOW(_csize, _maxsize) ((_maxsize >= 0) ? ((_csize) <= (_maxsize) * 1048576L * 0.75) : true) - +#define CTG_CACHE_OVERFLOW(_csize, _maxsize) ((_maxsize >= 0) ? ((_csize) >= (_maxsize)*1048576L * 0.9) : false) +#define CTG_CACHE_LOW(_csize, _maxsize) ((_maxsize >= 0) ? ((_csize) <= (_maxsize)*1048576L * 0.75) : true) #define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__) @@ -821,12 +819,12 @@ typedef struct SCtgCacheItemInfo { #define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__) -#define ctgTaskFatal(param, ...) qFatal("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) -#define ctgTaskError(param, ...) qError("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) -#define ctgTaskWarn(param, ...) qWarn("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) -#define ctgTaskInfo(param, ...) qInfo("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) -#define ctgTaskDebug(param, ...) qDebug("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) -#define ctgTaskTrace(param, ...) qTrace("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskFatal(param, ...) qFatal("qid:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskError(param, ...) qError("qid:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskWarn(param, ...) qWarn("qid:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskInfo(param, ...) qInfo("qid:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskDebug(param, ...) qDebug("qid:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskTrace(param, ...) qTrace("qid:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) #define CTG_LOCK_DEBUG(...) \ do { \ @@ -849,62 +847,62 @@ typedef struct SCtgCacheItemInfo { #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define CTG_LOCK(type, _lock) \ - do { \ - if (CTG_READ == (type)) { \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value before read lock"); \ - break; \ - } \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) <= 0) { \ - qError("invalid lock value after read lock"); \ - break; \ - } \ - } else { \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value before write lock"); \ - break; \ - } \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ - qError("invalid lock value after write lock"); \ - break; \ - } \ - } \ +#define CTG_LOCK(type, _lock) \ + do { \ + if (CTG_READ == (type)) { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before read lock"); \ + break; \ + } \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value after read lock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before write lock"); \ + break; \ + } \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value after write lock"); \ + break; \ + } \ + } \ } while (0) -#define CTG_UNLOCK(type, _lock) \ - do { \ - if (CTG_READ == (type)) { \ - if (atomic_load_32((_lock)) <= 0) { \ - qError("invalid lock value before read unlock"); \ - break; \ - } \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value after read unlock"); \ - break; \ - } \ - } else { \ - if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ - qError("invalid lock value before write unlock"); \ - break; \ - } \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value after write unlock"); \ - break; \ - } \ - } \ +#define CTG_UNLOCK(type, _lock) \ + do { \ + if (CTG_READ == (type)) { \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value before read unlock"); \ + break; \ + } \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after read unlock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value before write unlock"); \ + break; \ + } \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after write unlock"); \ + break; \ + } \ + } \ } while (0) #define CTG_ERR_RET(c) \ @@ -945,7 +943,7 @@ typedef struct SCtgCacheItemInfo { CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); \ CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \ return; \ - } while (0) + } while (0) #define CTG_API_ENTER() \ do { \ @@ -963,7 +961,7 @@ typedef struct SCtgCacheItemInfo { if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \ CTG_API_NLEAVE(); \ } \ - } while (0) + } while (0) #define CTG_API_JENTER() \ do { \ @@ -1001,7 +999,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMe int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst); int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action); -int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation); +int32_t ctgOpUpdateDbCfg(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropDbCache(SCtgCacheOperation* action); int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); @@ -1024,29 +1022,30 @@ int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, bool syncReq); int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* tbName, bool syncReq); int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq); -int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDbCfgInfo *cfgInfo, bool syncOp); +int32_t ctgUpdateDbCfgEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, SDbCfgInfo* cfgInfo, bool syncOp); int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput* output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp* pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char* dbFName, int32_t vgId, SEpSet* pEpSet); int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex** pIndex, bool syncOp); -int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp); +int32_t ctgDropViewMetaEnqueue(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* viewName, + uint64_t viewId, bool syncOp); int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool clearMeta, bool freeCtg, bool stopQueue, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt* mgmt, uint32_t rentSec, int8_t type, int32_t size); int32_t ctgMetaRentAdd(SCtgRentMgmt* mgmt, void* meta, int64_t id, int32_t size); -int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, +int32_t ctgMetaRentUpdate(SCtgRentMgmt* mgmt, void* meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare); int32_t ctgMetaRentGet(SCtgRentMgmt* mgmt, void** res, uint32_t* num, int32_t size); -int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare); -void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache); -void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); +int32_t ctgMetaRentRemove(SCtgRentMgmt* mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare); +void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache* dbCache); +void ctgRemoveViewRent(SCatalog* pCtg, SCtgDBCache* dbCache); void ctgRemoveTSMARent(SCatalog* pCtg, SCtgDBCache* dbCache); -int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, - SCtgTbCache *pCache); -int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache *pCache); +int32_t ctgUpdateRentStbVersion(SCatalog* pCtg, char* dbFName, char* tbName, uint64_t dbId, uint64_t suid, + SCtgTbCache* pCache); +int32_t ctgUpdateRentViewVersion(SCatalog* pCtg, char* dbFName, char* viewName, uint64_t dbId, uint64_t viewId, + SCtgViewCache* pCache); int32_t ctgUpdateRentTSMAVersion(SCatalog* pCtg, char* dbFName, const STSMACache* pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); -int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); +int32_t ctgUpdateViewMetaToCache(SCatalog* pCtg, SViewMetaRsp* pRsp, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); @@ -1055,11 +1054,11 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp); int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation); -int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation); +int32_t ctgOpUpdateViewMeta(SCtgCacheOperation* operation); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); -int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, - int32_t *fetchIdx, int32_t baseResIdx, SArray *pList); +int32_t ctgGetViewsFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgViewsCtx* ctx, int32_t dbIdx, + int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildUseDBInput* input, SUseDbOutput* out, SCtgTaskReq* tReq); @@ -1106,9 +1105,10 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList); void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); -int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); -int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, - char* dbFName, SArray* pNames, bool update); +int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* dbInfo, const SName* pTableName, + SVgroupInfo* pVgroup); +int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, + SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId); void ctgResetTbMetaTask(SCtgTask* pTask); @@ -1143,8 +1143,8 @@ int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName); int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); -int32_t ctgChkSetAuthRes(SCatalog *pCtg, SCtgAuthReq *req, SCtgAuthRsp* res); -int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* pDbCfg); +int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); +int32_t ctgReadDBCfgFromCache(SCatalog* pCtg, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t ctgAcquireVgMetaFromCache(SCatalog* pCtg, const char* dbFName, const char* tbName, SCtgDBCache** pDb, SCtgTbCache** pTb); @@ -1157,20 +1157,20 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx); int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta); void ctgDestroySMetaData(SMetaData* pData); -void ctgGetGlobalCacheSize(uint64_t *pSize); -uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex); -uint64_t ctgGetViewMetaCacheSize(SViewMeta *pMeta); -uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta); -uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo *pVg); -uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth); -uint64_t ctgGetClusterCacheSize(SCatalog *pCtg); -void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardNum, bool *roundDone); -void ctgClearAllHandleMeta(int64_t *clearedSize, int64_t *clearedNum, bool *roundDone); -void ctgProcessTimerEvent(void *param, void *tmrId); +void ctgGetGlobalCacheSize(uint64_t* pSize); +uint64_t ctgGetTbIndexCacheSize(STableIndex* pIndex); +uint64_t ctgGetViewMetaCacheSize(SViewMeta* pMeta); +uint64_t ctgGetTbMetaCacheSize(STableMeta* pMeta); +uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo* pVg); +uint64_t ctgGetUserCacheSize(SGetUserAuthRsp* pAuth); +uint64_t ctgGetClusterCacheSize(SCatalog* pCtg); +void ctgClearHandleMeta(SCatalog* pCtg, int64_t* pClearedSize, int64_t* pCleardNum, bool* roundDone); +void ctgClearAllHandleMeta(int64_t* clearedSize, int64_t* clearedNum, bool* roundDone); +void ctgProcessTimerEvent(void* param, void* tmrId); int32_t ctgBuildUseDbOutput(SUseDbOutput** ppOut, SDBVgInfo* vgInfo); int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName); +int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char** stbName); int32_t ctgGetTbTagCb(SCtgTask* pTask); int32_t ctgGetUserCb(SCtgTask* pTask); @@ -1194,7 +1194,8 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag, CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName); int32_t ctgOpUpdateDbTsmaVersion(SCtgCacheOperation* pOper); -int32_t ctgUpdateDbTsmaVersionEnqueue(SCatalog* pCtg, int32_t tsmaVersion, const char* dbFName, int64_t dbId, bool syncOper); +int32_t ctgUpdateDbTsmaVersionEnqueue(SCatalog* pCtg, int32_t tsmaVersion, const char* dbFName, int64_t dbId, + bool syncOper); void ctgFreeTask(SCtgTask* pTask, bool freeRes); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 4954ff6675..7e15fd5f4f 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -72,7 +72,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -94,7 +94,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, + qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -105,7 +105,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; @@ -133,7 +133,7 @@ int32_t ctgInitGetDbVgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -161,7 +161,7 @@ int32_t ctgInitGetDbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -189,7 +189,7 @@ int32_t ctgInitGetDbInfoTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -223,7 +223,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -245,7 +245,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, + qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -256,7 +256,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; @@ -275,7 +275,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -293,7 +293,7 @@ int32_t ctgInitGetDnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -320,7 +320,7 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -348,7 +348,7 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -376,7 +376,7 @@ int32_t ctgInitGetUserTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); return TSDB_CODE_SUCCESS; @@ -394,7 +394,7 @@ int32_t ctgInitGetSvrVerTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -426,7 +426,7 @@ int32_t ctgInitGetTbIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -459,7 +459,7 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -492,7 +492,7 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -514,7 +514,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, + qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -525,7 +525,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, + qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum); return TSDB_CODE_SUCCESS; @@ -546,7 +546,7 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, + qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -574,7 +574,7 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, + qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -828,7 +828,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { - ctgError("failed to calloc, size:%d, QID:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId); + ctgError("failed to calloc, size:%d, qid:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1010,7 +1010,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const (void)taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); double el = (taosGetTimestampUs() - st) / 1000.0; - qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", + qDebug("qid:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", pJob->queryId, pJob->refId, taskNum, pReq->forceUpdate, el); return TSDB_CODE_SUCCESS; @@ -1400,11 +1400,11 @@ _return: int32_t ctgCallUserCb(void* param) { SCtgJob* pJob = (SCtgJob*)param; - qDebug("QID:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("qid:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode); - qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); + qDebug("qid:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); @@ -1415,7 +1415,7 @@ void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { if (!NEED_CLIENT_REFRESH_VG_ERROR(errCode) || errCode == TSDB_CODE_SUCCESS) return; atomic_store_32(&pJob->jobResCode, errCode); - qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); + qDebug("qid:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); return; } @@ -1427,7 +1427,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { return TSDB_CODE_SUCCESS; } - qDebug("QID:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); + qDebug("qid:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); pTask->code = rspCode; pTask->status = CTG_TASK_DONE; @@ -1436,7 +1436,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { - qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, + qDebug("qid:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); ctgUpdateJobErrCode(pJob, rspCode); @@ -4029,7 +4029,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); + qDebug("qid:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask = taosArrayGet(pJob->pTasks, i); @@ -4042,7 +4042,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { } if (taskNum <= 0) { - qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("qid:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); CTG_ERR_RET(taosAsyncExec(ctgCallUserCb, pJob, NULL)); #if CTG_BATCH_FETCH diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 77501b990f..fb444ed9d9 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -47,7 +47,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = 0; } - ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, + ctgDebug("qid:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); SHashObj* pBatchs = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -114,7 +114,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu pMsgCtx->pBatchs = pBatchs; - ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, + ctgDebug("qid:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); (void)(*gCtgAsyncFps[pTask->type].handleRspFp)( @@ -433,7 +433,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, + qDebug("qid:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); #if CTG_BATCH_FETCH @@ -538,7 +538,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob, CTG_ERR_JRET(code); } - ctgDebug("ctg req msg sent, QID:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType)); + ctgDebug("ctg req msg sent, qid:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: @@ -777,7 +777,7 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs) { SCtgBatch* pBatch = (SCtgBatch*)p; int32_t msgSize = 0; - ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); + ctgDebug("qid:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg, &msgSize)); code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->pMsgIdxs, diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 5d8107ca44..daa2199421 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -245,36 +245,36 @@ void ctgFreeStbMetaCache(SCtgDBCache* dbCache) { void ctgFreeTbCacheImpl(SCtgTbCache* pCache, bool lock) { if (pCache->pMeta) { - if (lock) { + if (lock) { CTG_LOCK(CTG_WRITE, &pCache->metaLock); } taosMemoryFreeClear(pCache->pMeta); - if (lock) { + if (lock) { CTG_UNLOCK(CTG_WRITE, &pCache->metaLock); } } if (pCache->pIndex) { - if (lock) { + if (lock) { CTG_LOCK(CTG_WRITE, &pCache->indexLock); } taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCache->pIndex); - if (lock) { + if (lock) { CTG_UNLOCK(CTG_WRITE, &pCache->indexLock); } } } void ctgFreeViewCacheImpl(SCtgViewCache* pCache, bool lock) { - if (lock) { + if (lock) { CTG_LOCK(CTG_WRITE, &pCache->viewLock); } if (pCache->pMeta) { ctgFreeSViewMeta(pCache->pMeta); taosMemoryFreeClear(pCache->pMeta); } - if (lock) { + if (lock) { CTG_UNLOCK(CTG_WRITE, &pCache->viewLock); } } @@ -398,10 +398,10 @@ void ctgFreeHandleImpl(SCatalog* pCtg) { } int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user) { - CTG_LOCK(CTG_WRITE, &pUser->lock); + CTG_LOCK(CTG_WRITE, &pUser->lock); ctgFreeSCtgUserAuth(pUser); - CTG_UNLOCK(CTG_WRITE, &pUser->lock); - + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + if (taosHashRemove(pCtg->userCache, user, strlen(user)) == 0) { return 0; // user found and removed } @@ -431,12 +431,12 @@ void ctgFreeHandle(SCatalog* pCtg) { ctgInfo("handle freed, clusterId:0x%" PRIx64, clusterId); } -void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardNum, bool *roundDone) { +void ctgClearHandleMeta(SCatalog* pCtg, int64_t* pClearedSize, int64_t* pCleardNum, bool* roundDone) { int64_t cacheSize = 0; - void* pIter = taosHashIterate(pCtg->dbCache, NULL); + void* pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache* dbCache = pIter; - + SCtgTbCache* pCache = taosHashIterate(dbCache->tbCache, NULL); while (NULL != pCache) { size_t len = 0; @@ -446,10 +446,11 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardN pCache = taosHashIterate(dbCache->tbCache, pCache); continue; } - + (void)taosHashRemove(dbCache->tbCache, key, len); - - cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); + + cacheSize = + len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); *pClearedSize += cacheSize; (*pCleardNum)++; @@ -457,14 +458,14 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardN if (pCache->pMeta) { CTG_META_NUM_DEC(pCache->pMeta->tableType); } - + ctgFreeTbCacheImpl(pCache, true); - + if (*pCleardNum >= CTG_CLEAR_CACHE_ROUND_TB_NUM) { taosHashCancelIterate(dbCache->tbCache, pCache); goto _return; } - + pCache = taosHashIterate(dbCache->tbCache, pCache); } @@ -478,12 +479,12 @@ _return: } } -void ctgClearAllHandleMeta(int64_t *clearedSize, int64_t *clearedNum, bool *roundDone) { - SCatalog *pCtg = NULL; +void ctgClearAllHandleMeta(int64_t* clearedSize, int64_t* clearedNum, bool* roundDone) { + SCatalog* pCtg = NULL; - void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { - pCtg = *(SCatalog **)pIter; + pCtg = *(SCatalog**)pIter; if (pCtg) { ctgClearHandleMeta(pCtg, clearedSize, clearedNum, roundDone); @@ -503,7 +504,7 @@ void ctgClearHandle(SCatalog* pCtg) { } int64_t clusterId = pCtg->clusterId; - + ctgFreeMetaRent(&pCtg->dbRent); ctgFreeMetaRent(&pCtg->stbRent); ctgFreeMetaRent(&pCtg->viewRent); @@ -725,7 +726,7 @@ void ctgFreeTbTSMARes(void* res) { SMetaRes* pRes = res; if (pRes->pRes) { - STableTSMAInfoRsp * pTsmaRsp = pRes->pRes; + STableTSMAInfoRsp* pTsmaRsp = pRes->pRes; tFreeTableTSMAInfoRsp(pTsmaRsp); taosMemoryFree(pTsmaRsp); pRes->pRes = NULL; @@ -1059,7 +1060,7 @@ void ctgFreeJob(void* job) { taosMemoryFree(job); - qDebug("QID:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); + qDebug("qid:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); } int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) { @@ -1175,7 +1176,8 @@ int32_t ctgHashValueComp(void const* lp, void const* rp) { return 0; } -int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { +int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* dbInfo, const SName* pTableName, + SVgroupInfo* pVgroup) { int32_t code = 0; CTG_ERR_RET(ctgMakeVgArray(dbInfo)); @@ -1188,7 +1190,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d if (pMgmtEps) { TAOS_MEMCPY(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet)); } - + return TSDB_CODE_SUCCESS; } @@ -1235,16 +1237,16 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d CTG_RET(code); } -int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, - char* dbFName, SArray* pNames, bool update) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SMetaRes res = {0}; +int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, + SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SMetaRes res = {0}; SVgroupInfo* vgInfo = NULL; CTG_ERR_RET(ctgMakeVgArray(dbInfo)); - int32_t tbNum = taosArrayGetSize(pNames); + int32_t tbNum = taosArrayGetSize(pNames); char* pSep = strchr(dbFName, '.'); if (pSep && IS_SYS_DBNAME(pSep + 1)) { @@ -1253,7 +1255,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR if (pMgmgEpSet) { TAOS_MEMCPY(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet)); } - + for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { @@ -1271,12 +1273,13 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); if (NULL == pFetch) { - ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, + (int32_t)taosArrayGetSize(pCtx->pResList)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + pRes->pRes = vgInfo; } else { res.pRes = vgInfo; @@ -1285,7 +1288,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR } } } - + return TSDB_CODE_SUCCESS; } @@ -1319,9 +1322,10 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); if (NULL == pRes) { - ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, + (int32_t)taosArrayGetSize(pCtx->pResList)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1380,9 +1384,10 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); if (NULL == pRes) { - ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, + (int32_t)taosArrayGetSize(pCtx->pResList)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1531,7 +1536,7 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { taosHashCancelIterate(dbInfo->vgHash, pIter); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + pIter = taosHashIterate(dbInfo->vgHash, pIter); } @@ -1581,7 +1586,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { if (NULL == (*dst)->vgArray) { taosHashCleanup((*dst)->vgHash); taosMemoryFreeClear(*dst); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -1603,7 +1608,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } - + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize + schemaExtSize); qDebug("tbMeta cloned, size:%d, p:%p", metaSize, (*pOutput)->tbMeta); if (NULL == (*pOutput)->tbMeta) { @@ -1614,7 +1619,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) TAOS_MEMCPY((*pOutput)->tbMeta, output->tbMeta, metaSize); if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { - (*pOutput)->tbMeta->schemaExt = (SSchemaExt *)((char *)(*pOutput)->tbMeta + metaSize); + (*pOutput)->tbMeta->schemaExt = (SSchemaExt*)((char*)(*pOutput)->tbMeta + metaSize); TAOS_MEMCPY((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); } else { (*pOutput)->tbMeta->schemaExt = NULL; @@ -1683,7 +1688,7 @@ int32_t ctgGetTablesReqNum(SArray* pList) { qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pList)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + total += taosArrayGetSize(pReq->pTables); } @@ -1718,7 +1723,7 @@ int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName) { qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); return TSDB_CODE_CTG_INTERNAL_ERROR; } - + *ppName = (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); if (NULL == *ppName) { qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); @@ -1728,7 +1733,7 @@ int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName) { return TSDB_CODE_SUCCESS; } -static int32_t ctgCloneDbVgroup(void* pSrc, void** ppDst) { +static int32_t ctgCloneDbVgroup(void* pSrc, void** ppDst) { #if 0 if (NULL == pSrc) { *ppDst = NULL; @@ -1749,10 +1754,10 @@ int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) { if (NULL == pDst) { return terrno; } - + TAOS_MEMCPY(pDst, pSrc, sizeof(SDbCfgInfo)); - if (((SDbCfgInfo *)pSrc)->pRetensions) { - pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); + if (((SDbCfgInfo*)pSrc)->pRetensions) { + pDst->pRetensions = taosArrayDup(((SDbCfgInfo*)pSrc)->pRetensions, NULL); if (NULL == pDst->pRetensions) { taosMemoryFree(pDst); return terrno; @@ -1760,12 +1765,12 @@ int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) { } *ppDst = pDst; - + return TSDB_CODE_SUCCESS; } -static void ctgFreeDbCfgInfo(void* p) { - SDbCfgInfo* pDst = (SDbCfgInfo *)((SMetaRes*)p)->pRes; +static void ctgFreeDbCfgInfo(void* p) { + SDbCfgInfo* pDst = (SDbCfgInfo*)((SMetaRes*)p)->pRes; freeDbCfgInfo(pDst); } @@ -1829,9 +1834,9 @@ static int32_t ctgCloneVgroupInfo(void* pSrc, void** ppDst) { static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static int32_t ctgCloneTableIndexs(void* pSrc, void** ppDst) { +static int32_t ctgCloneTableIndexs(void* pSrc, void** ppDst) { #if 0 - return taosArrayDup((const SArray*)pSrc, NULL); + return taosArrayDup((const SArray*)pSrc, NULL); #else return TSDB_CODE_CTG_INTERNAL_ERROR; #endif @@ -1884,9 +1889,9 @@ static int32_t ctgCloneUserAuth(void* pSrc) { static void ctgFreeUserAuth(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static int32_t ctgCloneQnodeList(void* pSrc) { +static int32_t ctgCloneQnodeList(void* pSrc) { #if 0 - return taosArrayDup((const SArray*)pSrc, NULL); + return taosArrayDup((const SArray*)pSrc, NULL); #else return TSDB_CODE_CTG_INTERNAL_ERROR; #endif @@ -1905,9 +1910,9 @@ static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p) static void ctgFreeTableCfg(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static int32_t ctgCloneDnodeList(void* pSrc) { +static int32_t ctgCloneDnodeList(void* pSrc) { #if 0 - return taosArrayDup((const SArray*)pSrc, NULL); + return taosArrayDup((const SArray*)pSrc, NULL); #else return TSDB_CODE_CTG_INTERNAL_ERROR; #endif @@ -1935,14 +1940,14 @@ static int32_t ctgCloneViewMeta(void* pSrc) { #endif } -static void ctgFreeViewMeta(void* p) { - SViewMeta* pMeta = ((SMetaRes*)p)->pRes; +static void ctgFreeViewMeta(void* p) { + SViewMeta* pMeta = ((SMetaRes*)p)->pRes; if (NULL == pMeta) { return; } taosMemoryFree(pMeta->user); taosMemoryFree(pMeta->querySql); - taosMemoryFree(pMeta->pSchema); + taosMemoryFree(pMeta->pSchema); taosMemoryFree(pMeta); } @@ -2054,8 +2059,8 @@ int32_t ctgChkSetBasicAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res } if (req->tbNotExists) { - //pRes->pass[AUTH_RES_BASIC] = true; - //return TSDB_CODE_SUCCESS; + // pRes->pass[AUTH_RES_BASIC] = true; + // return TSDB_CODE_SUCCESS; pReq->tbName.type = TSDB_DB_NAME_T; } @@ -2152,7 +2157,7 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) switch (pReq->type) { case AUTH_TYPE_READ: { - char *value = taosHashGet(pInfo->readViews, viewFName, len); + char* value = taosHashGet(pInfo->readViews, viewFName, len); if (NULL != value) { pRes->pass[AUTH_RES_VIEW] = true; return TSDB_CODE_SUCCESS; @@ -2160,7 +2165,7 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) break; } case AUTH_TYPE_WRITE: { - char *value = taosHashGet(pInfo->writeViews, viewFName, len); + char* value = taosHashGet(pInfo->writeViews, viewFName, len); if (NULL != value) { pRes->pass[AUTH_RES_VIEW] = true; return TSDB_CODE_SUCCESS; @@ -2168,7 +2173,7 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) break; } case AUTH_TYPE_ALTER: { - char *value = taosHashGet(pInfo->alterViews, viewFName, len); + char* value = taosHashGet(pInfo->alterViews, viewFName, len); if (NULL != value) { pRes->pass[AUTH_RES_VIEW] = true; return TSDB_CODE_SUCCESS; @@ -2288,7 +2293,7 @@ void ctgDestroySMetaData(SMetaData* pData) { taosMemoryFreeClear(pData->pSvrVer); } -uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex) { +uint64_t ctgGetTbIndexCacheSize(STableIndex* pIndex) { if (NULL == pIndex) { return 0; } @@ -2296,7 +2301,7 @@ uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex) { return sizeof(*pIndex) + pIndex->indexSize; } -uint64_t ctgGetViewMetaCacheSize(SViewMeta *pMeta) { +uint64_t ctgGetViewMetaCacheSize(SViewMeta* pMeta) { if (NULL == pMeta) { return 0; } @@ -2304,8 +2309,7 @@ uint64_t ctgGetViewMetaCacheSize(SViewMeta *pMeta) { return sizeof(*pMeta) + strlen(pMeta->querySql) + 1 + strlen(pMeta->user) + 1 + pMeta->numOfCols * sizeof(SSchema); } - -FORCE_INLINE uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta) { +FORCE_INLINE uint64_t ctgGetTbMetaCacheSize(STableMeta* pMeta) { if (NULL == pMeta) { return 0; } @@ -2322,22 +2326,22 @@ FORCE_INLINE uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta) { return 0; } -uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo *pVg) { +uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo* pVg) { if (NULL == pVg) { return 0; } - return sizeof(*pVg) + taosHashGetSize(pVg->vgHash) * (sizeof(SVgroupInfo) + sizeof(int32_t)) - + taosArrayGetSize(pVg->vgArray) * sizeof(SVgroupInfo); + return sizeof(*pVg) + taosHashGetSize(pVg->vgHash) * (sizeof(SVgroupInfo) + sizeof(int32_t)) + + taosArrayGetSize(pVg->vgArray) * sizeof(SVgroupInfo); } -uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { +uint64_t ctgGetUserCacheSize(SGetUserAuthRsp* pAuth) { if (NULL == pAuth) { return 0; } uint64_t cacheSize = 0; - char* p = taosHashIterate(pAuth->createdDbs, NULL); + char* p = taosHashIterate(pAuth->createdDbs, NULL); while (p != NULL) { size_t len = 0; void* key = taosHashGetKey(p, &len); @@ -2353,7 +2357,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->readDbs, p); - } + } p = taosHashIterate(pAuth->writeDbs, NULL); while (p != NULL) { @@ -2362,7 +2366,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->writeDbs, p); - } + } p = taosHashIterate(pAuth->readTbs, NULL); while (p != NULL) { @@ -2371,7 +2375,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->readTbs, p); - } + } p = taosHashIterate(pAuth->writeTbs, NULL); while (p != NULL) { @@ -2380,7 +2384,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->writeTbs, p); - } + } p = taosHashIterate(pAuth->alterTbs, NULL); while (p != NULL) { @@ -2389,7 +2393,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->alterTbs, p); - } + } p = taosHashIterate(pAuth->readViews, NULL); while (p != NULL) { @@ -2398,7 +2402,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->readViews, p); - } + } p = taosHashIterate(pAuth->writeViews, NULL); while (p != NULL) { @@ -2407,7 +2411,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->writeViews, p); - } + } p = taosHashIterate(pAuth->alterViews, NULL); while (p != NULL) { @@ -2416,23 +2420,23 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { cacheSize += len + strlen(p) + 1; p = taosHashIterate(pAuth->alterViews, p); - } + } - int32_t *ref = taosHashIterate(pAuth->useDbs, NULL); + int32_t* ref = taosHashIterate(pAuth->useDbs, NULL); while (ref != NULL) { size_t len = 0; void* key = taosHashGetKey(ref, &len); cacheSize += len + sizeof(*ref); ref = taosHashIterate(pAuth->useDbs, ref); - } + } return cacheSize; } -uint64_t ctgGetClusterCacheSize(SCatalog *pCtg) { +uint64_t ctgGetClusterCacheSize(SCatalog* pCtg) { uint64_t cacheSize = sizeof(SCatalog); - + SCtgUserAuth* pAuth = taosHashIterate(pCtg->userCache, NULL); while (pAuth != NULL) { size_t len = 0; @@ -2523,7 +2527,7 @@ void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) { TAOS_MEMCPY(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache)); } -void ctgGetGlobalCacheSize(uint64_t *pSize) { +void ctgGetGlobalCacheSize(uint64_t* pSize) { *pSize = 0; SCatalog* pCtg = NULL; @@ -2531,8 +2535,8 @@ void ctgGetGlobalCacheSize(uint64_t *pSize) { while (pIter) { size_t len = 0; void* key = taosHashGetKey(pIter, &len); - *pSize += len + POINTER_BYTES; - + *pSize += len + POINTER_BYTES; + pCtg = *(SCatalog**)pIter; if (pCtg) { *pSize += ctgGetClusterCacheSize(pCtg); @@ -2544,18 +2548,18 @@ void ctgGetGlobalCacheSize(uint64_t *pSize) { int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx) { SCatalog* pCtg = pTask->pJob->pCtg; - int32_t dbNum = taosArrayGetSize(pCtx->pNames); + int32_t dbNum = taosArrayGetSize(pCtx->pNames); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); if (NULL == pReq) { qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + int32_t viewNum = taosArrayGetSize(pReq->pTables); - + ctgDebug("start to check views in db %s, viewNum %d", pReq->dbFName, viewNum); - + for (int32_t m = 0; m < viewNum; ++m) { if (NULL == taosArrayPush(pCtx->pResList, &(SMetaData){0})) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -2584,13 +2588,12 @@ int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) { if (pViewMeta->pSchema == NULL) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + TAOS_MEMCPY(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); return TSDB_CODE_SUCCESS; } - int32_t ctgBuildUseDbOutput(SUseDbOutput** ppOut, SDBVgInfo* vgInfo) { *ppOut = taosMemoryCalloc(1, sizeof(SUseDbOutput)); if (NULL == *ppOut) { @@ -2620,7 +2623,7 @@ uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) { if (pTsmaInfo->pUsedCols) { size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size; } - + return size; } @@ -2637,7 +2640,7 @@ bool hasOutOfDateTSMACache(SArray* pTsmas) { return true; } } - + return false; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b2cbef8919..14a1ae9b59 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -492,7 +492,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI taosWLockLatch(&pTaskInfo->lock); for (int32_t i = 0; i < numOfQualifiedTables; ++i) { - uint64_t* uid = taosArrayGet(qa, i); + uint64_t* uid = taosArrayGet(qa, i); if (!uid) { taosMemoryFree(keyBuf); taosArrayDestroy(qa); @@ -600,7 +600,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; (void)taosThreadOnce(&initPoolOnce, initRefPool); - qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId); + qDebug("start to create task, TID:0x%" PRIx64 " qid:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId); int32_t code = createExecTaskInfo(pSubplan, pTask, readHandle, taskId, vgId, sql, model); if (code != TSDB_CODE_SUCCESS || NULL == *pTask) { @@ -629,7 +629,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, code = dsCreateDataSinker(pSinkManager, pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); } - qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); + qDebug("subplan task create completed, TID:0x%" PRIx64 " qid:0x%" PRIx64, taskId, pSubplan->id.queryId); _error: // if failed to add ref for all tables in this query, abort current query @@ -887,7 +887,7 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); continue; } - SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pStop->refId); + SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pStop->refId); if (pExchangeInfo) { (void)tsem_post(&pExchangeInfo->ready); (void)taosReleaseRef(exchangeObjRefPool, pStop->refId); @@ -1383,7 +1383,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; - SOperatorInfo* p = NULL; + SOperatorInfo* p = NULL; code = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id, &p); if (code != 0) { @@ -1399,7 +1399,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SMetaTableInfo mtInfo = {0}; code = pTaskInfo->storageAPI.snapshotFn.getMetaTableInfoFromSnapshot(sContext, &mtInfo); - if (code != 0){ + if (code != 0) { return code; } pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); @@ -1439,7 +1439,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSchemaWrapper(mtInfo.schema); return code; } - int32_t size = tableListGetSize(pTableListInfo); + int32_t size = tableListGetSize(pTableListInfo); code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**)&pInfo->dataReader, NULL, NULL); @@ -1524,7 +1524,7 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { for (int32_t i = 0; i < numOfTables; ++i) { STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); QUERY_CHECK_NULL(pKeyInfo, code, lino, _end, terrno); - void* tmp = taosArrayPush(pUidList, &pKeyInfo->uid); + void* tmp = taosArrayPush(pUidList, &pKeyInfo->uid); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 881b4f9316..c39ccd6ba5 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -84,7 +84,7 @@ bool isTaskKilled(void* pTaskInfo) { return (0 != ((SExecTaskInfo*)pTaskInfo)->c void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; - (void) stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI); + (void)stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI); } void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { @@ -128,14 +128,15 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand void cleanupQueriedTableScanInfo(void* p) { SSchemaInfo* pSchemaInfo = p; - + taosMemoryFreeClear(pSchemaInfo->dbname); taosMemoryFreeClear(pSchemaInfo->tablename); tDeleteSchemaWrapper(pSchemaInfo->sw); tDeleteSchemaWrapper(pSchemaInfo->qsw); } -int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo) { +int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, + SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; if (pHandle == NULL) { return TSDB_CODE_INVALID_PARA; @@ -286,7 +287,7 @@ void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst) { memcpy(p, "TID:0x", offset); offset += tintToHex(taskId, &p[offset]); - memcpy(&p[offset], " QID:0x", 7); + memcpy(&p[offset], " qid:0x", 7); offset += 7; offset += tintToHex(queryId, &p[offset]); diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 1b53819c7b..43159bce20 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -20,7 +20,7 @@ #include "tglobal.h" static int32_t debugPrintNode(SNode* pNode) { - char* pStr = NULL; + char* pStr = NULL; int32_t code = nodesNodeToString(pNode, false, &pStr, NULL); if (TSDB_CODE_SUCCESS == code) { (void)printf("%s\n", pStr); @@ -37,7 +37,7 @@ static int32_t dumpQueryPlan(SQueryPlan* pPlan) { char* pStr = NULL; code = nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); if (TSDB_CODE_SUCCESS == code) { - planDebugL("QID:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); + planDebugL("qid:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); taosMemoryFree(pStr); } return code; @@ -117,14 +117,13 @@ static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDown return code; } -int32_t qContinuePlanPostQuery(void *pPostPlan) { - //TODO +int32_t qContinuePlanPostQuery(void* pPostPlan) { + // TODO return TSDB_CODE_SUCCESS; } - int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { - planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); + planDebug("qid:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } @@ -144,7 +143,7 @@ static void clearSubplanExecutionNode(SPhysiNode* pNode) { } void qClearSubplanExecutionNode(SSubplan* pSubplan) { - planDebug("QID:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); + planDebug("qid:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); clearSubplanExecutionNode(pSubplan->pNode); } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 68355b628e..7a902bdd66 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -142,10 +142,10 @@ typedef struct SQWTaskCtx { int8_t events[QW_EVENT_MAX]; - SArray *explainRes; - void *taskHandle; - void *sinkHandle; - SArray *tbInfo; // STbVerInfo + SArray *explainRes; + void *taskHandle; + void *sinkHandle; + SArray *tbInfo; // STbVerInfo } SQWTaskCtx; typedef struct SQWSchStatus { @@ -201,7 +201,7 @@ typedef struct SQWorker { SQWStat stat; int32_t *destroyed; - int8_t nodeStopped; + int8_t nodeStopped; } SQWorker; typedef struct SQWorkerMgmt { @@ -212,7 +212,8 @@ typedef struct SQWorkerMgmt { int32_t paramIdx; } SQWorkerMgmt; -#define QW_CTX_NOT_EXISTS_ERR_CODE(mgmt) (atomic_load_8(&(mgmt)->nodeStopped) ? TSDB_CODE_VND_STOPPED : TSDB_CODE_QRY_TASK_CTX_NOT_EXIST) +#define QW_CTX_NOT_EXISTS_ERR_CODE(mgmt) \ + (atomic_load_8(&(mgmt)->nodeStopped) ? TSDB_CODE_VND_STOPPED : TSDB_CODE_QRY_TASK_CTX_NOT_EXIST) #define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId #define QW_IDS() sId, qId, tId, rId, eId @@ -229,31 +230,31 @@ typedef struct SQWorkerMgmt { #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) -#define QW_SET_PHASE(ctx, _value) \ - do { \ - switch (_value) { \ - case QW_PHASE_PRE_FETCH: \ - ctx->inFetch = 1; \ - break; \ - case QW_PHASE_POST_FETCH: \ - ctx->inFetch = 0; \ - break; \ - case QW_PHASE_PRE_QUERY: \ - case QW_PHASE_POST_QUERY: \ - case QW_PHASE_PRE_CQUERY: \ - case QW_PHASE_POST_CQUERY: \ - atomic_store_8(&(ctx)->phase, _value); \ - break; \ - default: \ - break; \ - } \ +#define QW_SET_PHASE(ctx, _value) \ + do { \ + switch (_value) { \ + case QW_PHASE_PRE_FETCH: \ + ctx->inFetch = 1; \ + break; \ + case QW_PHASE_POST_FETCH: \ + ctx->inFetch = 0; \ + break; \ + case QW_PHASE_PRE_QUERY: \ + case QW_PHASE_POST_QUERY: \ + case QW_PHASE_PRE_CQUERY: \ + case QW_PHASE_POST_CQUERY: \ + atomic_store_8(&(ctx)->phase, _value); \ + break; \ + default: \ + break; \ + } \ } while (0) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) #define QW_UPDATE_RSP_CODE(ctx, code) (void)atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) -#define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) -#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) +#define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) +#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) #define QW_QUERY_NOT_STARTED(ctx) (QW_GET_PHASE(ctx) == -1) #define QW_SET_QTID(id, qId, tId, eId) \ @@ -309,24 +310,24 @@ typedef struct SQWorkerMgmt { #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) #define QW_TASK_DLOGL(param, ...) \ - qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) + qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) -#define QW_TASK_ELOG_E(param) qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) -#define QW_TASK_WLOG_E(param) qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) -#define QW_TASK_DLOG_E(param) qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) +#define QW_TASK_ELOG_E(param) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) +#define QW_TASK_WLOG_E(param) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) +#define QW_TASK_DLOG_E(param) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) #define QW_SCH_TASK_ELOG(param, ...) \ - qError("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ + qError("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ __VA_ARGS__) #define QW_SCH_TASK_WLOG(param, ...) \ - qWarn("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ + qWarn("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ __VA_ARGS__) #define QW_SCH_TASK_DLOG(param, ...) \ - qDebug("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ + qDebug("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ __VA_ARGS__) #define QW_LOCK_DEBUG(...) \ @@ -338,62 +339,62 @@ typedef struct SQWorkerMgmt { #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define QW_LOCK(type, _lock) \ - do { \ - if (QW_READ == (type)) { \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value before read lock"); \ - break; \ - } \ - QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) <= 0) { \ - qError("invalid lock value after read lock"); \ - break; \ - } \ - } else { \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value before write lock"); \ - break; \ - } \ - QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ - qError("invalid lock value after write lock"); \ - break; \ - } \ - } \ +#define QW_LOCK(type, _lock) \ + do { \ + if (QW_READ == (type)) { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before read lock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value after read lock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before write lock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value after write lock"); \ + break; \ + } \ + } \ } while (0) -#define QW_UNLOCK(type, _lock) \ - do { \ - if (QW_READ == (type)) { \ - if (atomic_load_32((_lock)) <= 0) { \ - qError("invalid lock value before read unlock"); \ - break; \ - } \ - QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value after read unlock"); \ - break; \ - } \ - } else { \ - if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ - qError("invalid lock value before write unlock"); \ - break; \ - } \ - QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value after write unlock"); \ - break; \ - } \ - } \ +#define QW_UNLOCK(type, _lock) \ + do { \ + if (QW_READ == (type)) { \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value before read unlock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after read unlock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value before write unlock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after write unlock"); \ + break; \ + } \ + } \ } while (0) extern SQWorkerMgmt gQwMgmt; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 6a64ff67e4..c723360b36 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -410,22 +410,22 @@ extern SSchedulerMgmt schMgmt; #define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps) #define SCH_TASK_NUM_OF_EPS(_addr) ((_addr)->epSet.numOfEps) -#define SCH_LOG_TASK_START_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ +#define SCH_LOG_TASK_START_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ (void)taosArrayPush((_task)->profile.execTime, &us); \ - if (0 == (_task)->execId) { \ - (_task)->profile.startTs = us; \ - } \ + if (0 == (_task)->execId) { \ + (_task)->profile.startTs = us; \ + } \ } while (0) -#define SCH_LOG_TASK_WAIT_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ - int64_t* startus = (int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - if (NULL != startus) { \ - (_task)->profile.waitTime += us - *startus; \ - } \ +#define SCH_LOG_TASK_WAIT_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + int64_t *startus = (int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + if (NULL != startus) { \ + (_task)->profile.waitTime += us - *startus; \ + } \ } while (0) #define SCH_LOG_TASK_END_TS(_task) \ @@ -434,28 +434,28 @@ extern SSchedulerMgmt schMgmt; int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ if (NULL != startts) { \ - *startts = us - *startts; \ + *startts = us - *startts; \ } \ (_task)->profile.endTs = us; \ } while (0) -#define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_ELOG(param, ...) qError("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_DLOG(param, ...) qDebug("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ __VA_ARGS__) #define SCH_TASK_TLOG(param, ...) \ - qTrace("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + qTrace("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ __VA_ARGS__) #define SCH_TASK_DLOGL(param, ...) \ - qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ __VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ __VA_ARGS__) #define SCH_SET_ERRNO(_err) \ @@ -498,62 +498,62 @@ extern SSchedulerMgmt schMgmt; #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define SCH_LOCK(type, _lock) \ - do { \ - if (SCH_READ == (type)) { \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value before read lock"); \ - break; \ - } \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) <= 0) { \ - qError("invalid lock value after read lock"); \ - break; \ - } \ - } else { \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value before write lock"); \ - break; \ - } \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ - qError("invalid lock value after write lock"); \ - break; \ - } \ - } \ +#define SCH_LOCK(type, _lock) \ + do { \ + if (SCH_READ == (type)) { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before read lock"); \ + break; \ + } \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value after read lock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before write lock"); \ + break; \ + } \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value after write lock"); \ + break; \ + } \ + } \ } while (0) -#define SCH_UNLOCK(type, _lock) \ - do { \ - if (SCH_READ == (type)) { \ - if (atomic_load_32((_lock)) <= 0) { \ - qError("invalid lock value before read unlock"); \ - break; \ - } \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value after read unlock"); \ - break; \ - } \ - } else { \ - if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ - qError("invalid lock value before write unlock"); \ - break; \ - } \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - if (atomic_load_32((_lock)) < 0) { \ - qError("invalid lock value after write unlock"); \ - break; \ - } \ - } \ +#define SCH_UNLOCK(type, _lock) \ + do { \ + if (SCH_READ == (type)) { \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value before read unlock"); \ + break; \ + } \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after read unlock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value before write unlock"); \ + break; \ + } \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after write unlock"); \ + break; \ + } \ + } \ } while (0) #define SCH_RESET_JOB_LEVEL_IDX(_job) \ @@ -607,7 +607,7 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); -int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes); +int32_t schDumpEpSet(SEpSet *pEpSet, char **ppRes); char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 9c9cce7516..cf4a132eb5 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -87,7 +87,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { if (JOB_TASK_STATUS_FETCH == newStatus) { return code; } - + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -125,7 +125,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { newStatus != JOB_TASK_STATUS_FETCH) { SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } - + break; case JOB_TASK_STATUS_SUCC: case JOB_TASK_STATUS_FAIL: @@ -171,14 +171,14 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, pJob->levelNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); if (NULL == pTask) { SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum: %d", m, pLevel->level, pLevel->taskNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SSubplan *pPlan = pTask->plan; int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; @@ -197,12 +197,12 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < childNum; ++n) { - SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); + SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); if (NULL == child) { SCH_JOB_ELOG("fail to get the %dth child subplan, childNum: %d", n, childNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -236,12 +236,12 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < parentNum; ++n) { - SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); + SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); if (NULL == parent) { SCH_JOB_ELOG("fail to get the %dth parent subplan, parentNum: %d", n, parentNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -265,7 +265,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", (int32_t)taosArrayGetSize(pJob->levels)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + if (SCH_IS_QUERY_JOB(pJob)) { if (pLevel->taskNum > 1) { SCH_JOB_ELOG("invalid query plan, level:0, taskNum:%d", pLevel->taskNum); @@ -354,7 +354,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + pLevel->level = i; plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -427,7 +427,7 @@ _return: void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; - + SCH_LOCK(SCH_WRITE, &pJob->resLock); pRes->res = pJob->execRes.res; pRes->msgType = pJob->execRes.msgType; @@ -550,7 +550,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { schPostJobRes(pJob, 0); return TSDB_CODE_SCH_IGNORE_ERROR; } - + schUpdateJobErrCode(pJob, errCode); int32_t code = atomic_load_32(&pJob->errCode); @@ -568,8 +568,8 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); // ignore error - + (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); // ignore error + return TSDB_CODE_SCH_IGNORE_ERROR; } @@ -580,8 +580,8 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); // ignore error - + (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); // ignore error + return TSDB_CODE_SCH_IGNORE_ERROR; } @@ -627,14 +627,14 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + for (int32_t i = 0; i < pLevel->taskNum; ++i) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, i); if (NULL == pTask) { SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", i, pLevel->level, pLevel->taskNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + if (pTask->children && taosArrayGetSize(pTask->children) > 0) { continue; } @@ -662,13 +662,14 @@ int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) { } } - if (NULL == taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo))) { + if (NULL == taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), + taosArrayGetSize(rsp->tbVerInfo))) { SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_ERR_RET(terrno); } - + taosArrayDestroy(rsp->tbVerInfo); - + pJob->execRes.msgType = TDMT_SCH_QUERY; SCH_UNLOCK(SCH_WRITE, &pJob->resLock); @@ -697,7 +698,7 @@ int32_t schLaunchJob(SSchJob *pJob) { SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_ERR_RET(schLaunchLevelTasks(pJob, level)); } @@ -723,7 +724,7 @@ void schFreeJobImpl(void *job) { uint64_t queryId = pJob->queryId; int64_t refId = pJob->refId; - qDebug("QID:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("qid:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); schDropJobAllTasks(pJob); @@ -765,12 +766,12 @@ void schFreeJobImpl(void *job) { destroyQueryExecRes(&pJob->execRes); qDestroyQueryPlan(pJob->pDag); - (void)nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); // ignore error + (void)nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); // ignore error taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); - (void)tsem_destroy(&pJob->rspSem); // ignore error + (void)tsem_destroy(&pJob->rspSem); // ignore error taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -778,7 +779,7 @@ void schFreeJobImpl(void *job) { schCloseJobRef(); } - qDebug("QID:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("qid:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); } int32_t schJobFetchRows(SSchJob *pJob) { @@ -789,7 +790,7 @@ int32_t schJobFetchRows(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - (void)tsem_wait(&pJob->rspSem); // ignore error + (void)tsem_wait(&pJob->rspSem); // ignore error SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } } else { @@ -808,7 +809,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("qid:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -818,7 +819,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->sql) { pJob->sql = taosStrdup(pReq->sql); if (NULL == pJob->sql) { - qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); + qError("qid:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -826,7 +827,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->allocatorRefId > 0) { pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); if (pJob->allocatorRefId <= 0) { - qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); + qError("qid:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); SCH_ERR_JRET(terrno); } } @@ -837,11 +838,12 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->source = pReq->source; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { - qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); + qDebug("qid:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); } else { pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); if (NULL == pJob->nodeList) { - qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList)); + qError("qid:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, + (int32_t)taosArrayGetSize(pReq->pNodeList)); SCH_ERR_JRET(terrno); } } @@ -892,7 +894,7 @@ _return: } else if (pJob->refId < 0) { schFreeJobImpl(pJob); } else { - (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error + (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error } SCH_RET(code); @@ -900,13 +902,13 @@ _return: int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { int32_t code = 0; - qDebug("QID:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); + qDebug("qid:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); SCH_ERR_RET(schLaunchJob(pJob)); if (pReq->syncReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - (void)tsem_wait(&pJob->rspSem); // ignore error + (void)tsem_wait(&pJob->rspSem); // ignore error } SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); @@ -951,7 +953,7 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { } *inRetry = true; - + SCH_ERR_RET(schChkResetJobRetry(pJob, rspCode)); int32_t code = 0; @@ -991,10 +993,9 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { return TSDB_CODE_SUCCESS; } - int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - bool inRetry = false; + bool inRetry = false; taosMemoryFreeClear(pMsg->pData); taosMemoryFreeClear(pMsg->pEpSet); @@ -1074,7 +1075,7 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int } if (errCode) { - (void)schHandleJobFailure(pJob, errCode); // handle internal + (void)schHandleJobFailure(pJob, errCode); // handle internal } SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode)); @@ -1153,11 +1154,11 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { } if (errCode) { - (void)schHandleJobFailure(pJob, errCode); // ignore error + (void)schHandleJobFailure(pJob, errCode); // ignore error } if (pJob) { - (void)schReleaseJob(pJob->refId); // ignore error + (void)schReleaseJob(pJob->refId); // ignore error } } @@ -1170,7 +1171,7 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); + qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -1194,7 +1195,7 @@ _return: SCH_UNLOCK_TASK(pTask); } if (pJob) { - (void)schReleaseJob(rId); // ignore error + (void)schReleaseJob(rId); // ignore error } SCH_RET(code); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 9215254f9c..ad720e15f5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -499,7 +499,7 @@ _return: int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error @@ -512,7 +512,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); if (pMsg) { taosMemoryFree(pMsg->pData); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 6dd6aa9aae..729c5066ac 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -301,7 +301,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("fail to get task %d parent, parentNum: %d", i, parentNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { .type = QUERY_NODE_DOWNSTREAM_SOURCE, @@ -319,7 +319,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_UNLOCK(SCH_WRITE, &parent->planLock); SCH_ERR_RET(code); - + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { @@ -331,7 +331,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (taskDone == pTask->level->taskNum) { SCH_ERR_RET(schLaunchJobLowerLevel(pJob, pTask)); } - + return TSDB_CODE_SUCCESS; } @@ -422,10 +422,10 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { schDropTaskOnExecNode(pJob, pTask); if (pTask->delayTimer) { - (void)taosTmrStopA(&pTask->delayTimer); // ignore error + (void)taosTmrStopA(&pTask->delayTimer); // ignore error } taosHashClear(pTask->execNodes); - (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error schDeregisterTaskHb(pJob, pTask); taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; @@ -453,17 +453,19 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 } else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { - SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, + (int32_t)taosArrayGetSize(pTask->candidateAddrs)); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - - SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; + + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d, code:%s", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port, tstrerror(rspCode)); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { - SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, + (int32_t)taosArrayGetSize(pTask->candidateAddrs)); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -495,7 +497,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 for (int32_t i = 0; i < childrenNum; ++i) { SSchTask *pChild = taosArrayGetP(pTask->children, i); SCH_LOCK_TASK(pChild); - (void)schDoTaskRedirect(pJob, pChild, NULL, rspCode); // error handled internal + (void)schDoTaskRedirect(pJob, pChild, NULL, rspCode); // error handled internal SCH_UNLOCK_TASK(pChild); } @@ -509,13 +511,13 @@ _return: int32_t schResetTaskSetLevelInfo(SSchJob *pJob, SSchTask *pTask) { SSchLevel *pLevel = pTask->level; - SCH_TASK_DLOG("start to reset level for current task set, execDone:%d, launched:%d", - atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); + SCH_TASK_DLOG("start to reset level for current task set, execDone:%d, launched:%d", + atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); if (SCH_GET_TASK_STATUS(pTask) >= JOB_TASK_STATUS_PART_SUCC) { (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); } - + (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); int32_t childrenNum = taosArrayGetSize(pTask->children); @@ -525,16 +527,16 @@ int32_t schResetTaskSetLevelInfo(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("fail to get the %dth child, childrenNum:%d", i, childrenNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_LOCK_TASK(pChild); pLevel = pChild->level; (void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1); (void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1); SCH_UNLOCK_TASK(pChild); - } + } - SCH_TASK_DLOG("end to reset level for current task set, execDone:%d, launched:%d", - atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); + SCH_TASK_DLOG("end to reset level for current task set, execDone:%d, launched:%d", + atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum)); return TSDB_CODE_SUCCESS; } @@ -737,7 +739,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { (void)atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); - (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -749,10 +751,11 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { - SCH_TASK_ELOG("fail to the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("fail to the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, + (int32_t)taosArrayGetSize(pTask->candidateAddrs)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_SWITCH_EPSET(addr); } else { SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask)); @@ -776,7 +779,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("fail to get the %dth node in nodeList, nodeNum:%d", i, nodeNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SQueryNodeAddr *naddr = &nload->addr; if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { @@ -859,7 +862,7 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe char *origEpset = NULL; char *newEpset = NULL; - + SCH_ERR_RET(schDumpEpSet(&pAddr->epSet, &origEpset)); SCH_ERR_JRET(schDumpEpSet(pEpSet, &newEpset)); @@ -932,7 +935,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { if (nodeInfo->handle) { SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); void *pExecId = taosHashGetKey(nodeInfo, NULL); - (void)schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); // ignore error and continue + (void)schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); // ignore error and continue SCH_TASK_DLOG("start to drop task's %dth execNode", i); } else { @@ -986,10 +989,10 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { qError("fail to get the %dth task status in hb rsp, taskNum:%d", i, taskNum); continue; } - - int32_t code = 0; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, + int32_t code = 0; + + qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status)); if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) { @@ -1036,12 +1039,12 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { continue; } - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); + qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); pJob = NULL; (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, + qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -1061,7 +1064,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { (void)schReleaseJob(pJob->refId); - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, + qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); SCH_ERR_JRET(code); @@ -1079,7 +1082,7 @@ _return: qError("in _return fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum); continue; } - + tFreeSExplainRsp(&localRsp->rsp); } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 28321d0099..2d04f2cb34 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -107,7 +107,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pDispatch->taskId, pDispatch->nodeId, idstr); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64 - " window:%" PRId64 "-%" PRId64 " QID:0x%" PRIx64, + " window:%" PRId64 "-%" PRId64 " qid:0x%" PRIx64, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); @@ -133,7 +133,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pVgInfo->taskId, pVgInfo->vgId, idstr); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 - " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, QID:0x%" PRIx64, + " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, qid:0x%" PRIx64, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId); (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } @@ -171,14 +171,14 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); SStreamTaskState pState = streamTaskGetStatus(pTask); - stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(QID:0x%" PRIx64 + stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(qid:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); streamMetaReleaseTask(pMeta, pTask); } else { pRsp->status = TASK_DOWNSTREAM_NOT_READY; - stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(QID:0x%" PRIx64 + stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(qid:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp check_status %d", pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); } @@ -432,7 +432,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { if (reqId != p->reqId) { - stError("s-task:%s QID:0x%" PRIx64 " expected:0x%" PRIx64 + stError("s-task:%s qid:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded", id, reqId, p->reqId, taskId); streamMutexUnlock(&pInfo->checkInfoLock); @@ -454,7 +454,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t } streamMutexUnlock(&pInfo->checkInfoLock); - stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, QID:%" PRIx64 " discarded", id, taskId, + stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, qid:%" PRIx64 " discarded", id, taskId, reqId); return TSDB_CODE_FAILED; } @@ -541,7 +541,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher; setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId); - stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) QID:0x%" PRIx64, id, + stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) qid:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); (void)streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); @@ -559,7 +559,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { setCheckDownstreamReqInfo(&req, p->reqId, pVgInfo->taskId, pVgInfo->vgId); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 - " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d QID:0x%" PRIx64, + " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d qid:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId); (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); break; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 8928138950..a6c2ae73e5 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -130,7 +130,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r return code; } - stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), QID:0x%" PRIx64, pTask->id.idStr, + stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), qid:0x%" PRIx64, pTask->id.idStr, pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId); } @@ -158,7 +158,7 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl pRetrieve->version = htobe64(pBlock->info.version); int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); - if(actualLen < 0) { + if (actualLen < 0) { taosMemoryFree(pRetrieve); return terrno; } @@ -1088,7 +1088,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); - if(actualLen < 0) { + if (actualLen < 0) { taosMemoryFree(buf); return terrno; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index a57e8557a6..7cb800c756 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -140,7 +140,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to stError("s-task:%s failed to add retrieve block", pTask->id.idStr); } - stDebug("s-task:%s(child %d) retrieve process completed, QID:0x%" PRIx64 " dump results", pTask->id.idStr, + stDebug("s-task:%s(child %d) retrieve process completed, qid:0x%" PRIx64 " dump results", pTask->id.idStr, pTask->info.selfChildId, pRetrieveBlock->reqId); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 164ba558f2..a184314714 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -1092,7 +1092,7 @@ static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* } // enqueue - stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), QID:0x%" PRIx64, pTask->id.idStr, + stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), qid:0x%" PRIx64, pTask->id.idStr, pTask->pMeta->vgId, pTask->info.taskLevel, pReq->srcTaskId, pReq->srcNodeId, pReq->reqId); pData->type = STREAM_INPUT__DATA_RETRIEVE; diff --git a/source/libs/transport/inc/transLog.h b/source/libs/transport/inc/transLog.h index 8f789821fd..278356b42e 100644 --- a/source/libs/transport/inc/transLog.h +++ b/source/libs/transport/inc/transLog.h @@ -32,12 +32,12 @@ extern "C" { #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }} #define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } } -#define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0) -#define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", QID:%s", __VA_ARGS__, buf); }} while (0) -#define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", QID:%s", __VA_ARGS__, buf);} } while(0) -#define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", QID:%s", __VA_ARGS__, buf); }} while(0) -#define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", QID:%s", __VA_ARGS__, buf); }} while(0) -#define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", QID:%s", __VA_ARGS__, buf); }} while(0) +#define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", qid:%s", __VA_ARGS__, buf);}} while(0) +#define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", qid:%s", __VA_ARGS__, buf); }} while (0) +#define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", qid:%s", __VA_ARGS__, buf);} } while(0) +#define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", qid:%s", __VA_ARGS__, buf); }} while(0) +#define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", qid:%s", __VA_ARGS__, buf); }} while(0) +#define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", qid:%s", __VA_ARGS__, buf); }} while(0) // clang-format on From b0714e13e93d2b62b5fad5e9ac69a19a7a0465b3 Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Sun, 25 Aug 2024 10:43:07 +0800 Subject: [PATCH 36/59] move ref 3.0 code to main --- docs/en/14-reference/05-connectors/10-cpp.mdx | 12 ++-- .../en/14-reference/05-connectors/14-java.mdx | 4 +- docs/en/14-reference/05-connectors/20-go.mdx | 2 +- .../en/14-reference/05-connectors/26-rust.mdx | 4 +- docs/examples/JDBC/SpringJdbcTemplate/pom.xml | 2 +- .../taosdata/example/jdbcTemplate/App.java | 55 ++++++++++++------- docs/examples/JDBC/mybatisplus-demo/pom.xml | 4 +- .../config/MybatisPlusConfig.java | 14 ++--- .../mapper/TemperatureMapperTest.java | 2 +- .../mapper/WeatherMapperTest.java | 2 +- docs/zh/14-reference/05-connector/10-cpp.mdx | 12 ++-- docs/zh/14-reference/05-connector/14-java.mdx | 2 +- docs/zh/14-reference/05-connector/20-go.mdx | 4 +- docs/zh/14-reference/05-connector/26-rust.mdx | 2 +- 14 files changed, 67 insertions(+), 54 deletions(-) diff --git a/docs/en/14-reference/05-connectors/10-cpp.mdx b/docs/en/14-reference/05-connectors/10-cpp.mdx index 79ff00fd5b..6a570b2490 100644 --- a/docs/en/14-reference/05-connectors/10-cpp.mdx +++ b/docs/en/14-reference/05-connectors/10-cpp.mdx @@ -66,18 +66,18 @@ In the above example code, `taos_connect()` establishes a connection to port 603 This section shows sample code for standard access methods to TDengine clusters using the client driver. -- [Synchronous query example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/demo.c) +- [Synchronous query example](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/demo.c) -- [Asynchronous query example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/asyncdemo.c) +- [Asynchronous query example](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/asyncdemo.c) -- [Parameter binding example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/prepare.c) +- [Parameter binding example](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/prepare.c) -- [Schemaless writing example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/schemaless.c) +- [Schemaless writing example](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/schemaless.c) -- [Subscription and consumption example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/tmq.c) +- [Subscription and consumption example](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/tmq.c) :::info -More example code and downloads are available at [GitHub](https://github.com/taosdata/TDengine/tree/develop/docs/examples/c). +More example code and downloads are available at [GitHub](https://github.com/taosdata/TDengine/tree/main/docs/examples/c). You can find it in the installation directory under the `examples/c` path. This directory has a makefile and can be compiled under Linux/macOS by executing `make` directly. **Hint:** When compiling in an ARM environment, please remove `-msse4.2` from the makefile. This option is only supported on the x64/x86 hardware platforms. diff --git a/docs/en/14-reference/05-connectors/14-java.mdx b/docs/en/14-reference/05-connectors/14-java.mdx index 46b13409a3..aa7e91b7fa 100644 --- a/docs/en/14-reference/05-connectors/14-java.mdx +++ b/docs/en/14-reference/05-connectors/14-java.mdx @@ -150,7 +150,7 @@ TDengine currently supports timestamp, number, character, Boolean type, and the Due to historical reasons, the BINARY type data in TDengine is not truly binary data and is no longer recommended for use. Please use VARBINARY type instead. GEOMETRY type is binary data in little endian byte order, which complies with the WKB specification. For detailed information, please refer to [Data Type](../../taos-sql/data-type/) For WKB specifications, please refer to [Well Known Binary (WKB)](https://libgeos.org/specifications/wkb/) -For Java connector, the jts library can be used to easily create GEOMETRY type objects, serialize them, and write them to TDengine. Here is an example [Geometry example](https://github.com/taosdata/TDengine/blob/3.0/docs/examples/java/src/main/java/com/taos/example/GeometryDemo.java) +For Java connector, the jts library can be used to easily create GEOMETRY type objects, serialize them, and write them to TDengine. Here is an example [Geometry example](https://github.com/taosdata/TDengine/blob/main/docs/examples/java/src/main/java/com/taos/example/GeometryDemo.java) ## Installation Steps @@ -671,7 +671,7 @@ The source code of the sample application is under `TDengine/docs/examples/JDBC` - springbootdemo: using taos-jdbcdriver in Springboot. - consumer-demo: consumer TDengine data example, the consumption rate can be controlled by parameters. -[JDBC example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/JDBC) +[JDBC example](https://github.com/taosdata/TDengine/tree/main/docs/examples/JDBC) ## Frequently Asked Questions diff --git a/docs/en/14-reference/05-connectors/20-go.mdx b/docs/en/14-reference/05-connectors/20-go.mdx index 7be57baf6b..39ee97687c 100644 --- a/docs/en/14-reference/05-connectors/20-go.mdx +++ b/docs/en/14-reference/05-connectors/20-go.mdx @@ -368,7 +368,7 @@ The TDengine Go client library supports subscription functionality with the foll ### More sample programs -* [sample program](https://github.com/taosdata/driver-go/tree/3.0/examples) +* [sample program](https://github.com/taosdata/driver-go/tree/main/examples) ## Frequently Asked Questions diff --git a/docs/en/14-reference/05-connectors/26-rust.mdx b/docs/en/14-reference/05-connectors/26-rust.mdx index 68770a40e6..310f9760a5 100644 --- a/docs/en/14-reference/05-connectors/26-rust.mdx +++ b/docs/en/14-reference/05-connectors/26-rust.mdx @@ -403,7 +403,7 @@ The following parameters can be configured for the TMQ DSN. Only `group.id` is m #### Full Sample Code -For more information, see [GitHub sample file](https://github.com/taosdata/TDengine/blob/3.0/docs/examples/rust/nativeexample/examples/subscribe_demo.rs). +For more information, see [GitHub sample file](https://github.com/taosdata/TDengine/blob/main/docs/examples/rust/nativeexample/examples/subscribe_demo.rs). ### Use with connection pool @@ -439,7 +439,7 @@ let taos = pool.get()?; The source code of the sample application is under `TDengine/docs/examples/rust` : -[rust example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/rust) +[rust example](https://github.com/taosdata/TDengine/tree/main/docs/examples/rust) ## Frequently Asked Questions diff --git a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml index 6e4941b4f1..99129e180a 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.3.1-SNAPSHOT diff --git a/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java b/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java index ce26b7504a..b70c31a8f7 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java +++ b/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java @@ -6,6 +6,8 @@ import com.taosdata.example.jdbcTemplate.dao.WeatherDao; import com.taosdata.example.jdbcTemplate.domain.Weather; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; import java.sql.Timestamp; import java.util.Date; @@ -18,30 +20,41 @@ public class App { public static void main(String[] args) { - ApplicationContext ctx = new ClassPathXmlApplicationContext("applicationContext.xml"); +// ApplicationContext ctx = new ClassPathXmlApplicationContext("applicationContext.xml"); +// +// ExecuteAsStatement executor = ctx.getBean(ExecuteAsStatement.class); +// // drop database +// executor.doExecute("drop database if exists test"); +// // create database +// executor.doExecute("create database if not exists test"); +// //use database +// executor.doExecute("use test"); +// // create table +// executor.doExecute("create table if not exists test.weather (ts timestamp, temperature float, humidity int)"); +// +// WeatherDao weatherDao = ctx.getBean(WeatherDao.class); +// Weather weather = new Weather(new Timestamp(new Date().getTime()), random.nextFloat() * 50.0f, random.nextInt(100)); +// // insert rows +// int affectedRows = weatherDao.add(weather); +// System.out.println("insert success " + affectedRows + " rows."); +// +// // query for list +// int limit = 10, offset = 0; +// List weatherList = weatherDao.queryForList(limit, offset); +// for (Weather w : weatherList) { +// System.out.println(w); +// } + DriverManagerDataSource dataSource = new DriverManagerDataSource(); + dataSource.setDriverClassName("com.taosdata.jdbc.rs.RestfulDriver"); + dataSource.setUrl("jdbc:TAOS-RS://vm98:6041/?batchfetch=true&user=root&password=taosdata"); - ExecuteAsStatement executor = ctx.getBean(ExecuteAsStatement.class); - // drop database - executor.doExecute("drop database if exists test"); - // create database - executor.doExecute("create database if not exists test"); - //use database - executor.doExecute("use test"); - // create table - executor.doExecute("create table if not exists test.weather (ts timestamp, temperature float, humidity int)"); - WeatherDao weatherDao = ctx.getBean(WeatherDao.class); - Weather weather = new Weather(new Timestamp(new Date().getTime()), random.nextFloat() * 50.0f, random.nextInt(100)); - // insert rows - int affectedRows = weatherDao.add(weather); - System.out.println("insert success " + affectedRows + " rows."); - // query for list - int limit = 10, offset = 0; - List weatherList = weatherDao.queryForList(limit, offset); - for (Weather w : weatherList) { - System.out.println(w); - } + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + + jdbcTemplate.query("select * from test.meters limit ?, ?", + (rs, rowNum) -> rs.getString("ts"), 1, 10) + .forEach(System.out::println) ; } diff --git a/docs/examples/JDBC/mybatisplus-demo/pom.xml b/docs/examples/JDBC/mybatisplus-demo/pom.xml index f792946c96..66746d5978 100644 --- a/docs/examples/JDBC/mybatisplus-demo/pom.xml +++ b/docs/examples/JDBC/mybatisplus-demo/pom.xml @@ -31,7 +31,7 @@ com.baomidou mybatis-plus-boot-starter - 3.1.2 + 3.5.1 com.h2database @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.2.4 + 3.3.1-SNAPSHOT diff --git a/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java b/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java index a6ac7f7fc2..321d72814c 100644 --- a/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java +++ b/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java @@ -1,6 +1,7 @@ package com.taosdata.example.mybatisplusdemo.config; -import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -22,13 +23,10 @@ public class MybatisPlusConfig { // } @Bean - public PaginationInterceptor paginationInterceptor() { -// return new PaginationInterceptor(); - PaginationInterceptor paginationInterceptor = new PaginationInterceptor(); - //TODO: mybatis-plus do not support TDengine, use postgresql Dialect - paginationInterceptor.setDialectType("postgresql"); - - return paginationInterceptor; + public MybatisPlusInterceptor mybatisPlusInterceptor() { + MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); + interceptor.addInnerInterceptor(new PaginationInnerInterceptor()); + return interceptor; } } diff --git a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java index 4d9dbf8d2f..763d2b14fc 100644 --- a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java +++ b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java @@ -107,7 +107,7 @@ public class TemperatureMapperTest { * **/ @Test public void testSelectCount() { - int count = mapper.selectCount(null); + long count = mapper.selectCount(null); Assert.assertEquals(10, count); } diff --git a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java index dba8abd1ed..3e5de3d6a6 100644 --- a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java +++ b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java @@ -82,7 +82,7 @@ public class WeatherMapperTest { @Test public void testSelectCount() { - int count = mapper.selectCount(null); + long count = mapper.selectCount(null); // Assert.assertEquals(5, count); System.out.println(count); } diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index e25cef1bf0..7c0da088a6 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -40,18 +40,18 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 本节展示了使用客户端驱动访问 TDengine 集群的常见访问方式的示例代码。 -- 同步查询示例:[同步查询](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/demo.c) +- 同步查询示例:[同步查询](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/demo.c) -- 异步查询示例:[异步查询](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/asyncdemo.c) +- 异步查询示例:[异步查询](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/asyncdemo.c) -- 参数绑定示例:[参数绑定](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/prepare.c) +- 参数绑定示例:[参数绑定](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/prepare.c) -- 无模式写入示例:[无模式写入](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/schemaless.c) +- 无模式写入示例:[无模式写入](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/schemaless.c) -- 订阅和消费示例:[订阅和消费](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/c/tmq.c) +- 订阅和消费示例:[订阅和消费](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/tmq.c) :::info -更多示例代码及下载请见 [GitHub](https://github.com/taosdata/TDengine/tree/develop/docs/examples/c)。 +更多示例代码及下载请见 [GitHub](https://github.com/taosdata/TDengine/tree/main/docs/examples/c)。 也可以在安装目录下的 `examples/c` 路径下找到。 该目录下有 makefile,在 Linux/macOS 环境下,直接执行 make 就可以编译得到执行文件。 **提示:**在 ARM 环境下编译时,请将 makefile 中的 `-msse4.2` 去掉,这个选项只有在 x64/x86 硬件平台上才能支持。 diff --git a/docs/zh/14-reference/05-connector/14-java.mdx b/docs/zh/14-reference/05-connector/14-java.mdx index 42322139a9..5f95233163 100644 --- a/docs/zh/14-reference/05-connector/14-java.mdx +++ b/docs/zh/14-reference/05-connector/14-java.mdx @@ -154,7 +154,7 @@ WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/w - springbootdemo: Springboot 中使用 taos-jdbcdriver。 - consumer-demo:Consumer 消费 TDengine 数据示例,可通过参数控制消费速度。 -请参考:[JDBC example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/JDBC) +请参考:[JDBC example](https://github.com/taosdata/TDengine/tree/main/docs/examples/JDBC) ## 常见问题 diff --git a/docs/zh/14-reference/05-connector/20-go.mdx b/docs/zh/14-reference/05-connector/20-go.mdx index 45782a7284..8d65fba321 100644 --- a/docs/zh/14-reference/05-connector/20-go.mdx +++ b/docs/zh/14-reference/05-connector/20-go.mdx @@ -66,6 +66,9 @@ TDengine 其他功能模块的报错,请参考 [错误码](../../../reference/ **注意**:JSON 类型仅在 tag 中支持。 +## 示例程序汇总 +示例程序源码请参考:[示例程序](https://github.com/taosdata/driver-go/tree/main/examples) + ## 常见问题 1. database/sql 中 stmt(参数绑定)相关接口崩溃 @@ -894,5 +897,4 @@ type TopicPartition struct { ## 附录 * [driver-go 文档](https://pkg.go.dev/github.com/taosdata/driver-go/v3)。 -* [示例程序](https://github.com/taosdata/driver-go/tree/3.0/examples)。 * [视频教程](https://www.taosdata.com/blog/2020/11/11/1951.html)。 diff --git a/docs/zh/14-reference/05-connector/26-rust.mdx b/docs/zh/14-reference/05-connector/26-rust.mdx index 25a90d4fab..88be297ac6 100644 --- a/docs/zh/14-reference/05-connector/26-rust.mdx +++ b/docs/zh/14-reference/05-connector/26-rust.mdx @@ -85,7 +85,7 @@ TDengine 目前支持时间戳、数字、字符、布尔类型,与 Rust 对 ## 示例程序汇总 -示例程序源码请参考:[rust example](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/rust) +示例程序源码请参考:[rust example](https://github.com/taosdata/TDengine/tree/main/docs/examples/rust) ## 常见问题 From 4af3ab50825590cab90521577f954359d22339a4 Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Sun, 25 Aug 2024 10:45:34 +0800 Subject: [PATCH 37/59] update java code example --- docs/examples/JDBC/SpringJdbcTemplate/pom.xml | 2 +- .../taosdata/example/jdbcTemplate/App.java | 55 +++++++------------ docs/examples/JDBC/mybatisplus-demo/pom.xml | 4 +- .../config/MybatisPlusConfig.java | 14 +++-- .../mapper/TemperatureMapperTest.java | 2 +- .../mapper/WeatherMapperTest.java | 2 +- 6 files changed, 34 insertions(+), 45 deletions(-) diff --git a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml index 99129e180a..6e4941b4f1 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.3.1-SNAPSHOT + 3.0.0 diff --git a/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java b/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java index b70c31a8f7..ce26b7504a 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java +++ b/docs/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java @@ -6,8 +6,6 @@ import com.taosdata.example.jdbcTemplate.dao.WeatherDao; import com.taosdata.example.jdbcTemplate.domain.Weather; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DriverManagerDataSource; import java.sql.Timestamp; import java.util.Date; @@ -20,41 +18,30 @@ public class App { public static void main(String[] args) { -// ApplicationContext ctx = new ClassPathXmlApplicationContext("applicationContext.xml"); -// -// ExecuteAsStatement executor = ctx.getBean(ExecuteAsStatement.class); -// // drop database -// executor.doExecute("drop database if exists test"); -// // create database -// executor.doExecute("create database if not exists test"); -// //use database -// executor.doExecute("use test"); -// // create table -// executor.doExecute("create table if not exists test.weather (ts timestamp, temperature float, humidity int)"); -// -// WeatherDao weatherDao = ctx.getBean(WeatherDao.class); -// Weather weather = new Weather(new Timestamp(new Date().getTime()), random.nextFloat() * 50.0f, random.nextInt(100)); -// // insert rows -// int affectedRows = weatherDao.add(weather); -// System.out.println("insert success " + affectedRows + " rows."); -// -// // query for list -// int limit = 10, offset = 0; -// List weatherList = weatherDao.queryForList(limit, offset); -// for (Weather w : weatherList) { -// System.out.println(w); -// } - DriverManagerDataSource dataSource = new DriverManagerDataSource(); - dataSource.setDriverClassName("com.taosdata.jdbc.rs.RestfulDriver"); - dataSource.setUrl("jdbc:TAOS-RS://vm98:6041/?batchfetch=true&user=root&password=taosdata"); + ApplicationContext ctx = new ClassPathXmlApplicationContext("applicationContext.xml"); + ExecuteAsStatement executor = ctx.getBean(ExecuteAsStatement.class); + // drop database + executor.doExecute("drop database if exists test"); + // create database + executor.doExecute("create database if not exists test"); + //use database + executor.doExecute("use test"); + // create table + executor.doExecute("create table if not exists test.weather (ts timestamp, temperature float, humidity int)"); + WeatherDao weatherDao = ctx.getBean(WeatherDao.class); + Weather weather = new Weather(new Timestamp(new Date().getTime()), random.nextFloat() * 50.0f, random.nextInt(100)); + // insert rows + int affectedRows = weatherDao.add(weather); + System.out.println("insert success " + affectedRows + " rows."); - JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); - - jdbcTemplate.query("select * from test.meters limit ?, ?", - (rs, rowNum) -> rs.getString("ts"), 1, 10) - .forEach(System.out::println) ; + // query for list + int limit = 10, offset = 0; + List weatherList = weatherDao.queryForList(limit, offset); + for (Weather w : weatherList) { + System.out.println(w); + } } diff --git a/docs/examples/JDBC/mybatisplus-demo/pom.xml b/docs/examples/JDBC/mybatisplus-demo/pom.xml index 66746d5978..f792946c96 100644 --- a/docs/examples/JDBC/mybatisplus-demo/pom.xml +++ b/docs/examples/JDBC/mybatisplus-demo/pom.xml @@ -31,7 +31,7 @@ com.baomidou mybatis-plus-boot-starter - 3.5.1 + 3.1.2 com.h2database @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.3.1-SNAPSHOT + 3.2.4 diff --git a/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java b/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java index 321d72814c..a6ac7f7fc2 100644 --- a/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java +++ b/docs/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java @@ -1,7 +1,6 @@ package com.taosdata.example.mybatisplusdemo.config; -import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; -import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; +import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -23,10 +22,13 @@ public class MybatisPlusConfig { // } @Bean - public MybatisPlusInterceptor mybatisPlusInterceptor() { - MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); - interceptor.addInnerInterceptor(new PaginationInnerInterceptor()); - return interceptor; + public PaginationInterceptor paginationInterceptor() { +// return new PaginationInterceptor(); + PaginationInterceptor paginationInterceptor = new PaginationInterceptor(); + //TODO: mybatis-plus do not support TDengine, use postgresql Dialect + paginationInterceptor.setDialectType("postgresql"); + + return paginationInterceptor; } } diff --git a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java index 763d2b14fc..4d9dbf8d2f 100644 --- a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java +++ b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java @@ -107,7 +107,7 @@ public class TemperatureMapperTest { * **/ @Test public void testSelectCount() { - long count = mapper.selectCount(null); + int count = mapper.selectCount(null); Assert.assertEquals(10, count); } diff --git a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java index 3e5de3d6a6..dba8abd1ed 100644 --- a/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java +++ b/docs/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java @@ -82,7 +82,7 @@ public class WeatherMapperTest { @Test public void testSelectCount() { - long count = mapper.selectCount(null); + int count = mapper.selectCount(null); // Assert.assertEquals(5, count); System.out.println(count); } From 5fd3e7dc81f21d334f7b79ee3eb4e4b7042956f3 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Mon, 26 Aug 2024 08:55:20 +0800 Subject: [PATCH 38/59] modify show_create_db.py case --- tests/develop-test/2-query/show_create_db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index 3fe68a6803..c53bda1a02 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -42,17 +42,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") self.restartTaosd(1, dbname='scd') @@ -60,16 +60,16 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.execute('drop database scd') From b8588feeff745e7cb57243abd00dffe56a47febe Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Mon, 26 Aug 2024 08:58:19 +0800 Subject: [PATCH 39/59] modify show_create_db.py case --- tests/develop-test/2-query/show_create_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index c53bda1a02..d7d093aa78 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -47,7 +47,7 @@ class TDTestCase: tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd4') tdSql.checkRows(1) @@ -60,7 +60,7 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) From 94f7bb7716e676ef0a5a6967e9d9ab5b24882623 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 26 Aug 2024 09:48:02 +0800 Subject: [PATCH 40/59] data check --- source/client/src/clientImpl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 039e17fd87..890d9baabc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1051,6 +1051,10 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock* for (int32_t i = 0; i < numOfRows; ++i) { TAOS_ROW pRow = taos_fetch_row(pRes); + if(NULL == pRow[0] || NULL == pRow[1] || NULL == pRow[2]) { + tscError("invalid data from vnode"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } int64_t ts = *(int64_t*)pRow[0]; if (lastTs < ts) { lastTs = ts; From 901107df162734025417d9351a20a90ac174de3c Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Aug 2024 10:13:09 +0800 Subject: [PATCH 41/59] enh: refactor return code --- source/dnode/mnode/impl/src/mndUser.c | 9 +++++++-- source/dnode/vnode/src/sma/smaEnv.c | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c4823dc62e..49cacb3dce 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -3147,7 +3147,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ if (TSDB_CODE_MND_USER_NOT_EXIST == code) { SGetUserAuthRsp rsp = {.dropped = 1}; (void)memcpy(rsp.user, pUsers[i].user, TSDB_USER_LEN); - (void)taosArrayPush(batchRsp.pArray, &rsp); + TSDB_CHECK_NULL(taosArrayPush(batchRsp.pArray, &rsp), code, lino, _OVER, TSDB_CODE_OUT_OF_MEMORY); } mError("user:%s, failed to auth user since %s", pUsers[i].user, tstrerror(code)); code = 0; @@ -3168,7 +3168,12 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ TAOS_CHECK_GOTO(code, &lino, _OVER); } - (void)taosArrayPush(batchRsp.pArray, &rsp); + if (!(taosArrayPush(batchRsp.pArray, &rsp))) { + code = TSDB_CODE_OUT_OF_MEMORY; + mndReleaseUser(pMnode, pUser); + tFreeSGetUserAuthRsp(&rsp); + TAOS_CHECK_GOTO(code, &lino, _OVER); + } mndReleaseUser(pMnode, pUser); } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index d90e869bd4..264758bf3e 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -213,7 +213,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS TAOS_CHECK_GOTO(code, &lino, _exit); } SSDataBlock datablock = {.info.type = STREAM_CHECKPOINT}; - (void)taosArrayPush(pRSmaStat->blocks, &datablock); + TSDB_CHECK_NULL(taosArrayPush(pRSmaStat->blocks, &datablock), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); // init smaMgmt TAOS_CHECK_GOTO(smaInit(), &lino, _exit); From 8dcc59495bf4fbffb66bb9d17f5f955e4485d6a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Aug 2024 10:23:21 +0800 Subject: [PATCH 42/59] ehn: remote some asserts --- source/common/src/tdataformat.c | 20 ++++++------ source/common/src/ttime.c | 24 ++------------ source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 4 ++- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 26 +++++++-------- source/libs/index/src/indexFst.c | 34 +------------------- source/libs/index/src/indexFstNode.c | 1 - 6 files changed, 26 insertions(+), 83 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1df68b5389..8e44c9c4b4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -934,7 +934,8 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; } else { - ASSERT(0); + uError("unexpected column id %d, %d", cid, pTColumn->colId); + goto _exit; } } else { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); @@ -1356,11 +1357,8 @@ int32_t tValueCompare(const SValue *tv1, const SValue *tv2) { int32_t ret = memcmp(tv1->pData, tv2->pData, tv1->nData < tv2->nData ? tv1->nData : tv2->nData); return ret ? ret : (tv1->nData < tv2->nData ? -1 : (tv1->nData > tv2->nData ? 1 : 0)); } - case TSDB_DATA_TYPE_DECIMAL: - ASSERT(0); - break; default: - ASSERT(0); + break; } return 0; @@ -2600,7 +2598,7 @@ static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); break; default: - ASSERT(0); + break; } } static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_VALUE @@ -2628,7 +2626,7 @@ static FORCE_INLINE void tColDataGetValue5(SColData *pColData, int32_t iVal, tColDataGetValue4(pColData, iVal, pColVal); break; default: - ASSERT(0); + break; } } static FORCE_INLINE void tColDataGetValue6(SColData *pColData, int32_t iVal, @@ -2641,7 +2639,7 @@ static FORCE_INLINE void tColDataGetValue6(SColData *pColData, int32_t iVal, tColDataGetValue4(pColData, iVal, pColVal); break; default: - ASSERT(0); + break; } } static FORCE_INLINE void tColDataGetValue7(SColData *pColData, int32_t iVal, @@ -2657,7 +2655,7 @@ static FORCE_INLINE void tColDataGetValue7(SColData *pColData, int32_t iVal, tColDataGetValue4(pColData, iVal, pColVal); break; default: - ASSERT(0); + break; } } static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal *pColVal) = { @@ -2671,7 +2669,6 @@ static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal tColDataGetValue7 // HAS_VALUE | HAS_NULL | HAS_NONE }; void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { - ASSERT(iVal >= 0 && iVal < pColData->nVal && pColData->flag); tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal); } @@ -3334,7 +3331,8 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd / } else if (bv == BIT_FLG_NULL) { flag |= HAS_NULL; } else { - ASSERT(0); + uError("invalid bit value:%d", bv); + return; } if (flag == pColData->flag) break; diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 2a8e7951b1..79847d4e4a 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -397,11 +397,6 @@ char getPrecisionUnit(int32_t precision) { } int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPrecision) { - ASSERT(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || - fromPrecision == TSDB_TIME_PRECISION_NANO); - ASSERT(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || - toPrecision == TSDB_TIME_PRECISION_NANO); - switch (fromPrecision) { case TSDB_TIME_PRECISION_MILLI: { switch (toPrecision) { @@ -418,7 +413,6 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre } return utime * 1000000; default: - ASSERT(0); return utime; } } // end from milli @@ -434,7 +428,6 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre } return utime * 1000; default: - ASSERT(0); return utime; } } // end from micro @@ -447,12 +440,10 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre case TSDB_TIME_PRECISION_NANO: return utime; default: - ASSERT(0); return utime; } } // end from nano default: { - ASSERT(0); return utime; // only to pass windows compilation } } // end switch fromPrecision @@ -463,10 +454,6 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre // !!!!notice:there are precision problems, double lose precison if time is too large, for example: // 1626006833631000000*1.0 = double = 1626006833631000064 // int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { -// assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || -// fromPrecision == TSDB_TIME_PRECISION_NANO); -// assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || -// toPrecision == TSDB_TIME_PRECISION_NANO); // static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}}; // ((double)time * factors[fromPrecision][toPrecision]); //} @@ -783,7 +770,6 @@ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interva int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->sliding == 0) { - ASSERT(pInterval->interval == 0); return ts; } @@ -814,7 +800,6 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { } else { if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { int64_t news = (ts / pInterval->sliding) * pInterval->sliding; - ASSERT(news <= ts); if (pInterval->slidingUnit == 'd' || pInterval->slidingUnit == 'w') { #if defined(WINDOWS) && _MSC_VER >= 1900 int64_t timezone = _timezone; @@ -887,8 +872,6 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { } } - ASSERT(pInterval->offset >= 0); - if (pInterval->offset > 0) { // try to move current window to the left-hande-side, due to the offset effect. int64_t newe = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; @@ -1284,7 +1267,6 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { } if (last) { // expand - assert(last->type == TS_FORMAT_NODE_TYPE_CHAR); last->len++; formatStr++; } else { @@ -1311,7 +1293,6 @@ static int32_t parseTsFormat(const char* formatStr, SArray* formats) { } } if (lastOtherFormat) { - assert(lastOtherFormat->type == TS_FORMAT_NODE_TYPE_CHAR); lastOtherFormat->len++; formatStr++; } else { @@ -1664,7 +1645,6 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec } continue; } - assert(node->type == TS_FORMAT_NODE_TYPE_KEYWORD); switch (node->key->id) { case TSFKW_A_M: case TSFKW_P_M: @@ -1929,7 +1909,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { if (!*formats) { *formats = taosArrayInit(8, sizeof(TSFormatNode)); - if (!*formats){ + if (!*formats) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } TAOS_CHECK_RETURN(parseTsFormat(format, *formats)); @@ -2002,7 +1982,7 @@ static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1, /*o*/ -1, -1, -1, -1, 3, -1, /*u*/ 1, -1, 7, -1, 9, -1}; -#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97] +#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx)-97] // clang-format off static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 51f8f15537..f80c898f68 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -1036,7 +1036,9 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, bData->suid != 0 ? bData->suid : bData->uid, &cmprInfo.pColCmpr); - ASSERT(code == TSDB_CODE_SUCCESS); + if (code) { + tsdbWarn("vgId:%d failed to get column compress algrithm", TD_VID(writer->config->tsdb->pVnode)); + } TAOS_CHECK_GOTO(tBlockDataCompress(bData, &cmprInfo, buffers, assist), &lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 901632894e..21c5d33ec9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -319,7 +319,7 @@ static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray } else { // all blocks are qualified taosArrayClear(pBlockLoadInfo->aSttBlk); px = taosArrayAddBatch(pBlockLoadInfo->aSttBlk, pArray->data, pArray->size); - if (px == NULL){ + if (px == NULL) { return terrno; } } @@ -336,7 +336,7 @@ static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray } if (p->suid == suid) { - void* px = taosArrayPush(pTmp, p); + void *px = taosArrayPush(pTmp, p); if (px == NULL) { code = terrno; break; @@ -372,7 +372,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl TStatisBlkArray *pStatisBlkArray, uint64_t suid, const char *id) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - void* px = NULL; + void *px = NULL; int32_t startIndex = 0; int32_t numOfBlocks = TARRAY2_SIZE(pStatisBlkArray); @@ -415,7 +415,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl // existed if (i < rows) { - SSttTableRowsInfo* pInfo = &pBlockLoadInfo->info; + SSttTableRowsInfo *pInfo = &pBlockLoadInfo->info; if (pInfo->pUid == NULL) { pInfo->pUid = taosArrayInit(rows, sizeof(int64_t)); @@ -530,7 +530,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl } } - _end: +_end: (void)tStatisBlockDestroy(&block); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -672,7 +672,7 @@ int32_t tLDataIterOpen2(SLDataIter *pIter, SSttFileReader *pSttFileReader, int32 } void tLDataIterClose2(SLDataIter *pIter) { - (void) tsdbSttFileReaderClose(&pIter->pReader); // always return 0 + (void)tsdbSttFileReaderClose(&pIter->pReader); // always return 0 pIter->pReader = NULL; } @@ -826,7 +826,7 @@ static int32_t findNextValidRow(SLDataIter *pIter, const char *idStr) { return code; } -int32_t tLDataIterNextRow(SLDataIter *pIter, const char *idStr, bool* hasNext) { +int32_t tLDataIterNextRow(SLDataIter *pIter, const char *idStr, bool *hasNext) { int32_t step = pIter->backward ? -1 : 1; int32_t code = 0; int32_t iBlockL = pIter->iSttBlk; @@ -835,7 +835,7 @@ int32_t tLDataIterNextRow(SLDataIter *pIter, const char *idStr, bool* hasNext) { *hasNext = false; terrno = 0; - + // no qualified last file block in current file, no need to fetch row if (pIter->pSttBlk == NULL) { return code; @@ -1020,7 +1020,7 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF // let's record the time window for current table of uid in the stt files if (pSttDataInfo != NULL && numOfRows > 0) { - void* px = taosArrayPush(pSttDataInfo->pKeyRangeList, &range); + void *px = taosArrayPush(pSttDataInfo->pKeyRangeList, &range); if (px == NULL) { return terrno; } @@ -1041,7 +1041,7 @@ _end: return code; } -void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { (void) tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { (void)tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree) { return pMTree->ignoreEarlierTs; } @@ -1050,14 +1050,12 @@ static void tLDataIterPinSttBlock(SLDataIter *pIter, const char *id) { if (pInfo->blockData[0].sttBlockIndex == pIter->iSttBlk) { pInfo->blockData[0].pin = true; - ASSERT(!pInfo->blockData[1].pin); tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id); return; } if (pInfo->blockData[1].sttBlockIndex == pIter->iSttBlk) { pInfo->blockData[1].pin = true; - ASSERT(!pInfo->blockData[0].pin); tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id); return; } @@ -1068,14 +1066,12 @@ static void tLDataIterPinSttBlock(SLDataIter *pIter, const char *id) { static void tLDataIterUnpinSttBlock(SLDataIter *pIter, const char *id) { SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo; if (pInfo->blockData[0].pin) { - ASSERT(!pInfo->blockData[1].pin); pInfo->blockData[0].pin = false; tsdbTrace("unpin stt-block:%d, stt-fileVer:%" PRId64 " %s", pInfo->blockData[0].sttBlockIndex, pIter->cid, id); return; } if (pInfo->blockData[1].pin) { - ASSERT(!pInfo->blockData[0].pin); pInfo->blockData[1].pin = false; tsdbTrace("unpin stt-block:%d, stt-fileVer:%" PRId64 " %s", pInfo->blockData[1].sttBlockIndex, pIter->cid, id); return; @@ -1117,7 +1113,7 @@ int32_t tMergeTreeNext(SMergeTree *pMTree, bool *pHasNext) { code = tLDataIterNextRow(pIter, pMTree->idStr, &hasVal); if (!hasVal || (code != 0)) { if (code == TSDB_CODE_FILE_CORRUPTED) { - code = 0; // suppress the file corrupt error to enable all queries within this cluster can run without failed. + code = 0; // suppress the file corrupt error to enable all queries within this cluster can run without failed. } pMTree->pIter = NULL; diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index e57ba6267d..3edd61bd08 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -318,8 +318,6 @@ void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode // set_comm_input void fstStateSetCommInput(FstState* s, uint8_t inp) { - // ASSERT(s->state == OneTransNext || s->state == OneTrans); - uint8_t val; COMMON_INDEX(inp, 0b111111, val); s->val = (s->val & fstStateDict[s->state].val) | val; @@ -327,7 +325,6 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) { // comm_input uint8_t fstStateCommInput(FstState* s, bool* null) { - // ASSERT(s->state == OneTransNext || s->state == OneTrans); uint8_t v = s->val & 0b00111111; if (v == 0) { *null = true; @@ -340,7 +337,6 @@ uint8_t fstStateCommInput(FstState* s, bool* null) { // input_len uint64_t fstStateInputLen(FstState* s) { - // ASSERT(s->state == OneTransNext || s->state == OneTrans); bool null = false; (void)fstStateCommInput(s, &null); return null ? 1 : 0; @@ -348,11 +344,9 @@ uint64_t fstStateInputLen(FstState* s) { // end_addr uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { - // ASSERT(s->state == OneTransNext); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); } uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { - // ASSERT(s->state == OneTrans); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); } @@ -366,7 +360,6 @@ uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* } // input uint8_t fstStateInput(FstState* s, FstNode* node) { - // ASSERT(s->state == OneTransNext || s->state == OneTrans); FstSlice* slice = &node->data; bool null = false; uint8_t inp = fstStateCommInput(s, &null); @@ -374,7 +367,6 @@ uint8_t fstStateInput(FstState* s, FstNode* node) { return null == false ? inp : data[node->start - 1]; } uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - // ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -386,7 +378,6 @@ uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { // trans_addr CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { - // ASSERT(s->state == OneTransNext || s->state == OneTrans); FstSlice* slice = &node->data; if (s->state == OneTransNext) { return (CompiledAddr)(node->end) - 1; @@ -402,8 +393,6 @@ CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { } } CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - // ASSERT(s->state == AnyTrans); - FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - @@ -414,7 +403,6 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i // sizes PackSizes fstStateSizes(FstState* s, FstSlice* slice) { - /// ASSERT(s->state == OneTrans || s->state == AnyTrans); uint64_t i; if (s->state == OneTrans) { i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; @@ -427,8 +415,6 @@ PackSizes fstStateSizes(FstState* s, FstSlice* slice) { } // Output Output fstStateOutput(FstState* s, FstNode* node) { - // ASSERT(s->state == OneTrans); - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { return 0; @@ -441,8 +427,6 @@ Output fstStateOutput(FstState* s, FstNode* node) { return unpackUint64(data + i, oSizes); } Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - // ASSERT(s->state == AnyTrans); - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { return 0; @@ -458,19 +442,14 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { // anyTrans specify function void fstStateSetFinalState(FstState* s, bool yes) { - // ASSERT(s->state == AnyTrans); if (yes) { s->val |= 0b01000000; } return; } -bool fstStateIsFinalState(FstState* s) { - // ASSERT(s->state == AnyTrans); - return (s->val & 0b01000000) == 0b01000000; -} +bool fstStateIsFinalState(FstState* s) { return (s->val & 0b01000000) == 0b01000000; } void fstStateSetStateNtrans(FstState* s, uint8_t n) { - // ASSERT(s->state == AnyTrans); if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } @@ -478,7 +457,6 @@ void fstStateSetStateNtrans(FstState* s, uint8_t n) { } // state_ntrans uint8_t fstStateStateNtrans(FstState* s, bool* null) { - // ASSERT(s->state == AnyTrans); *null = false; uint8_t n = s->val & 0b00111111; @@ -488,16 +466,13 @@ uint8_t fstStateStateNtrans(FstState* s, bool* null) { return n; } uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { - // ASSERT(s->state == AnyTrans); uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; } uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { - // ASSERT(s->state == AnyTrans); return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; } uint64_t fstStateNtransLen(FstState* s) { - // ASSERT(s->state == AnyTrans); bool null = false; (void)fstStateStateNtrans(s, &null); return null == true ? 1 : 0; @@ -526,7 +501,6 @@ Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackS return unpackUint64(data + at, (uint8_t)oSizes); } uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { - // ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -672,17 +646,14 @@ bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { bool s = true; FstState* st = &node->state; if (st->state == OneTransNext) { - /// ASSERT(i == 0); (void)fstStateTransAddr(st, node); } else if (st->state == OneTrans) { - // ASSERT(i == 0); (void)fstStateTransAddr(st, node); } else if (st->state == AnyTrans) { (void)fstStateTransAddrForAnyTrans(st, node, i); } else if (FST_STATE_EMPTY_FINAL(node)) { s = false; } else { - // ASSERT(0); } return s; } @@ -718,7 +689,6 @@ bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { int32_t sz = taosArrayGetSize(builderNode->trans); - // ASSERT(sz < 256); if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { return true; } else if (sz != 1 || builderNode->isFinal) { @@ -800,7 +770,6 @@ void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); if (prefixLen == FST_SLICE_LEN(s)) { - // ASSERT(out == 0); return; } @@ -844,7 +813,6 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { addr = fstBuilderCompile(b, bn); fstBuilderNodeDestroy(bn); - // ASSERT(addr != NONE_ADDRESS); } fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; diff --git a/source/libs/index/src/indexFstNode.c b/source/libs/index/src/indexFstNode.c index 041444f1c9..7f1a86bc55 100644 --- a/source/libs/index/src/indexFstNode.c +++ b/source/libs/index/src/indexFstNode.c @@ -98,7 +98,6 @@ void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { // bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile *wrt, CompiledAddr lastAddr, CompiledAddr // startAddr) { // size_t sz = taosArrayGetSize(b->trans); -// ASSERT(sz < 256); // if (FST_BUILDER_NODE_IS_FINAL(b) // && FST_BUILDER_NODE_TRANS_ISEMPTY(b) // && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { From e4a4832bb937630fb1fe857875ffd403e8a325db Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 26 Aug 2024 02:39:34 +0000 Subject: [PATCH 43/59] fix/TD-31542-remove-assert-sync2 --- source/libs/sync/src/syncElection.c | 6 ++- source/libs/sync/src/syncMain.c | 59 ++++++++++++++++++++++------- source/libs/sync/src/syncPipeline.c | 5 +++ 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index fc056c9eba..901b43023a 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -100,7 +100,11 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { if (voteGrantedMajority(pSyncNode->pVotesGranted)) { // only myself, to leader - ASSERT(!pSyncNode->pVotesGranted->toLeader); + if (pSyncNode->pVotesGranted->toLeader) { + ret = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("vgId:%d, failed to elect since already be to leader", pSyncNode->vgId); + return ret; + } syncNodeCandidate2Leader(pSyncNode); pSyncNode->pVotesGranted->toLeader = true; return ret; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 179bb4d503..2e76fcdf91 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -726,7 +726,12 @@ int32_t syncUpdateArbTerm(int64_t rid, SyncTerm arbTerm) { } SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapshotLastApplyIndex) { - ASSERT(pSyncNode->raftCfg.configIndexCount >= 1); + if (!(pSyncNode->raftCfg.configIndexCount >= 1)) { + sError("vgId:%d, failed get snapshot config index, configIndexCount:%d", pSyncNode->vgId, + pSyncNode->raftCfg.configIndexCount); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; + } SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0]; for (int32_t i = 0; i < pSyncNode->raftCfg.configIndexCount; ++i) { @@ -1477,9 +1482,18 @@ int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) { #endif void syncNodePreClose(SSyncNode* pSyncNode) { - ASSERT(pSyncNode != NULL); - ASSERT(pSyncNode->pFsm != NULL); - ASSERT(pSyncNode->pFsm->FpApplyQueueItems != NULL); + if (pSyncNode == NULL) { + sError("failed to pre close sync node since sync node is null"); + return; + } + if (pSyncNode->pFsm == NULL) { + sError("failed to pre close sync node since fsm is null"); + return; + } + if (pSyncNode->pFsm->FpApplyQueueItems == NULL) { + sError("failed to pre close sync node since FpApplyQueueItems is null"); + return; + } // stop elect timer (void)syncNodeStopElectTimer(pSyncNode); @@ -2080,7 +2094,10 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { SyncIndex lastIndex; SyncTerm lastTerm; int32_t code = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to become leader since %s", pSyncNode->vgId, tstrerror(code)); + return; + } pSyncNode->pNextIndex->index[i] = lastIndex + 1; } @@ -2153,7 +2170,10 @@ void syncNodeBecomeAssignedLeader(SSyncNode* pSyncNode) { SyncIndex lastIndex; SyncTerm lastTerm; int32_t code = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to become assigned leader since %s", pSyncNode->vgId, tstrerror(code)); + return; + } pSyncNode->pNextIndex->index[i] = lastIndex + 1; } @@ -2196,7 +2216,10 @@ void syncNodeBecomeAssignedLeader(SSyncNode* pSyncNode) { } void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { - ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + if (pSyncNode->state != TAOS_SYNC_STATE_CANDIDATE) { + sError("vgId:%d, failed leader from candidate since node state is wrong:%d", pSyncNode->vgId, pSyncNode->state); + return; + } bool granted = voteGrantedMajority(pSyncNode->pVotesGranted); if (!granted) { sError("vgId:%d, not granted by majority.", pSyncNode->vgId); @@ -2229,7 +2252,10 @@ int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) { } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { - ASSERT(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); + if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { + sError("vgId:%d, failed candidate from follower since node state is wrong:%d", pSyncNode->vgId, pSyncNode->state); + return; + } pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; pSyncNode->roleTimeMs = taosGetTimestampMs(); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); @@ -2259,11 +2285,17 @@ int32_t syncNodeAssignedLeader2Leader(SSyncNode* pSyncNode) { } // just called by syncNodeVoteForSelf -// need assert void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { - ASSERT(term == raftStoreGetTerm(pSyncNode)); + SyncTerm storeTerm = raftStoreGetTerm(pSyncNode); + if (term != storeTerm) { + sError("vgId:%d, failed to vote for term, term:%" PRId64 ", storeTerm:%" PRId64, pSyncNode->vgId, term, storeTerm); + return; + } bool voted = raftStoreHasVoted(pSyncNode); - ASSERT(!voted); + if (voted) { + sError("vgId:%d, failed to vote for term since not voted", pSyncNode->vgId); + return; + } raftStoreVote(pSyncNode, pRaftId); } @@ -2407,7 +2439,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { .lastConfigIndex = SYNC_INDEX_INVALID}; if (code == 0) { - ASSERT(pPreEntry != NULL); + if (pPreEntry == NULL) return -1; preTerm = pPreEntry->term; if (h) { @@ -3238,7 +3270,6 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { // append to log buffer if ((code = syncLogBufferAppend(ths->pLogBuf, ths, pEntry)) < 0) { sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); - ASSERT(terrno != 0); (void)syncFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno, false); syncEntryDestroy(pEntry); goto _out; @@ -3347,7 +3378,7 @@ static int32_t syncNodeAppendNoopOld(SSyncNode* ths) { SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = raftStoreGetTerm(ths); SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); - ASSERT(pEntry != NULL); + if (pEntry == NULL) return -1; LRUHandle* h = NULL; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 22947bdd8e..5687321579 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -617,6 +617,11 @@ int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTe SFsmCbMeta cbMeta = {0}; cbMeta.index = pEntry->index; cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); + if (cbMeta.lastConfigIndex < 0) { + code = TSDB_CODE_SYN_INTERNAL_ERROR; + if (terrno != 0) code = terrno; + return code; + } cbMeta.isWeak = pEntry->isWeak; cbMeta.code = applyCode; cbMeta.state = role; From 8a5cf9cebcb9362dac015e088b8ea951b1ddbdb9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 26 Aug 2024 10:44:23 +0800 Subject: [PATCH 44/59] tetst:add memory leak detection for TD in psiminfo to the CI --- tests/script/sh/checkAsan.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index b1571aa173..aa73d4cb23 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -49,6 +49,7 @@ fi indirect_leak=$(cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l) python_error=$(cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l) +python_taos_error=$(cat ${LOG_DIR}/*.info |grep "#" | grep -w "TDinternal" | wc -l) # ignore @@ -84,17 +85,18 @@ echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" echo -e "\033[44;32;1m"asan indirect_leak: $indirect_leak"\033[0m" echo -e "\033[44;32;1m"asan runtime error: $runtime_error"\033[0m" echo -e "\033[44;32;1m"asan python error: $python_error"\033[0m" +echo -e "\033[44;32;1m"asan python taos error: $python_taos_error"\033[0m" -let "errors=$error_num+$memory_leak+$indirect_leak+$runtime_error+$python_error" +let "errors=$error_num+$memory_leak+$indirect_leak+$runtime_error+$python_error+$python_taos_error" if [ $errors -eq 0 ]; then echo -e "\033[44;32;1m"no asan errors"\033[0m" exit 0 else echo -e "\033[44;31;1m"asan total errors: $errors"\033[0m" - if [ $python_error -ne 0 ]; then - cat ${LOG_DIR}/*.info + if [ $python_error -ne 0 ] || [ $python_taos_error -ne 0 ] ; then + cat ${LOG_DIR}/*.info |grep "#" | grep -w "TDinternal" fi cat ${LOG_DIR}/*.asan exit 1 -fi +fi \ No newline at end of file From e2e782d7248b845310b8fb2125cce4d8cc52baec Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Aug 2024 11:30:56 +0800 Subject: [PATCH 45/59] enh: remove assert --- source/dnode/vnode/src/sma/smaRollup.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 4acdc3e92c..d77b933e28 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1430,7 +1430,7 @@ static void tdFreeRSmaSubmitItems(SArray *pItems, int32_t type) { blockDataDestroy(packData->pDataBlock); } } else { - ASSERTS(0, "unknown type:%d", type); + smaWarn("%s:%d unknown type:%d", __func__, __LINE__, type); } taosArrayClear(pItems); } @@ -1540,14 +1540,13 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA ++nDelete; } } else { - ASSERTS(0, "unknown msg type:%d", inputType); + smaWarn("%s:%d unknown msg type:%d", __func__, __LINE__, inputType); break; } } if (nSubmit > 0 || nDelete > 0) { int32_t size = TARRAY_SIZE(pSubmitArr); - ASSERTS(size > 0, "size is %d", size); int32_t inputType = nSubmit > 0 ? STREAM_INPUT__MERGED_SUBMIT : STREAM_INPUT__REF_DATA_BLOCK; for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { TAOS_CHECK_EXIT(tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, version, inputType, pInfo, type, i)); @@ -1677,7 +1676,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } } } else { - ASSERTS(0, "unknown rsma exec type:%d", (int32_t)type); + smaWarn("%s:%d unknown rsma exec type:%d", __func__, __LINE__, (int32_t)type); code = TSDB_CODE_APP_ERROR; TSDB_CHECK_CODE(code, lino, _exit); } From f54e28bd5e522907d73f9d7cddc3b3082e99db11 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 26 Aug 2024 11:31:26 +0800 Subject: [PATCH 46/59] fix(query):fix mem leak for fill operator --- source/libs/executor/src/filloperator.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index fe9d0d3cf0..63f7667890 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -415,7 +415,8 @@ static int32_t createPrimaryTsExprIfNeeded(SFillOperatorInfo* pInfo, SFillPhysiN int32_t code = createExprFromTargetNode(&pExpr[pExprSupp->numOfExprs], (STargetNode*)pPhyFillNode->pWStartTs); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - taosMemoryFreeClear(pExpr); + pExprSupp->numOfExprs += 1; + pExprSupp->pExprInfo = pExpr; return code; } @@ -447,6 +448,7 @@ int32_t createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFi QUERY_CHECK_CODE(code, lino, _error); pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp; code = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pNoFillSupp->pExprInfo, &pNoFillSupp->numOfExprs); @@ -511,7 +513,6 @@ int32_t createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFi } setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); From 8ac690638fb806c9308a503fb53ad33a2e1032c7 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 26 Aug 2024 04:00:05 +0000 Subject: [PATCH 47/59] fix/TD-31542-remove-assert-sync3 --- source/libs/sync/src/syncCommit.c | 3 - source/libs/sync/src/syncMain.c | 3 + source/libs/sync/src/syncPipeline.c | 163 ++++++++++++++++++++++------ 3 files changed, 131 insertions(+), 38 deletions(-) diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 1c129a0ed1..5054339e8e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -45,9 +45,6 @@ // static inline int64_t syncNodeAbs64(int64_t a, int64_t b) { - ASSERT(a >= 0); - ASSERT(b >= 0); - int64_t c = a > b ? a - b : b - a; return c; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 179bb4d503..d947a488bf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3523,6 +3523,9 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { return 0; } SyncTerm matchTerm = syncLogBufferGetLastMatchTerm(ths->pLogBuf); + if (matchTerm < 0) { + return TSDB_CODE_SYN_INTERNAL_ERROR; + } if (pMsg->currentTerm == matchTerm) { (void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex); } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 22947bdd8e..5e6204dde8 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -46,8 +46,8 @@ int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf) { int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry) { int32_t code = 0; + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); (void)taosThreadMutexLock(&pBuf->mutex); - (void)syncLogBufferValidate(pBuf); SyncIndex index = pEntry->index; if (index - pBuf->startIndex >= pBuf->size) { @@ -102,13 +102,13 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt pBuf->entries[index % pBuf->size] = tmp; pBuf->endIndex = index + 1; - (void)syncLogBufferValidate(pBuf); (void)taosThreadMutexUnlock(&pBuf->mutex); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); return 0; _err: - (void)syncLogBufferValidate(pBuf); (void)taosThreadMutexUnlock(&pBuf->mutex); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); taosMsleep(1); TAOS_RETURN(code); } @@ -134,7 +134,11 @@ int32_t syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI if (prevIndex >= pBuf->startIndex) { pEntry = pBuf->entries[(prevIndex + pBuf->size) % pBuf->size].pItem; - ASSERTS(pEntry != NULL, "no log entry found"); + if (pEntry == NULL) { + sError("vgId:%d, failed to get pre log term since no log entry found", pNode->vgId); + *pSyncTerm = -1; + TAOS_RETURN(TSDB_CODE_SYN_INTERNAL_ERROR); + } prevLogTerm = pEntry->term; *pSyncTerm = prevLogTerm; return 0; @@ -142,9 +146,18 @@ int32_t syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI if (pMgr && pMgr->startIndex <= prevIndex && prevIndex < pMgr->endIndex) { int64_t timeMs = pMgr->states[(prevIndex + pMgr->size) % pMgr->size].timeMs; - ASSERTS(timeMs != 0, "no log entry found"); + if (timeMs == 0) { + sError("vgId:%d, failed to get pre log term since timeMs is 0", pNode->vgId); + *pSyncTerm = -1; + TAOS_RETURN(TSDB_CODE_SYN_INTERNAL_ERROR); + } prevLogTerm = pMgr->states[(prevIndex + pMgr->size) % pMgr->size].term; - ASSERT(prevIndex == 0 || prevLogTerm != 0); + if (!(prevIndex == 0 || prevLogTerm != 0)) { + sError("vgId:%d, failed to get pre log term prevIndex:%" PRId64 ", prevLogTerm:%" PRId64, pNode->vgId, prevIndex, + prevLogTerm); + *pSyncTerm = -1; + TAOS_RETURN(TSDB_CODE_SYN_INTERNAL_ERROR); + } *pSyncTerm = prevLogTerm; return 0; } @@ -289,7 +302,7 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); // validate - (void)syncLogBufferValidate(pBuf); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); return 0; _exit: @@ -307,8 +320,8 @@ int32_t syncLogBufferInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { } int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); (void)taosThreadMutexLock(&pBuf->mutex); - (void)syncLogBufferValidate(pBuf); for (SyncIndex index = pBuf->startIndex; index < pBuf->endIndex; index++) { SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; if (pEntry == NULL) continue; @@ -321,15 +334,19 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { if (code < 0) { sError("vgId:%d, failed to re-initialize sync log buffer since %s.", pNode->vgId, tstrerror(code)); } - (void)syncLogBufferValidate(pBuf); (void)taosThreadMutexUnlock(&pBuf->mutex); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); return code; } FORCE_INLINE SyncTerm syncLogBufferGetLastMatchTermWithoutLock(SSyncLogBuffer* pBuf) { SyncIndex index = pBuf->matchIndex; SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; - ASSERT(pEntry != NULL); + if (pEntry == NULL) { + sError("failed to get last match term since entry is null"); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; + } return pEntry->term; } @@ -348,8 +365,8 @@ bool syncLogBufferIsEmpty(SSyncLogBuffer* pBuf) { } int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm) { + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); (void)taosThreadMutexLock(&pBuf->mutex); - (void)syncLogBufferValidate(pBuf); int32_t code = 0; SyncIndex index = pEntry->index; SyncIndex prevIndex = pEntry->index - 1; @@ -357,6 +374,12 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt SSyncRaftEntry* pExist = NULL; bool inBuf = true; + if (lastMatchTerm < 0) { + sError("vgId:%d, failed to accept, lastMatchTerm:%" PRId64, pNode->vgId, lastMatchTerm); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } + if (index <= pBuf->commitIndex) { sTrace("vgId:%d, already committed. index:%" PRId64 ", term:%" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", @@ -364,7 +387,11 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt pBuf->endIndex); SyncTerm term = -1; code = syncLogReplGetPrevLogTerm(NULL, pNode, index + 1, &term); - ASSERT(pEntry->term >= 0); + if (pEntry->term < 0) { + sError("vgId:%d, failed to accept, pEntry->term:%" PRId64, pNode->vgId, pEntry->term); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } if (term == pEntry->term) { code = 0; } @@ -401,7 +428,12 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt // check current in buffer code = syncLogBufferGetOneEntry(pBuf, pNode, index, &inBuf, &pExist); if (pExist != NULL) { - ASSERT(pEntry->index == pExist->index); + if (pEntry->index != pExist->index) { + sError("vgId:%d, failed to accept, pEntry->index:%" PRId64 ", pExist->index:%" PRId64, pNode->vgId, pEntry->index, + pExist->index); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } if (pEntry->term != pExist->term) { (void)syncLogBufferRollback(pBuf, pNode, index); } else { @@ -411,7 +443,14 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt pBuf->endIndex); SyncTerm existPrevTerm = -1; (void)syncLogReplGetPrevLogTerm(NULL, pNode, index, &existPrevTerm); - ASSERT(pEntry->term == pExist->term && (pEntry->index > pBuf->matchIndex || prevTerm == existPrevTerm)); + if (!(pEntry->term == pExist->term && (pEntry->index > pBuf->matchIndex || prevTerm == existPrevTerm))) { + sError("vgId:%d, failed to accept, pEntry->term:%" PRId64 ", pExist->indexpExist->term:%" PRId64 + ", pEntry->index:%" PRId64 ", pBuf->matchIndex:%" PRId64 ", prevTerm:%" PRId64 + ", existPrevTerm:%" PRId64, + pNode->vgId, pEntry->term, pExist->term, pEntry->index, pBuf->matchIndex, prevTerm, existPrevTerm); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } code = 0; goto _out; } @@ -446,8 +485,8 @@ _out: syncEntryDestroy(pExist); pExist = NULL; } - (void)syncLogBufferValidate(pBuf); (void)taosThreadMutexUnlock(&pBuf->mutex); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); TAOS_RETURN(code); } @@ -479,8 +518,8 @@ int32_t syncLogStorePersist(SSyncLogStore* pLogStore, SSyncNode* pNode, SSyncRaf } int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm, char* str) { + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); (void)taosThreadMutexLock(&pBuf->mutex); - (void)syncLogBufferValidate(pBuf); SSyncLogStore* pLogStore = pNode->pLogStore; int64_t matchIndex = pBuf->matchIndex; @@ -488,7 +527,11 @@ int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* p while (pBuf->matchIndex + 1 < pBuf->endIndex) { int64_t index = pBuf->matchIndex + 1; - ASSERT(index >= 0); + if (index < 0) { + sError("vgId:%d, failed to proceed index:%" PRId64, pNode->vgId, index); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } // try to proceed SSyncLogBufEntry* pBufEntry = &pBuf->entries[index % pBuf->size]; @@ -501,14 +544,37 @@ int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* p goto _out; } - ASSERT(index == pEntry->index); + if (index != pEntry->index) { + sError("vgId:%d, failed to proceed index:%" PRId64 ", pEntry->index:%" PRId64, pNode->vgId, index, pEntry->index); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } // match SSyncRaftEntry* pMatch = pBuf->entries[(pBuf->matchIndex + pBuf->size) % pBuf->size].pItem; - ASSERT(pMatch != NULL); - ASSERT(pMatch->index == pBuf->matchIndex); - ASSERT(pMatch->index + 1 == pEntry->index); - ASSERT(prevLogIndex == pMatch->index); + if (pMatch == NULL) { + sError("vgId:%d, failed to proceed since pMatch is null", pNode->vgId); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } + if (pMatch->index != pBuf->matchIndex) { + sError("vgId:%d, failed to proceed, pMatch->index:%" PRId64 ", pBuf->matchIndex:%" PRId64, pNode->vgId, + pMatch->index, pBuf->matchIndex); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } + if (pMatch->index + 1 != pEntry->index) { + sError("vgId:%d, failed to proceed, pMatch->index:%" PRId64 ", pEntry->index:%" PRId64, pNode->vgId, + pMatch->index, pEntry->index); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } + if (prevLogIndex != pMatch->index) { + sError("vgId:%d, failed to proceed, prevLogIndex:%" PRId64 ", pMatch->index:%" PRId64, pNode->vgId, prevLogIndex, + pMatch->index); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } if (pMatch->term != prevLogTerm) { sInfo( @@ -567,7 +633,12 @@ int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* p // replicate on demand (void)syncNodeReplicateWithoutLock(pNode); - ASSERT(pEntry->index == pBuf->matchIndex); + if (pEntry->index != pBuf->matchIndex) { + sError("vgId:%d, failed to proceed, pEntry->index:%" PRId64 ", pBuf->matchIndex:%" PRId64, pNode->vgId, + pEntry->index, pBuf->matchIndex); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _out; + } // update my match index matchIndex = pBuf->matchIndex; @@ -579,8 +650,8 @@ _out: if (pMatchTerm) { *pMatchTerm = pBuf->entries[(matchIndex + pBuf->size) % pBuf->size].pItem->term; } - (void)syncLogBufferValidate(pBuf); (void)taosThreadMutexUnlock(&pBuf->mutex); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); return matchIndex; } @@ -643,17 +714,36 @@ _exit: } int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf) { - ASSERT(pBuf->startIndex <= pBuf->matchIndex); - ASSERT(pBuf->commitIndex <= pBuf->matchIndex); - ASSERT(pBuf->matchIndex < pBuf->endIndex); - ASSERT(pBuf->endIndex - pBuf->startIndex <= pBuf->size); - ASSERT(pBuf->entries[(pBuf->matchIndex + pBuf->size) % pBuf->size].pItem); + if (pBuf->startIndex > pBuf->matchIndex) { + sError("failed to validate, pBuf->startIndex:%" PRId64 ", pBuf->matchIndex:%" PRId64, pBuf->startIndex, + pBuf->matchIndex); + return TSDB_CODE_SYN_INTERNAL_ERROR; + } + if (pBuf->commitIndex > pBuf->matchIndex) { + sError("failed to validate, pBuf->commitIndex:%" PRId64 ", pBuf->matchIndex:%" PRId64, pBuf->commitIndex, + pBuf->matchIndex); + return TSDB_CODE_SYN_INTERNAL_ERROR; + } + if (pBuf->matchIndex >= pBuf->endIndex) { + sError("failed to validate, pBuf->matchIndex:%" PRId64 ", pBuf->endIndex:%" PRId64, pBuf->matchIndex, + pBuf->endIndex); + return TSDB_CODE_SYN_INTERNAL_ERROR; + } + if (pBuf->endIndex - pBuf->startIndex > pBuf->size) { + sError("failed to validate, pBuf->endIndex:%" PRId64 ", pBuf->startIndex:%" PRId64 ", pBuf->size:%" PRId64, + pBuf->endIndex, pBuf->startIndex, pBuf->size); + return TSDB_CODE_SYN_INTERNAL_ERROR; + } + if (pBuf->entries[(pBuf->matchIndex + pBuf->size) % pBuf->size].pItem == NULL) { + sError("failed to validate since pItem is null"); + return TSDB_CODE_SYN_INTERNAL_ERROR; + } return 0; } int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t commitIndex) { + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); (void)taosThreadMutexLock(&pBuf->mutex); - (void)syncLogBufferValidate(pBuf); SSyncLogStore* pLogStore = pNode->pLogStore; SSyncFSM* pFsm = pNode->pFsm; @@ -778,15 +868,18 @@ _out: syncEntryDestroy(pNextEntry); pNextEntry = NULL; } - (void)syncLogBufferValidate(pBuf); (void)taosThreadMutexUnlock(&pBuf->mutex); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); TAOS_RETURN(code); } void syncLogReplReset(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) return; - ASSERT(pMgr->startIndex >= 0); + if (pMgr->startIndex < 0) { + sError("failed to reset, pMgr->startIndex:%" PRId64, pMgr->startIndex); + return; + } for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) { (void)memset(&pMgr->states[index % pMgr->size], 0, sizeof(pMgr->states[0])); } @@ -1285,13 +1378,13 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex } if (pBuf->endIndex != toIndex) return TSDB_CODE_SYN_INTERNAL_ERROR; - (void)syncLogBufferValidate(pBuf); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); return 0; } int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); (void)taosThreadMutexLock(&pBuf->mutex); - (void)syncLogBufferValidate(pBuf); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); if (lastVer != pBuf->matchIndex) return TSDB_CODE_SYN_INTERNAL_ERROR; SyncIndex index = pBuf->endIndex - 1; @@ -1308,8 +1401,8 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; syncLogReplReset(pMgr); } - (void)syncLogBufferValidate(pBuf); (void)taosThreadMutexUnlock(&pBuf->mutex); + TAOS_CHECK_RETURN(syncLogBufferValidate(pBuf)); return 0; } From 8ca7b967a60c635a02cf711470618d179257e05c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 26 Aug 2024 13:49:30 +0800 Subject: [PATCH 48/59] test:commit the failed case in cases-task --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3553f0d913..c51fa6fe2a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -310,7 +310,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py +,,n,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py From 33c284a1d98a28b1751fac19908911a5641537d7 Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Mon, 26 Aug 2024 14:23:37 +0800 Subject: [PATCH 49/59] Optimising the duration and keep of the show create database --- source/libs/command/src/command.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 1882db1dda..be7a254f38 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -344,23 +344,27 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) { return TSDB_CACHE_MODEL_NONE_STR; } -static void formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { +static int32_t formatDurationOrKeep(char** buffer, int32_t timeInMinutes) { int len = 0; if (timeInMinutes % 1440 == 0) { int days = timeInMinutes / 1440; len = snprintf(NULL, 0, "%dd", days); *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); + if(*buffer == NULL) return terrno; sprintf(*buffer, "%dd", days); } else if (timeInMinutes % 60 == 0) { int hours = timeInMinutes / 60; len = snprintf(NULL, 0, "%dh", hours); *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); + if(*buffer == NULL) return terrno; sprintf(*buffer, "%dh", hours); } else { len = snprintf(NULL, 0, "%dm", timeInMinutes); *buffer = (char*)taosMemoryCalloc(len + 1, sizeof(char)); + if(*buffer == NULL) return terrno; sprintf(*buffer, "%dm", timeInMinutes); } + return TSDB_CODE_SUCCESS; } static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { @@ -401,13 +405,24 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; } char* durationStr = NULL; - (void)formatDurationOrKeep(&durationStr, pCfg->daysPerFile); char* keep0Str = NULL; - (void)formatDurationOrKeep(&keep0Str, pCfg->daysToKeep0); char* keep1Str = NULL; - (void)formatDurationOrKeep(&keep1Str, pCfg->daysToKeep1); char* keep2Str = NULL; - (void)formatDurationOrKeep(&keep2Str, pCfg->daysToKeep2); + int32_t codeDuration = formatDurationOrKeep(&durationStr, pCfg->daysPerFile); + int32_t codeKeep0 = formatDurationOrKeep(&keep0Str, pCfg->daysToKeep0); + int32_t codeKeep1 = formatDurationOrKeep(&keep1Str, pCfg->daysToKeep1); + int32_t codeKeep2 = formatDurationOrKeep(&keep2Str, pCfg->daysToKeep2); + if(codeDuration != TSDB_CODE_SUCCESS || codeKeep0 != TSDB_CODE_SUCCESS || codeKeep1 != TSDB_CODE_SUCCESS || codeKeep2 != TSDB_CODE_SUCCESS) { + int32_t firstErrorCode = codeDuration != TSDB_CODE_SUCCESS ? codeDuration : + codeKeep0 != TSDB_CODE_SUCCESS ? codeKeep0 : + codeKeep1 != TSDB_CODE_SUCCESS ? codeKeep1 : codeKeep2; + taosMemoryFree(pRetentions); + taosMemoryFree(durationStr); + taosMemoryFree(keep0Str); + taosMemoryFree(keep1Str); + taosMemoryFree(keep2Str); + return firstErrorCode; + } if (IS_SYS_DBNAME(dbName)) { len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); } else { From 0ad4ffd0ca4c7d741fd28947fa8c01e7d8ad6d7c Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Fri, 23 Aug 2024 16:39:18 +0800 Subject: [PATCH 50/59] docs:[TD-31673] Update description of timetruncate function to clarify week truncation behavior based on Unix epoch starting on Thursday. --- docs/en/14-reference/03-taos-sql/10-function.md | 1 + docs/zh/14-reference/03-taos-sql/10-function.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index e130442279..72ca878ce8 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -656,6 +656,7 @@ use_current_timezone: { For example, if the time zone configured by the Client is UTC + 0800, TIMETRUNCATE ('2020-01-01 23:00:00', 1d, 0) returns the result of '2020-01-01 08:00:00'. When using TIMETRUNCATE ('2020-01-01 23:00:00', 1d, 1), the result is 2020-01-01 00:00:00 '. When use_current_timezone is not specified, use_current_timezone defaults to 1. +- When truncating a time value to the week (1w), weeks are determined using the Unix epoch (1970-01-01T00:00:00Z UTC). The Unix epoch was on a Thursday, so all calculated weeks begin on Thursday. #### TIMEZONE diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index ca6ba378ca..ac9311184e 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -656,7 +656,8 @@ use_current_timezone: { 例如客户端所配置时区为 UTC+0800, 则 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 0) 返回结果为东八区时间 '2020-01-01 08:00:00'。 而使用 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 1) 时,返回结果为东八区时间 '2020-01-01 00:00:00'。 当不指定 use_current_timezone 时,use_current_timezone 默认值为 1 。 - +- 当将时间值截断到一周(1w)时,timetruncate 的计算是基于 Unix 时间戳(1970年1月1日00:00:00 UTC)进行的。Unix 时间戳始于星期四, + 因此所有截断后的日期都是星期四。 #### TIMEZONE From e786901316c462374cf4fb6a0909abf8c5bba368 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 26 Aug 2024 14:49:02 +0800 Subject: [PATCH 51/59] fix merge error --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7bc95f0a74..f908ef5984 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1451,7 +1451,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); tDeleteSchemaWrapper(mtInfo.schema); return code; - + } code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**)&pInfo->dataReader, NULL, NULL); From 7322f68240603edbfc840ecb4e2d7f098b5f3dca Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 26 Aug 2024 06:51:09 +0000 Subject: [PATCH 52/59] fix/TD-31542-fix-case --- source/libs/sync/src/syncMain.c | 2 +- source/libs/sync/src/syncPipeline.c | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 2e76fcdf91..c8da543927 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -730,7 +730,7 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho sError("vgId:%d, failed get snapshot config index, configIndexCount:%d", pSyncNode->vgId, pSyncNode->raftCfg.configIndexCount); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; + return -2; } SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0]; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 5687321579..f3f6148ae5 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -611,17 +611,18 @@ int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTe int32_t code = 0, lino = 0; bool retry = false; do { - SRpcMsg rpcMsg = {.code = applyCode}; - TAOS_CHECK_EXIT(syncEntry2OriginalRpc(pEntry, &rpcMsg)); - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pEntry->index; cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); - if (cbMeta.lastConfigIndex < 0) { + if (cbMeta.lastConfigIndex < -1) { code = TSDB_CODE_SYN_INTERNAL_ERROR; if (terrno != 0) code = terrno; return code; } + + SRpcMsg rpcMsg = {.code = applyCode}; + TAOS_CHECK_EXIT(syncEntry2OriginalRpc(pEntry, &rpcMsg)); + + cbMeta.index = pEntry->index; cbMeta.isWeak = pEntry->isWeak; cbMeta.code = applyCode; cbMeta.state = role; From a706c4d49de81b696a757a33f73b935420565936 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 26 Aug 2024 14:58:12 +0800 Subject: [PATCH 53/59] fix(query): check error code for tsort --- source/libs/executor/src/scanoperator.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7a729b4611..7cad03db50 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -5749,7 +5749,11 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* while (1) { pTupleHandle = NULL; int32_t code = tsortNextTuple(pHandle, &pTupleHandle); - if (pTupleHandle == NULL || code != 0) { + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + T_LONG_JMP(pOperator->pTaskInfo->env, code); + } + if (pTupleHandle == NULL) { break; } From a944dd0d8b7423c0f30474d27f5370a9b4c97e3d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 26 Aug 2024 15:16:07 +0800 Subject: [PATCH 54/59] fix: remove asserts and memory issue --- source/libs/executor/src/scanoperator.c | 2 +- source/util/src/talgo.c | 4 +--- source/util/src/tcache.c | 20 ++++++++++---------- source/util/src/tscalablebf.c | 1 - 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8a0cd143ed..6a775dfc79 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4752,7 +4752,7 @@ int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* p QUERY_CHECK_NULL(pInfo->filterCtx.colHash, code, lino, _error, terrno); pInfo->filterCtx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo)); - QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno); + QUERY_CHECK_NULL(pInfo->filterCtx.cInfoList, code, lino, _error, terrno); if (pInfo->pTagCond != NULL) { nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx); diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index d7cd0348f4..9c241516ca 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -39,8 +39,6 @@ static void median(void *src, int64_t size, int64_t s, int64_t e, const void *pa doswap(elePtrAt(src, size, s), elePtrAt(src, size, e), size, buf); } - ASSERT(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && - comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); } static void tInsertSort(void *src, int64_t size, int32_t s, int32_t e, const void *param, __ext_compar_fn_t comparFn, @@ -323,7 +321,7 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size } else if (flags == TD_LT) { return (c > 0) ? p : (midx > 0 ? p - size : NULL); } else { - ASSERT(0); + uError("Invalid bsearch flags:%d", flags); return NULL; } } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index c150ca3fbd..b7ca99d26a 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -262,7 +262,7 @@ static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { pNode->pNext = pEntry->next; pEntry->next = pNode; pEntry->num += 1; - ASSERT((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); + //A S S E R T((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); } static void removeNodeInEntryList(SCacheEntry *pe, SCacheNode *prev, SCacheNode *pNode) { @@ -274,7 +274,7 @@ static void removeNodeInEntryList(SCacheEntry *pe, SCacheNode *prev, SCacheNode pNode->pNext = NULL; pe->num -= 1; - ASSERT((pe->next && pe->num > 0) || (NULL == pe->next && pe->num == 0)); + //A S S E R T((pe->next && pe->num > 0) || (NULL == pe->next && pe->num == 0)); } static FORCE_INLINE SCacheEntry *doFindEntry(SCacheObj *pCacheObj, const void *key, size_t keyLen) { @@ -499,7 +499,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { uDebug("cache:%s, data: %p acquired by data in cache, refcnt:%d", pCacheObj->name, ptNode->data, ref); // the data if referenced by at least one object, so the reference count must be greater than the value of 2. - ASSERT(ref >= 2); + //A S S E R T(ref >= 2); return data; } @@ -574,19 +574,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (ref == 1) { // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be // destroyed by refresh worker if decrease ref count before removing it from linked-list. - ASSERT(pNode->pTNodeHeader->pData == pNode); + //A S S E R T(pNode->pTNodeHeader->pData == pNode); __trashcan_wr_lock(pCacheObj); (void)doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); __trashcan_unlock(pCacheObj); ref = T_REF_DEC(pNode); - ASSERT(ref == 0); + //A S S E R T(ref == 0); doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); } else { ref = T_REF_DEC(pNode); - ASSERT(ref >= 0); + //A S S E R T(ref >= 0); } } else { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread @@ -608,7 +608,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { "others already, prev must in trashcan", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data, T_REF_VAL_GET(pNode)); - ASSERT(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); + //A S S E R T(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); } else { removeNodeInEntryList(pe, prev, p); uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, @@ -668,7 +668,7 @@ void doTraverseElems(SCacheObj *pCacheObj, bool (*fp)(void *param, SCacheNode *p } else { *pPre = next; pEntry->num -= 1; - ASSERT((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); + //A S S E R T((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); (void)atomic_sub_fetch_ptr(&pCacheObj->numOfElems, 1); pNode = next; @@ -734,7 +734,7 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { if (pNode->inTrashcan) { /* node is already in trash */ - ASSERT(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); + //A S S E R T(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; } @@ -780,7 +780,7 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { STrashElem *pElem = pCacheObj->pTrash; while (pElem) { T_REF_VAL_CHECK(pElem->pData); - ASSERT(pElem->next != pElem && pElem->prev != pElem); + //A S S E R T(pElem->next != pElem && pElem->prev != pElem); if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 407223e937..5a5cba5f0a 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -118,7 +118,6 @@ int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int3 } SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); - ASSERT(pNormalBf); if (tBloomFilterIsFull(pNormalBf)) { code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); From 1f4b487f36f9e6e6d8b8cb3c0cebcf071d0d9d5f Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Aug 2024 15:23:18 +0800 Subject: [PATCH 55/59] enh: remove asserts --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4d486518ee..73b9786f99 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2329,7 +2329,7 @@ int8_t taosGranted(int8_t type) { case TSDB_GRANT_VIEW: return atomic_load_8(&tsGrant) & GRANT_FLAG_VIEW; default: - ASSERTS(0, "undefined grant type:%" PRIi8, type); + uWarn("undefined grant type:%" PRIi8, type); break; } return 0; From d3b3fdb498b5960e9d42b6ed97975fccda04f63f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 26 Aug 2024 15:54:47 +0800 Subject: [PATCH 56/59] array/retval: fix array return values --- source/common/src/cos.c | 7 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 87 ++++++++++++++++++------- source/libs/wal/src/walMeta.c | 15 ++++- source/util/src/tlrucache.c | 21 ++++-- 4 files changed, 98 insertions(+), 32 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 6e9c7dd50d..3b5fca0fea 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -1024,7 +1024,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w } static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size, - int8_t epIndex) { + int8_t epIndex) { int32_t code = 0; int32_t lmtime = 0; const char *filename = 0; @@ -1136,7 +1136,10 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int const S3ListBucketContent *content = &(contents[i]); // printf("%-50s", content->key); char *object_key = strdup(content->key); - (void)taosArrayPush(data->objectArray, &object_key); + if (!taosArrayPush(data->objectArray, &object_key)) { + taosMemoryFree(object_key); + return S3StatusOutOfMemory; + } } data->keyCount += contentsCount; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ae57576279..6d7a77688d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -553,7 +553,7 @@ static int32_t reallocVarDataVal(SValue *pValue) { static int32_t reallocVarData(SColVal *pColVal) { return reallocVarDataVal(&pColVal->value); } // realloc pk data and col data. -static int32_t tsdbCacheReallocSLastCol(SLastCol *pCol, size_t* pCharge) { +static int32_t tsdbCacheReallocSLastCol(SLastCol *pCol, size_t *pCharge) { int32_t code = TSDB_CODE_SUCCESS, lino = 0; size_t charge = sizeof(SLastCol); @@ -587,8 +587,8 @@ _exit: TAOS_RETURN(code); } -void tsdbCacheFreeSLastColItem(void* pItem) { - SLastCol* pCol = (SLastCol*)pItem; +void tsdbCacheFreeSLastColItem(void *pItem) { + SLastCol *pCol = (SLastCol *)pItem; for (int i = 0; i < pCol->rowKey.numOfPKs; i++) { if (IS_VAR_DATA_TYPE(pCol->rowKey.pks[i].type)) { taosMemoryFree(pCol->rowKey.pks[i].pData); @@ -1162,8 +1162,13 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray } else { if (!remainCols) { remainCols = taosArrayInit(num_keys * 2, sizeof(SIdxKey)); + if (!remainCols) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } + } + if (!taosArrayPush(remainCols, &(SIdxKey){i, *key})) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - (void)taosArrayPush(remainCols, &(SIdxKey){i, *key}); } } @@ -1309,14 +1314,20 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 int32_t iCol = 0; for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; - (void)taosArrayPush(ctxArray, &updateCtx); + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } if (!COL_VAL_IS_VALUE(pColVal)) { - (void)tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0); + if (!tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } continue; } updateCtx.lflag = LFLAG_LAST; - (void)taosArrayPush(ctxArray, &updateCtx); + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } } tsdbRowClose(&iter); @@ -1340,7 +1351,9 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 if (COL_VAL_IS_VALUE(&colVal)) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal}; - (void)taosArrayPush(ctxArray, &updateCtx); + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } (void)tSimpleHashIterateRemove(iColHash, &iCol, sizeof(iCol), &pIte, &iter); } } @@ -1358,7 +1371,7 @@ _exit: } int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) { - int32_t code = 0; + int32_t code = 0, lino = 0; TSDBROW lRow = tsdbRowFromBlockData(pBlockData, pBlockData->nRow - 1); @@ -1380,7 +1393,9 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo .tsdbRowKey = tsdbRowKey, .colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = lRow.pBlockData->aTSKEY[lRow.iRow]}))}; - (void)taosArrayPush(ctxArray, &updateCtx); + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } } TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0); @@ -1401,7 +1416,9 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo tColDataGetValue(pColData, tRow.iRow, &colVal); SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal}; - (void)taosArrayPush(ctxArray, &updateCtx); + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } break; } } @@ -1412,7 +1429,9 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo (void)tsdbRowIterOpen(&iter, &lRow, pTSchema); for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; - (void)taosArrayPush(ctxArray, &updateCtx); + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } } tsdbRowClose(&iter); @@ -1481,7 +1500,9 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } } - (void)taosArrayPush(lastTmpIndexArray, &(i)); + if (!taosArrayPush(lastTmpIndexArray, &(i))) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } lastColIds[lastIndex] = idxKey->key.cid; lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx]; lastIndex++; @@ -1492,7 +1513,9 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } } - (void)taosArrayPush(lastrowTmpIndexArray, &(i)); + if (!taosArrayPush(lastrowTmpIndexArray, &(i))) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } lastrowColIds[lastrowIndex] = idxKey->key.cid; lastrowSlotIds[lastrowIndex] = pr->pSlotIds[idxKey->idx]; lastrowIndex++; @@ -1819,7 +1842,7 @@ _exit: } int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { - int32_t code = 0; + int32_t code = 0, lino = 0; // fetch schema STSchema *pTSchema = NULL; int sver = -1; @@ -1850,7 +1873,9 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE if (!remainCols) { remainCols = taosArrayInit(numCols * 2, sizeof(SLastKey)); } - (void)taosArrayPush(remainCols, &lastKey); + if (!taosArrayPush(remainCols, &lastKey)) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } } } } @@ -2034,10 +2059,12 @@ static int32_t getTableDelDataFromTbData(STbData *pTbData, SArray *aDelData) { SDelData *pDelData = pTbData ? pTbData->pHead : NULL; for (; pDelData; pDelData = pDelData->pNext) { - (void)taosArrayPush(aDelData, pDelData); + if (!taosArrayPush(aDelData, pDelData)) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } - return code; + TAOS_RETURN(code); } static void freeTableInfoFunc(void *param) { @@ -2169,7 +2196,9 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);*/ SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; - (void)taosArrayPush(pInfo->pTombData, &delData); + if (!taosArrayPush(pInfo->pTombData, &delData)) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -2387,7 +2416,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie SBrinBlk *pBrinBlk = &pBlkArray->data[i]; if (state->suid >= pBrinBlk->minTbid.suid && state->suid <= pBrinBlk->maxTbid.suid) { if (state->uid >= pBrinBlk->minTbid.uid && state->uid <= pBrinBlk->maxTbid.uid) { - (void)taosArrayPush(state->pIndexList, pBrinBlk); + if (!taosArrayPush(state->pIndexList, pBrinBlk)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); + } } } else if (state->suid > pBrinBlk->maxTbid.suid || (state->suid == pBrinBlk->maxTbid.suid && state->uid > pBrinBlk->maxTbid.uid)) { @@ -3004,7 +3035,9 @@ static int32_t initLastColArrayPartial(STSchema *pTSchema, SArray **ppColArray, int16_t slotId = slotIds[i]; SLastCol col = {.rowKey.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[slotId].colId, pTSchema->columns[slotId].type)}; - (void)taosArrayPush(pColArray, &col); + if (!taosArrayPush(pColArray, &col)) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } *ppColArray = pColArray; @@ -3058,7 +3091,11 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC } for (int i = 0; i < nCols; ++i) { - (void)taosArrayPush(aColArray, &aCols[i]); + if (!taosArrayPush(aColArray, &aCols[i])) { + taosArrayDestroy(pColArray); + + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } STsdbRowKey lastRowKey = {.key.ts = TSKEY_MAX}; @@ -3227,7 +3264,11 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, } for (int i = 0; i < nCols; ++i) { - (void)taosArrayPush(aColArray, &aCols[i]); + if (!taosArrayPush(aColArray, &aCols[i])) { + taosArrayDestroy(pColArray); + + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } // inverse iterator diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index d0c7dea451..0eb011113a 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -417,7 +417,13 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo fileInfo; (void)memset(&fileInfo, -1, sizeof(SWalFileInfo)); (void)sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer); - (void)taosArrayPush(actualLog, &fileInfo); + if (!taosArrayPush(actualLog, &fileInfo)) { + regfree(&logRegPattern); + regfree(&idxRegPattern); + (void)taosCloseDir(&pDir); + + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -730,9 +736,12 @@ int32_t walRollFileInfo(SWal* pWal) { pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; pNewInfo->syncedOffset = 0; - (void)taosArrayPush(pArray, pNewInfo); - taosMemoryFree(pNewInfo); + if (!taosArrayPush(pArray, pNewInfo)) { + taosMemoryFree(pNewInfo); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } + taosMemoryFree(pNewInfo); TAOS_RETURN(TSDB_CODE_SUCCESS); } diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index ddcf4aebfa..eaca2f73a4 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -310,7 +310,9 @@ static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArr TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); shard->usage -= old->totalCharge; - (void)taosArrayPush(deleted, &old); + if (!taosArrayPush(deleted, &old)) { + // ignore this round's eviting + } } } @@ -382,7 +384,11 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * if (shard->usage + e->totalCharge > shard->capacity && (shard->strictCapacity || handle == NULL)) { TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); if (handle == NULL) { - (void)taosArrayPush(lastReferenceList, &e); + if (!taosArrayPush(lastReferenceList, &e)) { + (void)taosThreadMutexUnlock(&shard->mutex); + taosLRUEntryFree(e); + return status; + } } else { if (freeOnFail) { taosMemoryFree(e); @@ -403,7 +409,11 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * taosLRUCacheShardLRURemove(shard, old); shard->usage -= old->totalCharge; - (void)taosArrayPush(lastReferenceList, &old); + if (!taosArrayPush(lastReferenceList, &old)) { + (void)taosThreadMutexUnlock(&shard->mutex); + taosLRUEntryFree(old); + return status; + } } } if (handle == NULL) { @@ -519,7 +529,10 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) { TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); shard->usage -= old->totalCharge; - (void)taosArrayPush(lastReferenceList, &old); + if (!taosArrayPush(lastReferenceList, &old)) { + taosLRUEntryFree(old); + return; + } } (void)taosThreadMutexUnlock(&shard->mutex); From 7adddf0fb636d46c920eb700ff07cc8fe0935edd Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 26 Aug 2024 16:48:12 +0800 Subject: [PATCH 57/59] fix(lru/init): free cache's shards if shard init failed --- source/util/src/tlrucache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index ddcf4aebfa..242e722083 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -674,6 +674,7 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo for (int i = 0; i < numShards; ++i) { if (TSDB_CODE_SUCCESS != taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits)) { + taosMemoryFree(cache->shards); taosMemoryFree(cache); return NULL; } From bb039e9b2db1691a0001b1aef7cfe3174baebc79 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 26 Aug 2024 16:49:59 +0800 Subject: [PATCH 58/59] update JDBC example dir --- packaging/tools/makeclient.sh | 15 +++++++------- packaging/tools/makepkg.sh | 39 ++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index f46a9adeff..d67d436fa7 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -182,15 +182,16 @@ if [[ $productName == "TDengine" ]] && [ "$verMode" != "cloud" ]; then # Copy example code mkdir -p ${install_dir}/examples examples_dir="${top_dir}/examples" + new_example_dir="${top_dir}/docs/examples" cp -r ${examples_dir}/c ${install_dir}/examples if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - cp -r ${examples_dir}/JDBC ${install_dir}/examples - cp -r ${examples_dir}/matlab ${install_dir}/examples - cp -r ${examples_dir}/python ${install_dir}/examples - cp -r ${examples_dir}/R ${install_dir}/examples - cp -r ${examples_dir}/go ${install_dir}/examples - cp -r ${examples_dir}/nodejs ${install_dir}/examples - cp -r ${examples_dir}/C# ${install_dir}/examples + cp -r ${new_example_dir}/JDBC ${install_dir}/examples ||: + cp -r ${examples_dir}/matlab ${install_dir}/examples ||: + cp -r ${examples_dir}/python ${install_dir}/examples ||: + cp -r ${examples_dir}/R ${install_dir}/examples ||: + cp -r ${examples_dir}/go ${install_dir}/examples ||: + cp -r ${examples_dir}/nodejs ${install_dir}/examples ||: + cp -r ${examples_dir}/C# ${install_dir}/examples ||: mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index b614170fd8..798d73d0f3 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -284,34 +284,35 @@ if [[ $dbName == "taos" ]]; then # Copy example code mkdir -p ${install_dir}/examples examples_dir="${top_dir}/examples" + new_example_dir="${top_dir}/docs/examples" cp -r ${examples_dir}/c ${install_dir}/examples if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then - rm -rf ${examples_dir}/JDBC/connectionPools/target + if [ -d ${new_example_dir}/JDBC/connectionPools/target ]; then + rm -rf ${new_example_dir}/JDBC/connectionPools/target fi - if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then - rm -rf ${examples_dir}/JDBC/JDBCDemo/target + if [ -d ${new_example_dir}/JDBC/JDBCDemo/target ]; then + rm -rf ${new_example_dir}/JDBC/JDBCDemo/target fi - if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then - rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target + if [ -d ${new_example_dir}/JDBC/mybatisplus-demo/target ]; then + rm -rf ${new_example_dir}/JDBC/mybatisplus-demo/target fi - if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then - rm -rf ${examples_dir}/JDBC/springbootdemo/target + if [ -d ${new_example_dir}/JDBC/springbootdemo/target ]; then + rm -rf ${new_example_dir}/JDBC/springbootdemo/target fi - if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then - rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target + if [ -d ${new_example_dir}/JDBC/SpringJdbcTemplate/target ]; then + rm -rf ${new_example_dir}/JDBC/SpringJdbcTemplate/target fi - if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then - rm -rf ${examples_dir}/JDBC/taosdemo/target + if [ -d ${new_example_dir}/JDBC/taosdemo/target ]; then + rm -rf ${new_example_dir}/JDBC/taosdemo/target fi - cp -r ${examples_dir}/JDBC ${install_dir}/examples - cp -r ${examples_dir}/matlab ${install_dir}/examples - cp -r ${examples_dir}/python ${install_dir}/examples - cp -r ${examples_dir}/R ${install_dir}/examples - cp -r ${examples_dir}/go ${install_dir}/examples - cp -r ${examples_dir}/nodejs ${install_dir}/examples - cp -r ${examples_dir}/C# ${install_dir}/examples + cp -r ${new_example_dir}/JDBC ${install_dir}/examples ||: + cp -r ${examples_dir}/matlab ${install_dir}/examples ||: + cp -r ${examples_dir}/python ${install_dir}/examples ||: + cp -r ${examples_dir}/R ${install_dir}/examples ||: + cp -r ${examples_dir}/go ${install_dir}/examples ||: + cp -r ${examples_dir}/nodejs ${install_dir}/examples ||: + cp -r ${examples_dir}/C# ${install_dir}/examples ||: mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi From 19f461c06b853183e15b601cab3d0ce9a8eb6393 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 26 Aug 2024 17:40:59 +0800 Subject: [PATCH 59/59] fix simple hash put retval --- source/dnode/vnode/src/tsdb/tsdbCache.c | 29 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6d7a77688d..e6c52d8bed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1319,7 +1319,7 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } if (!COL_VAL_IS_VALUE(pColVal)) { - if (!tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { + if (tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } continue; @@ -1464,7 +1464,9 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr tsdbCacheUpdateLastColToNone(pLastCol, TSDB_LAST_CACHE_NO_CACHE); SLastKey *key = &(SLastKey){.lflag = ltype, .uid = uid, .cid = PRIMARYKEY_TIMESTAMP_COL_ID}; - (void)taosArrayInsert(remainCols, 0, &(SIdxKey){0, *key}); + if (!taosArrayInsert(remainCols, 0, &(SIdxKey){0, *key})) { + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } int num_keys = TARRAY_SIZE(remainCols); @@ -1530,16 +1532,20 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr if (lastTmpIndexArray != NULL) { TAOS_CHECK_EXIT(mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds)); for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { - (void)taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), - taosArrayGet(lastTmpColArray, i)); + if (!taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), + taosArrayGet(lastTmpColArray, i))) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } } } if (lastrowTmpIndexArray != NULL) { TAOS_CHECK_EXIT(mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds)); for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { - (void)taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i), - taosArrayGet(lastrowTmpColArray, i)); + if (!taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i), + taosArrayGet(lastrowTmpColArray, i))) { + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -2075,6 +2081,9 @@ static void freeTableInfoFunc(void *param) { static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) { if (!pReader->pTableMap) { pReader->pTableMap = tSimpleHashInit(pReader->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (!pReader->pTableMap) { + return NULL; + } tSimpleHashSetFreeFp(pReader->pTableMap, freeTableInfoFunc); } @@ -2084,7 +2093,9 @@ static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) if (!ppInfo) { pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo)); if (pInfo) { - (void)tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + if (tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES)) { + return NULL; + } } return pInfo; @@ -2988,7 +2999,9 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI TSDB_CHECK_NULL(pInfo->pTombData, code, lino, _err, TSDB_CODE_OUT_OF_MEMORY); } - (void)taosArrayAddAll(pInfo->pTombData, pIter->pMemDelData); + if (!taosArrayAddAll(pInfo->pTombData, pIter->pMemDelData)) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); + } size_t delSize = TARRAY_SIZE(pInfo->pTombData); if (delSize > 0) {