From 9ddf1178b9a3c426c2da47718de192bab742c34a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 1 Sep 2023 15:45:31 +0800 Subject: [PATCH 01/96] add websocket connection --- tests/system-test/test.py | 96 +++++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 34 deletions(-) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 7082bb0f22..0af047c818 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -35,6 +35,7 @@ from util.taosadapter import * import taos import taosrest +import taosws def checkRunTimeError(): import win32gui @@ -105,12 +106,13 @@ if __name__ == "__main__": queryPolicy = 1 createDnodeNums = 1 restful = False + websocket = False replicaVar = 1 asan = False independentMnode = True previousCluster = False - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:aP', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode','previous']) + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -131,6 +133,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-Q set queryPolicy in one dnode') tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') tdLog.printNoPrefix('-R restful realization form') + tdLog.printNoPrefix('-W websocket connection') tdLog.printNoPrefix('-D taosadapter update cfg dict ') tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-i independentMnode Mnode') @@ -203,6 +206,9 @@ if __name__ == "__main__": if key in ['-R', '--restful']: restful = True + + if key in ['-W', '--websocket']: + websocket = True if key in ['-a', '--asan']: asan = True @@ -224,7 +230,7 @@ if __name__ == "__main__": # do exeCmd command # if not execCmd == "": - if restful: + if restful or websocket: tAdapter.init(deployPath) else: tdDnodes.init(deployPath) @@ -263,7 +269,7 @@ if __name__ == "__main__": if valgrind: time.sleep(2) - if restful: + if restful or websocket: toBeKilled = "taosadapter" # killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled @@ -358,7 +364,7 @@ if __name__ == "__main__": tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) - if restful: + if restful or websocket: tAdapter.deploy(adapter_cfg_dict) tAdapter.start() @@ -366,6 +372,8 @@ if __name__ == "__main__": queryPolicy=int(queryPolicy) if restful: conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) @@ -395,14 +403,16 @@ if __name__ == "__main__": tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) - if restful: + if restful or websocket: tAdapter.deploy(adapter_cfg_dict) tAdapter.start() - if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) - else: + if restful: conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") + else: + conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) # tdLog.info(tdDnodes.getSimCfgPath(),host) if createDnodeNums == 1: createDnodeNums=dnodeNums @@ -419,6 +429,8 @@ if __name__ == "__main__": queryPolicy=int(queryPolicy) if restful: conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) @@ -438,10 +450,12 @@ if __name__ == "__main__": if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True: conn = None else: - if not restful: - conn = taos.connect(host="%s"%(host), config=tdDnodes.sim.getCfgDir()) + if restful: + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: - conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) if testCluster: tdLog.info("Procedures for testing cluster") @@ -451,10 +465,12 @@ if __name__ == "__main__": tdCases.runOneCluster(fileName) else: tdLog.info("Procedures for testing self-deployment") - if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + if restful: + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: - conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllWindows(conn) @@ -470,10 +486,12 @@ if __name__ == "__main__": tdDnodes.stopAll() tdDnodes.start(1) time.sleep(1) - if not restful: - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) - else: + if restful: conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") + else: + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) tdLog.info("Procedures for tdengine deployed in %s" % (host)) tdLog.info("query test after taosd restart") tdCases.runOneWindows(conn, sp[0] + "_" + "restart.py", replicaVar) @@ -505,7 +523,7 @@ if __name__ == "__main__": except: pass - if restful: + if restful or websocket: tAdapter.init(deployPath, masterIp) tAdapter.stop(force_kill=True) @@ -515,16 +533,18 @@ if __name__ == "__main__": tdDnodes.start(1) tdCases.logSql(logSql) - if restful: + if restful or websocket: tAdapter.deploy(adapter_cfg_dict) tAdapter.start() if queryPolicy != 1: queryPolicy=int(queryPolicy) - if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) - else: + if restful: conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") + else: + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) # tdSql.init(conn.cursor()) # tdSql.execute("create qnode on dnode 1") # tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) @@ -566,15 +586,17 @@ if __name__ == "__main__": tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) - if restful: + if restful or websocket: tAdapter.deploy(adapter_cfg_dict) tAdapter.start() # create taos connect - if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + if restful: + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: - conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) print(tdDnodes.getSimCfgPath(),host) if createDnodeNums == 1: createDnodeNums=dnodeNums @@ -594,8 +616,10 @@ if __name__ == "__main__": queryPolicy=int(queryPolicy) if restful: conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) cursor = conn.cursor() cursor.execute("create qnode on dnode 1") @@ -620,10 +644,12 @@ if __name__ == "__main__": tdCases.runOneCluster(fileName) else: tdLog.info("Procedures for testing self-deployment") - if not restful: - conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + if restful: + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: - conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllLinux(conn) @@ -640,10 +666,12 @@ if __name__ == "__main__": tdDnodes.stopAll() tdDnodes.start(1) time.sleep(1) - if not restful: - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) - else: + if restful: conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + elif websocket: + conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") + else: + conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) tdLog.info("Procedures for tdengine deployed in %s" % (host)) tdLog.info("query test after taosd restart") tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py", replicaVar) From 83e2e0dffd06af1c46b70e47963a8fdb34e36e27 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 4 Sep 2023 09:50:43 +0800 Subject: [PATCH 02/96] fix: remove tags keyword from document --- docs/en/12-taos-sql/06-select.md | 10 +--------- docs/zh/12-taos-sql/06-select.md | 12 +----------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 7ef47f2131..d38fc86975 100755 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -9,7 +9,7 @@ description: This document describes how to query data in TDengine. ```sql SELECT {DATABASE() | CLIENT_VERSION() | SERVER_VERSION() | SERVER_STATUS() | NOW() | TODAY() | TIMEZONE() | CURRENT_USER() | USER() } -SELECT [hints] [DISTINCT] [TAGS] select_list +SELECT [hints] [DISTINCT] select_list from_clause [WHERE condition] [partition_by_clause] @@ -225,14 +225,6 @@ The \_IROWTS pseudocolumn can only be used with INTERP function. This pseudocolu select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear); ``` -### TAGS Query - -The TAGS keyword returns only tag columns from all child tables when only tag columns are specified. One row containing tag columns is returned for each child table. - -```sql -SELECT TAGS tag_name [, tag_name ...] FROM stb_name -``` - ## Query Objects `FROM` can be followed by a number of tables or super tables, or can be followed by a sub-query. diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 222e166019..6d32d86f83 100755 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -9,7 +9,7 @@ description: 查询数据的详细语法 ```sql SELECT {DATABASE() | CLIENT_VERSION() | SERVER_VERSION() | SERVER_STATUS() | NOW() | TODAY() | TIMEZONE() | CURRENT_USER() | USER() } -SELECT [hints] [DISTINCT] [TAGS] select_list +SELECT [hints] [DISTINCT] select_list from_clause [WHERE condition] [partition_by_clause] @@ -160,16 +160,6 @@ SELECT DISTINCT col_name [, col_name ...] FROM tb_name; ::: -### 标签查询 - -当查询的列只有标签列时，`TAGS` 关键字可以指定返回所有子表的标签列。每个子表只返回一行标签列。 - -返回所有子表的标签列： - -```sql -SELECT TAGS tag_name [, tag_name ...] FROM stb_name -``` - ### 结果集列名 `SELECT`子句中，如果不指定返回结果集合的列名，结果集列名称默认使用`SELECT`子句中的表达式名称作为列名称。此外，用户可使用`AS`来重命名返回结果集合中列的名称。例如： From 03cddfa7ce03e1ee4dab4bf59b0cea60b102e92d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 4 Sep 2023 10:33:01 +0800 Subject: [PATCH 03/96] fix rc --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 2d58a10e51..4275e5626b 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -248,10 +248,14 @@ int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t tDecoderClear(&decoder); // tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) int64_t key[2] = {task.streamId, task.taskId}; + + taosWLockLatch(&pTq->pStreamMeta->lock); if (tdbTbUpsert(pTq->pStreamMeta->pTaskDb, key, sizeof(int64_t) << 1, (uint8_t*)pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr), pWriter->txn) < 0) { + taosWUnLockLatch(&pTq->pStreamMeta->lock); return -1; } + taosWUnLockLatch(&pTq->pStreamMeta->lock); } else if (pHdr->type == SNAP_DATA_STREAM_TASK_CHECKPOINT) { // do nothing } From 2cf1da0dc4ffe822f55c7e6540e943b012e085fa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Sep 2023 16:22:58 +0800 Subject: [PATCH 04/96] other: update the version. --- cmake/cmake.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 0e4785f643..27e0f1d68a 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.2.0.0.alpha") + SET(TD_VER_NUMBER "3.1.2.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) From 1a0480c834403b2e5eed894bf8a71174730a93f5 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 4 Sep 2023 17:14:29 +0800 Subject: [PATCH 05/96] fix/TD-22062 --- source/dnode/mnode/impl/src/mndStb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8484148642..1234b5f7de 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2342,9 +2342,9 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { alterReq.alterType, alterReq.numOfFields, alterReq.ttl); SName name = {0}; - tNameFromString(&name, pDb->name, T_NAME_ACCT | T_NAME_DB); + tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - auditRecord(pReq, pMnode->clusterId, "alterStb", name.dbname, alterReq.name, detail); + auditRecord(pReq, pMnode->clusterId, "alterStb", name.dbname, name.tname, detail); _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { From a0fadb07ea8f785fc483518cbbaee45efacbc294 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 4 Sep 2023 17:15:03 +0800 Subject: [PATCH 06/96] fix:support length function for varbinary --- source/libs/function/src/builtins.c | 4 ---- utils/test/c/varbinary_test.c | 10 +++++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 256123d62b..0057df7902 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1839,10 +1839,6 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - if (TSDB_DATA_TYPE_VARBINARY == ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } diff --git a/utils/test/c/varbinary_test.c b/utils/test/c/varbinary_test.c index e29b94ad1f..1d432ffbba 100644 --- a/utils/test/c/varbinary_test.c +++ b/utils/test/c/varbinary_test.c @@ -157,10 +157,6 @@ void varbinary_sql_test() { taos_free_result(pRes); // string function test, not support - pRes = taos_query(taos, "select length(c2) from stb"); - ASSERT(taos_errno(pRes) != 0); - taos_free_result(pRes); - pRes = taos_query(taos, "select ltrim(c2) from stb"); ASSERT(taos_errno(pRes) != 0); taos_free_result(pRes); @@ -190,7 +186,7 @@ void varbinary_sql_test() { ASSERT(taos_errno(pRes) != 0); taos_free_result(pRes); - // support first/last/last_row/count/hyperloglog/sample/tail/mode + // support first/last/last_row/count/hyperloglog/sample/tail/mode/length pRes = taos_query(taos, "select first(c2) from stb"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); @@ -207,6 +203,10 @@ void varbinary_sql_test() { ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); + pRes = taos_query(taos, "select length(c2) from stb where c2 = '\\x7F8290'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + pRes = taos_query(taos, "select cast(t2 as varbinary(16)) from stb order by ts"); while ((row = taos_fetch_row(pRes)) != NULL) { int32_t* length = taos_fetch_lengths(pRes); From 43ebdd9ce718e4204b51d4cfd7227dccce752bbc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 4 Sep 2023 17:18:22 +0800 Subject: [PATCH 07/96] docs:modify doc --- docs/en/12-taos-sql/10-function.md | 4 ++-- docs/zh/12-taos-sql/10-function.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 9d2b54dab3..340a3e917b 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -292,11 +292,11 @@ CONCAT_WS(separator_expr, expr1, expr2 [, expr] ...) LENGTH(expr) ``` -**Description**: The length in bytes of a string +**Description**: The length in bytes **Return value type**: Bigint -**Applicable data types**: VARCHAR and NCHAR fields or columns +**Applicable data types**: VARCHAR and NCHAR and VARBINARY **Nested query**: It can be used in both the outer query and inner query in a nested query. diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index cfec71934c..8b87a18e54 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -292,11 +292,11 @@ CONCAT_WS(separator_expr, expr1, expr2 [, expr] ...) LENGTH(expr) ``` -**功能说明**：以字节计数的字符串长度。 +**功能说明**：以字节计数的长度。 **返回结果类型**：BIGINT。 -**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。 +**适用数据类型**：VARCHAR, NCHAR, VARBINARY。 **嵌套子查询支持**：适用于内层查询和外层查询。 From 22b53c3eceb6b0b5e5effc1b28a2a4c404117135 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 1 Sep 2023 16:26:42 +0800 Subject: [PATCH 08/96] fix: 1. select tag from meters partition by tag limit returned wrong num of rows 2. select tag from meters partition by tag returned wrong num of rows --- source/libs/executor/src/sortoperator.c | 18 +++++++++--------- .../system-test/2-query/interval_limit_opt.py | 9 +++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9bd0991435..a59a7bb1ea 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -669,8 +669,13 @@ static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* p->info.id.groupId = tupleGroupId; pInfo->groupId = tupleGroupId; } else { - pInfo->prefetchedTuple = pTupleHandle; - break; + if (p->info.rows == 0) { + appendOneRowToDataBlock(p, pTupleHandle); + p->info.id.groupId = pInfo->groupId = tupleGroupId; + } else { + pInfo->prefetchedTuple = pTupleHandle; + break; + } } } else { appendOneRowToDataBlock(p, pTupleHandle); @@ -715,14 +720,9 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData resetLimitInfoForNextGroup(&pInfo->limitInfo); } - bool limitReached = applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); - // if limit is reached within a group, do not clear limiInfo otherwise the next block - // will be processed. - if (newgroup && limitReached) { - resetLimitInfoForNextGroup(&pInfo->limitInfo); - } + applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); - if (p->info.rows > 0 || limitReached) { + if (p->info.rows > 0) { break; } } diff --git a/tests/system-test/2-query/interval_limit_opt.py b/tests/system-test/2-query/interval_limit_opt.py index fef6e9facd..851138fed3 100644 --- a/tests/system-test/2-query/interval_limit_opt.py +++ b/tests/system-test/2-query/interval_limit_opt.py @@ -251,10 +251,19 @@ class TDTestCase: tdSql.checkData(2, 4, 9) tdSql.checkData(3, 4, 9) + def test_partition_by_limit_no_agg(self): + sql_template = 'select t1 from meters partition by t1 limit %d' + + for i in range(1, 5000, 1000): + tdSql.query(sql_template % i) + tdSql.checkRows(5 * i) + + def run(self): self.prepareTestEnv() self.test_interval_limit_offset() self.test_interval_partition_by_slimit_limit() + self.test_partition_by_limit_no_agg() def stop(self): tdSql.close() From b7b97e7a66ad5af699ba62684101f28b9f1d61a2 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 4 Sep 2023 18:33:19 +0800 Subject: [PATCH 09/96] TD-23722: add case for TMQ unsubscribe with taosAdapter --- tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/tmqCommon.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1377bc34b0..2b2b960356 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -48,6 +48,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCommon.py -R ,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 7f972d857e..087e5a7c62 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -233,7 +233,7 @@ class TMQCom: #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): rowsBatched = 0 - sql += " %s%d values "%(stbName,i) + sql += " %s.%s%d values "%(dbName, stbName, i) for j in range(rowsPerTbl): sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) rowsBatched += 1 @@ -241,7 +241,7 @@ class TMQCom: tsql.execute(sql) rowsBatched = 0 if j < rowsPerTbl - 1: - sql = "insert into %s%d values " %(stbName,i) + sql = "insert into %s.%s%d values " %(dbName, stbName,i) else: sql = "insert into " #end sql @@ -263,7 +263,7 @@ class TMQCom: #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): rowsBatched = 0 - sql += " %s%d values "%(ctbPrefix,i) + sql += " %s.%s%d values "%(dbName, ctbPrefix,i) for j in range(rowsPerTbl): if (j % 2 == 0): sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j) @@ -274,7 +274,7 @@ class TMQCom: tsql.execute(sql) rowsBatched = 0 if j < rowsPerTbl - 1: - sql = "insert into %s%d values " %(ctbPrefix,i) + sql = "insert into %s.%s%d values " %(dbName, ctbPrefix, i) else: sql = "insert into " #end sql @@ -296,7 +296,7 @@ class TMQCom: #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): rowsBatched = 0 - sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx) + sql += " %s.%s%d values "%(dbName, ctbPrefix, i+ctbStartIdx) for j in range(rowsPerTbl): if (j % 2 == 0): sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j) @@ -307,7 +307,7 @@ class TMQCom: tsql.execute(sql) rowsBatched = 0 if j < rowsPerTbl - 1: - sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx) + sql = "insert into %s.%s%d values " %(dbName, ctbPrefix, i+ctbStartIdx) else: sql = "insert into " #end sql From 0fed045a127d8fa876f704cea26d7ea754078c5e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 4 Sep 2023 18:33:50 +0800 Subject: [PATCH 10/96] fix: fix last(*)/last_row(*) return MD5 hashed column name --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e81d0ef989..797adee18b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2902,7 +2902,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { taosCreateMD5Hash(buf, len); strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName); - taosCreateMD5Hash(buf, len); + // note: userAlias could be truncated here strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); } } else { @@ -2910,7 +2910,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { taosCreateMD5Hash(buf, len); strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); - taosCreateMD5Hash(buf, len); + // note: userAlias could be truncated here strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); } From 0ab0b4507bcaf08917cabddc383b70843647b1ff Mon Sep 17 00:00:00 2001 From: haoranchen Date: Mon, 4 Sep 2023 19:23:14 +0800 Subject: [PATCH 11/96] comment 5dnode3mnodeRoll.py --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1377bc34b0..24f6ad33b6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -467,7 +467,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,n,system-test,python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 -,,n,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 +#,,n,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 From 6f8c642c9aa1f8569c2b9e43b4c474146e987487 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 28 Aug 2023 16:06:43 +0800 Subject: [PATCH 12/96] fix: use int64_t for num of rows returned --- tools/shell/src/shellEngine.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 72cf3cd1cc..a870cf5755 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -28,12 +28,12 @@ static void shellRecordCommandToHistory(char *command); static int32_t shellRunCommand(char *command, bool recordHistory); static void shellRunSingleCommandImp(char *command); static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); -static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); +static int64_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); static void shellPrintNChar(const char *str, int32_t length, int32_t width); static void shellPrintGeometry(const unsigned char *str, int32_t length, int32_t width); -static int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql); -static int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql); -static int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql); +static int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql); +static int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql); +static int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql); static void shellReadHistory(); static void shellWriteHistory(); static void shellPrintError(TAOS_RES *tres, int64_t st); @@ -238,14 +238,14 @@ void shellRunSingleCommandImp(char *command) { if (pFields != NULL) { // select and show kinds of commands int32_t error_no = 0; - int32_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode, command); + int64_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode, command); if (numOfRows < 0) return; et = taosGetTimestampUs(); if (error_no == 0) { - printf("Query OK, %d row(s) in set (%.6fs)\r\n", numOfRows, (et - st) / 1E6); + printf("Query OK, %"PRId64 " row(s) in set (%.6fs)\r\n", numOfRows, (et - st) / 1E6); } else { - printf("Query interrupted (%s), %d row(s) in set (%.6fs)\r\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); + printf("Query interrupted (%s), %"PRId64 " row(s) in set (%.6fs)\r\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } taos_free_result(pSql); } else { @@ -430,7 +430,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i } } -int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres) { +int64_t shellDumpResultToFile(const char *fname, TAOS_RES *tres) { char fullname[PATH_MAX] = {0}; if (taosExpandDir(fname, fullname, PATH_MAX) != 0) { tstrncpy(fullname, fname, PATH_MAX); @@ -459,7 +459,7 @@ int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres) { } taosFprintfFile(pFile, "\r\n"); - int32_t numOfRows = 0; + int64_t numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); for (int32_t i = 0; i < num_fields; i++) { @@ -702,7 +702,7 @@ bool shellIsShowQuery(const char *sql) { return false; } -int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { +int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; @@ -726,7 +726,7 @@ int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } - int32_t numOfRows = 0; + int64_t numOfRows = 0; int32_t showMore = 1; do { if (numOfRows < resShowMaxNum) { @@ -856,7 +856,7 @@ void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) { putchar('\n'); } -int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { +int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; @@ -879,7 +879,7 @@ int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } - int32_t numOfRows = 0; + int64_t numOfRows = 0; int32_t showMore = 1; do { @@ -915,8 +915,8 @@ int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { return numOfRows; } -int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql) { - int32_t numOfRows = 0; +int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql) { + int64_t numOfRows = 0; if (fname != NULL) { numOfRows = shellDumpResultToFile(fname, tres); } else if (vertical) { From 1185bba98b797e7d3102a6935765c7a93952e8b4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 4 Sep 2023 15:20:10 +0800 Subject: [PATCH 13/96] fix: pass compilation --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index a870cf5755..a7f79fc9db 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -730,7 +730,7 @@ int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { int32_t showMore = 1; do { if (numOfRows < resShowMaxNum) { - printf("*************************** %d.row ***************************\r\n", numOfRows + 1); + printf("*************************** %"PRId64".row ***************************\r\n", numOfRows + 1); int32_t *length = taos_fetch_lengths(tres); From 74b9eab63e32e95d482ebdf6d96c2b40ad044c05 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 5 Sep 2023 10:24:31 +0800 Subject: [PATCH 14/96] add test cases --- tests/script/tsim/query/multires_func.sim | 20 ++++++++++++ .../script/tsim/query/r/multires_func.result | 31 +++++++++++++++++++ tests/script/tsim/query/t/multires_func.sql | 5 +++ 3 files changed, 56 insertions(+) create mode 100644 tests/script/tsim/query/multires_func.sim create mode 100644 tests/script/tsim/query/r/multires_func.result create mode 100644 tests/script/tsim/query/t/multires_func.sql diff --git a/tests/script/tsim/query/multires_func.sim b/tests/script/tsim/query/multires_func.sim new file mode 100644 index 0000000000..34aadffe2e --- /dev/null +++ b/tests/script/tsim/query/multires_func.sim @@ -0,0 +1,20 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test +sql use test +sql CREATE TABLE `tb` (`ts` TIMESTAMP, `c0` INT, `c1` FLOAT, `c2` BINARY(10)) + + +sql insert into tb values("2022-05-15 00:01:08.000", 1, 1.0, "abc") +sql insert into tb values("2022-05-16 00:01:08.000", 2, 2.0, "bcd") +sql insert into tb values("2022-05-17 00:01:08.000", 3, 3.0, "cde") + + +#sleep 10000000 +system taos -P7100 -s 'source tsim/query/t/multires_func.sql' | grep -v 'Query OK' | grep -v 'Client Version' > /tmp/multires_func.result +system echo ----------------------diff start----------------------- +system git diff --exit-code --color tsim/query/r/multires_func.result /tmp/multires_func.result +system echo ----------------------diff succeed----------------------- diff --git a/tests/script/tsim/query/r/multires_func.result b/tests/script/tsim/query/r/multires_func.result new file mode 100644 index 0000000000..c221b00b97 --- /dev/null +++ b/tests/script/tsim/query/r/multires_func.result @@ -0,0 +1,31 @@ +Copyright (c) 2022 by TDengine, all rights reserved. + +taos> source tsim/query/t/multires_func.sql +taos> use test; +Database changed. + +taos> select count(*) from tb\G; +*************************** 1.row *************************** +count(*): 3 + +taos> select last(*) from tb\G; +*************************** 1.row *************************** +ts: 2022-05-17 00:01:08.000 +c0: 3 +c1: 3.0000000 +c2: cde + +taos> select last_row(*) from tb\G; +*************************** 1.row *************************** +ts: 2022-05-17 00:01:08.000 +c0: 3 +c1: 3.0000000 +c2: cde + +taos> select first(*) from tb\G; +*************************** 1.row *************************** +ts: 2022-05-15 00:01:08.000 +c0: 1 +c1: 1.0000000 +c2: abc + diff --git a/tests/script/tsim/query/t/multires_func.sql b/tests/script/tsim/query/t/multires_func.sql new file mode 100644 index 0000000000..6a191233b9 --- /dev/null +++ b/tests/script/tsim/query/t/multires_func.sql @@ -0,0 +1,5 @@ +use test; +select count(*) from tb\G; +select last(*) from tb\G; +select last_row(*) from tb\G; +select first(*) from tb\G; From 8dbdeccc8f9f0fe5a411e2f54348324b54227351 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 5 Sep 2023 11:41:36 +0800 Subject: [PATCH 15/96] fix: coverity scan --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 9 +++----- source/dnode/vnode/src/tsdb/tsdbFS2.c | 23 ++++++++++++++------ source/dnode/vnode/src/tsdb/tsdbFSet2.c | 20 ++++++++++------- source/dnode/vnode/src/tsdb/tsdbFile2.c | 8 +++++-- source/dnode/vnode/src/tsdb/tsdbIter.c | 6 +++-- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index dc5e3649cc..c9cbd37b09 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -982,7 +982,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TS goto _exit; } - for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + if (writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray)) { const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); if (brinBlk->minTbid.uid != writer->ctx->tbid->uid) { @@ -995,7 +995,6 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TS writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; - break; } } @@ -1112,7 +1111,7 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA break; } - for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + if (writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray)) { const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); @@ -1120,7 +1119,6 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; - break; } } @@ -1253,7 +1251,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom break; } - for (; writer->ctx->tombBlkArrayIdx < TARRAY2_SIZE(writer->ctx->tombBlkArray); ++writer->ctx->tombBlkArrayIdx) { + if (writer->ctx->tombBlkArrayIdx < TARRAY2_SIZE(writer->ctx->tombBlkArray)) { const STombBlk *tombBlk = TARRAY2_GET_PTR(writer->ctx->tombBlkArray, writer->ctx->tombBlkArrayIdx); code = tsdbDataFileReadTombBlock(writer->ctx->reader, tombBlk, writer->ctx->tombBlock); @@ -1261,7 +1259,6 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom writer->ctx->tombBlockIdx = 0; writer->ctx->tombBlkArrayIdx++; - break; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index a997c3eea5..f43bb52d05 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -174,11 +174,17 @@ int32_t save_fs(const TFileSetArray *arr, const char *fname) { // fset cJSON *ajson = cJSON_AddArrayToObject(json, "fset"); - if (!ajson) TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + if (!ajson) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } const STFileSet *fset; TARRAY2_FOREACH(arr, fset) { cJSON *item = cJSON_CreateObject(); - if (!item) TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + if (!item) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } cJSON_AddItemToArray(ajson, item); code = tsdbTFileSetToJson(fset, item); @@ -231,7 +237,8 @@ static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) { TSDB_CHECK_CODE(code, lino, _exit); } } else { - TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + code = TSDB_CODE_FILE_CORRUPTED; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: @@ -312,7 +319,8 @@ static int32_t commit_edit(STFileSystem *fs) { int32_t code; int32_t lino; if ((code = taosRenameFile(current_t, current))) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + code = TAOS_SYSTEM_ERROR(code); + TSDB_CHECK_CODE(code, lino, _exit); } code = apply_commit(fs); @@ -345,7 +353,8 @@ static int32_t abort_edit(STFileSystem *fs) { int32_t code; int32_t lino; if ((code = taosRemoveFile(fname))) { - TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + code = TAOS_SYSTEM_ERROR(code); + TSDB_CHECK_CODE(code, lino, _exit); } code = apply_abort(fs); @@ -398,7 +407,7 @@ static int32_t tsdbFSAddEntryToFileObjHash(STFileHash *hash, const char *fname) STFileHashEntry *entry = taosMemoryMalloc(sizeof(*entry)); if (entry == NULL) return TSDB_CODE_OUT_OF_MEMORY; - strcpy(entry->fname, fname); + strncpy(entry->fname, fname, TSDB_FILENAME_LEN); uint32_t idx = MurmurHash3_32(fname, strlen(fname)) % hash->numBucket; @@ -873,7 +882,7 @@ int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { STFileSet *fset1; fsetArr[0] = taosMemoryMalloc(sizeof(TFileSetArray)); - if (fsetArr == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (fsetArr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; TARRAY2_INIT(fsetArr[0]); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 13ef7b3ba6..03a14a7162 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -46,7 +46,8 @@ static int32_t tsdbSttLvlInitEx(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lvl return code; } - TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + code = TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + return code; } return 0; } @@ -185,7 +186,8 @@ static int32_t tsdbJsonToSttLvl(STsdb *pTsdb, const cJSON *json, SSttLvl **lvl) return code; } - TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + code = TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + if (code) return code; } return 0; } @@ -263,7 +265,8 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset) { return code; } - TARRAY2_APPEND((*fset)->lvlArr, lvl); + code = TARRAY2_APPEND((*fset)->lvlArr, lvl); + if (code) return code; } } else { return TSDB_CODE_FILE_CORRUPTED; @@ -326,11 +329,12 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj; STFileObj **fobjPtr = TARRAY2_SEARCH(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); - tfobjp = (fobjPtr ? *fobjPtr : NULL); - - ASSERT(tfobjp); - - tfobjp->f[0] = op->nf; + if (fobjPtr) { + tfobjp = *fobjPtr; + tfobjp->f[0] = op->nf; + } else { + tsdbError("file not found, cid:%" PRId64, op->of.cid); + } } else { fset->farr[op->nf.type]->f[0] = op->nf; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 585316469a..3d8964d41b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -42,8 +42,12 @@ static const struct { }; void remove_file(const char *fname) { - taosRemoveFile(fname); - tsdbInfo("file:%s is removed", fname); + int32_t code = taosRemoveFile(fname); + if (code) { + tsdbError("file:%s remove failed", fname); + } else { + tsdbInfo("file:%s is removed", fname); + } } static int32_t tfile_to_json(const STFile *file, cJSON *json) { diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.c b/source/dnode/vnode/src/tsdb/tsdbIter.c index 9780cc6be6..447832108d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbIter.c @@ -356,7 +356,8 @@ static int32_t tsdbSttIterOpen(STsdbIter *iter) { } iter->sttData->sttBlkArrayIdx = 0; - tBlockDataCreate(iter->sttData->blockData); + code = tBlockDataCreate(iter->sttData->blockData); + if (code) return code; iter->sttData->blockDataIdx = 0; return tsdbSttIterNext(iter, NULL); @@ -381,7 +382,8 @@ static int32_t tsdbDataIterOpen(STsdbIter *iter) { iter->dataData->brinBlockIdx = 0; // SBlockData - tBlockDataCreate(iter->dataData->blockData); + code = tBlockDataCreate(iter->dataData->blockData); + if (code) return code; iter->dataData->blockDataIdx = 0; return tsdbDataIterNext(iter, NULL); From d0503a8a3f80228134ecfccdcd3a22d436d16c0e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 5 Sep 2023 12:11:58 +0800 Subject: [PATCH 16/96] fix invalid read --- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 61fc3c7ae9..4a1b3961cd 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -51,7 +51,7 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS SStreamSnapReader* pSnapReader = NULL; - if (streamSnapReaderOpen(pTq, sver, chkpId, pTq->path, &pSnapReader) == 0) { + if (streamSnapReaderOpen(meta, sver, chkpId, pTq->path, &pSnapReader) == 0) { pReader->complete = 1; } else { code = -1; From a3b749ae6dcd859cd63a88b1728d7c7fd14f98e7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 5 Sep 2023 14:04:00 +0800 Subject: [PATCH 17/96] update test case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f825557033..cd37462cde 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -59,11 +59,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCommon.py -R ,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py From db75bc4885a70613399b155e8f179ee1fc878b76 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 5 Sep 2023 14:44:14 +0800 Subject: [PATCH 18/96] update Dockerfile --- tests/ci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/Dockerfile b/tests/ci/Dockerfile index 213570dfb2..38cab07792 100644 --- a/tests/ci/Dockerfile +++ b/tests/ci/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.8 RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro +RUN pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro taos-ws-py RUN apt-get update RUN apt-get install -y psmisc sudo tree libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config build-essential valgrind \ vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping From 3e72258386f464a6aa1a5f00ad3af5107c729afc Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 5 Sep 2023 18:05:15 +0800 Subject: [PATCH 19/96] update jdbc version --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 655629b92c..9e70a6bbf1 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -338,7 +338,7 @@ if [ "$verMode" == "cluster" ]; then tmp_pwd=`pwd` cd ${install_dir}/connector if [ ! -d taos-connector-jdbc ];then - git clone -b 3.2.1 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: fi cd taos-connector-jdbc mvn clean package -Dmaven.test.skip=true From 600f89aa8999b6de43fa2660fbe5b873c12ebea8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 5 Sep 2023 18:17:25 +0800 Subject: [PATCH 20/96] test:replace clock_gettime in cases for win --- tests/system-test/0-others/splitVGroup.py | 2 +- tests/system-test/0-others/timeRangeWise.py | 2 +- tests/system-test/1-insert/precisionUS.py | 2 +- tests/system-test/2-query/orderBy.py | 2 +- tests/system-test/2-query/smaBasic.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/system-test/0-others/splitVGroup.py b/tests/system-test/0-others/splitVGroup.py index 9fd00892e4..ed80505ce2 100644 --- a/tests/system-test/0-others/splitVGroup.py +++ b/tests/system-test/0-others/splitVGroup.py @@ -198,7 +198,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): - seed = time.clock_gettime(time.CLOCK_REALTIME) + seed = time.time() % 10000 random.seed(seed) self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") diff --git a/tests/system-test/0-others/timeRangeWise.py b/tests/system-test/0-others/timeRangeWise.py index a7dc18aa82..5ef5aa4a75 100644 --- a/tests/system-test/0-others/timeRangeWise.py +++ b/tests/system-test/0-others/timeRangeWise.py @@ -210,7 +210,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): - seed = time.clock_gettime(time.CLOCK_REALTIME) + seed = time.time() % 10000 random.seed(seed) self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") diff --git a/tests/system-test/1-insert/precisionUS.py b/tests/system-test/1-insert/precisionUS.py index 1b41d66010..d634149297 100644 --- a/tests/system-test/1-insert/precisionUS.py +++ b/tests/system-test/1-insert/precisionUS.py @@ -220,7 +220,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): - seed = time.clock_gettime(time.CLOCK_REALTIME) + seed = time.time() % 10000 random.seed(seed) self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") diff --git a/tests/system-test/2-query/orderBy.py b/tests/system-test/2-query/orderBy.py index fed1651b3a..fa447cbca4 100644 --- a/tests/system-test/2-query/orderBy.py +++ b/tests/system-test/2-query/orderBy.py @@ -220,7 +220,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): - seed = time.clock_gettime(time.CLOCK_REALTIME) + seed = time.time() % 10000 random.seed(seed) self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index c221a70605..a82d190e2b 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -269,7 +269,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): - seed = time.clock_gettime(time.CLOCK_REALTIME) + seed = time.time() % 10000 random.seed(seed) self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") From 253001165ae8564cf3268881264fd72a3480eeae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 5 Sep 2023 18:53:24 +0800 Subject: [PATCH 21/96] Update mxml_CMakeLists.txt.in (#22722) Co-authored-by: sunpeng --- cmake/mxml_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/mxml_CMakeLists.txt.in b/cmake/mxml_CMakeLists.txt.in index 1ac90ebdd4..762df40e10 100644 --- a/cmake/mxml_CMakeLists.txt.in +++ b/cmake/mxml_CMakeLists.txt.in @@ -1,7 +1,7 @@ # cos ExternalProject_Add(mxml GIT_REPOSITORY https://github.com/michaelrsweet/mxml.git - GIT_TAG release-2.12 + GIT_TAG v2.12 SOURCE_DIR "${TD_CONTRIB_DIR}/mxml" #BINARY_DIR "" BUILD_IN_SOURCE TRUE From 6d5d4cba260c1fa18510a4338259c71d8a0b455c Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Tue, 5 Sep 2023 19:34:55 +0800 Subject: [PATCH 22/96] docs: correct links to removed page --- docs/zh/14-reference/05-taosbenchmark.md | 2 +- docs/zh/14-reference/14-taosKeeper.md | 2 +- docs/zh/20-third-party/11-kafka.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index d5ae95b20b..e42d921dc7 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -13,7 +13,7 @@ taosBenchmark (曾用名 taosdemo ) 是一个用于测试 TDengine 产品性能 taosBenchmark 有两种安装方式: -- 安装 TDengine 官方安装包的同时会自动安装 taosBenchmark, 详情请参考[ TDengine 安装](../../operation/pkg-install)。 +- 安装 TDengine 官方安装包的同时会自动安装 taosBenchmark, 详情请参考 [TDengine 安装](../../get-started/)。 - 单独编译 taos-tools 并安装, 详情请参考 [taos-tools](https://github.com/taosdata/taos-tools) 仓库。 diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index 738d351d45..92c85e7a2c 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -16,7 +16,7 @@ taosKeeper 是 TDengine 3.0 版本监控指标的导出工具，通过简单的 taosKeeper 有两种安装方式： taosKeeper 安装方式： -- 安装 TDengine 官方安装包的同时会自动安装 taosKeeper, 详情请参考[ TDengine 安装](../../operation/pkg-install)。 +- 安装 TDengine 官方安装包的同时会自动安装 taosKeeper, 详情请参考[ TDengine 安装](../../get-started/)。 - 单独编译 taosKeeper 并安装，详情请参考 [taosKeeper](https://github.com/taosdata/taoskeeper) 仓库。 diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 2f09e50f0b..3d8822fdae 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -23,7 +23,7 @@ TDengine Source Connector 用于把数据实时地从 TDengine 读出来发送 1. Linux 操作系统 2. 已安装 Java 8 和 Maven 3. 已安装 Git、curl、vi -4. 已安装并启动 TDengine。如果还没有可参考[安装和卸载](../../operation/pkg-install) +4. 已安装并启动 TDengine。如果还没有可参考[安装和卸载](../../get-started/) ## 安装 Kafka From 6b799474026beb6261aca428876724c6c4215b98 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 6 Sep 2023 08:48:46 +0800 Subject: [PATCH 23/96] fix: fix coverity scan issue --- source/libs/function/src/udfd.c | 4 ++-- source/libs/planner/src/planLogicCreater.c | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index bd459af9f5..c8eb7580ed 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -69,7 +69,7 @@ const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strncpy(processFuncName, udfName, sizeof(processFuncName)); + snprintf(processFuncName, sizeof(processFuncName), "%s", udfName); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)); char startFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; @@ -103,7 +103,7 @@ int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { if (udf->funcType == UDF_FUNC_TYPE_SCALAR) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strncpy(processFuncName, udfName, sizeof(processFuncName)); + snprintf(processFuncName, sizeof(processFuncName), "%s", udfName); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc)); } else if (udf->funcType == UDF_FUNC_TYPE_AGG) { udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index d460c8074d..96d253494d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -46,8 +46,8 @@ static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartit pCol->colType = COLUMN_TYPE_TBNAME; SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); if (pVal) { - strcpy(pCol->tableName, pVal->literal); - strcpy(pCol->tableAlias, pVal->literal); + snprintf(pCol->tableName, sizeof(pCol->tableName), "%s", pVal->literal); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pVal->literal); } break; case FUNCTION_TYPE_WSTART: @@ -531,6 +531,9 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pLeft); } + if (TSDB_CODE_SUCCESS != code) { + pLeft = NULL; + } } SLogicNode* pRight = NULL; @@ -584,7 +587,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } } - if (NULL == pJoin->node.pTargets) { + if (NULL == pJoin->node.pTargets && NULL != pLeft) { pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); if (NULL == pJoin->node.pTargets) { code = TSDB_CODE_OUT_OF_MEMORY; From f9f75580077127cedaf17ddb3ab8d4110759230c Mon Sep 17 00:00:00 2001 From: songshuqi <35394385+ssq001@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:05:50 +0800 Subject: [PATCH 24/96] Update 02-intro.md (#22759) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改TDengine pro为TDengine Enterprise --- docs/zh/02-intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md index 68a2541717..bb989f27da 100644 --- a/docs/zh/02-intro.md +++ b/docs/zh/02-intro.md @@ -10,7 +10,7 @@ TDengine 是一款开源、高性能、云原生的[时序数据库](https://tde ## 主要产品 -TDengine 有三个主要产品：TDengine Pro （即 TDengine 企业版），TDengine Cloud，和 TDengine OSS，关于它们的具体定义请参考 +TDengine 有三个主要产品：TDengine Enterprise （即 TDengine 企业版），TDengine Cloud，和 TDengine OSS，关于它们的具体定义请参考 - [TDengine 企业版](https://www.taosdata.com/tdengine-pro) - [TDengine 云服务](https://cloud.taosdata.com/?utm_source=menu&utm_medium=webcn) - [TDengine 开源版](https://www.taosdata.com/tdengine-oss) From 5dedaebb01116168dae2996e46d8e67a0c944538 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 4 Sep 2023 16:22:21 +0800 Subject: [PATCH 25/96] fix: desc fill missed one window --- include/common/ttime.h | 2 +- source/common/src/ttime.c | 73 ++++++++++++++------ source/libs/executor/src/tfill.c | 10 ++- tests/system-test/2-query/fill_with_group.py | 6 ++ 4 files changed, 64 insertions(+), 27 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index de74e48100..37e3045817 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -75,7 +75,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); -int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); +int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, int32_t order); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index e9313e0591..7d65ac424f 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -692,34 +692,67 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { return (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision) + fraction); } -int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision) { +/** + * @brief calc how many windows after filling between skey and ekey + * @notes for asc order + * skey ---> ekey + * ^ ^ + * _____!_____.........._____|_____.. + * |__1__) + * |__2__)...-->|_ret+1_) + * skey + ret * interval <= ekey + * skey + ret * interval + interval > ekey + * ======> (ekey - skey - interval) / interval < ret <= (ekey - skey) / interval + * For keys from blocks which do not need filling, skey + ret * interval == ekey. + * For keys need filling, skey + ret * interval <= ekey. + * Total num of windows is ret + 1(the last window) + * + * for desc order + * skey <--- ekey + * ^ ^ + * _____|____..........______!____... + * |_first_) + * |__1__) + * |_ret_)<--...|__2__) + * skey >= ekey - ret * interval + * skey < ekey - ret * interval + interval + *=======> (ekey - skey) / interval <= ret < (ekey - skey + interval) / interval + * For keys from blocks which do not need filling, skey == ekey - ret * interval. + * For keys need filling, skey >= ekey - ret * interval. + * Total num of windows is ret + 1(the first window) + */ +int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, + int32_t order) { if (ekey < skey) { int64_t tmp = ekey; ekey = skey; skey = tmp; } + int32_t ret; if (unit != 'n' && unit != 'y') { - return (int32_t)((ekey - skey) / interval); + ret = (int32_t)((ekey - skey) / interval); + if (order == TSDB_ORDER_DESC && ret * interval < (ekey - skey)) ret += 1; + } else { + skey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); + ekey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); + + struct tm tm; + time_t t = (time_t)skey; + taosLocalTime(&t, &tm, NULL); + int32_t smon = tm.tm_year * 12 + tm.tm_mon; + + t = (time_t)ekey; + taosLocalTime(&t, &tm, NULL); + int32_t emon = tm.tm_year * 12 + tm.tm_mon; + + if (unit == 'y') { + interval *= 12; + } + ret = (emon - smon) / (int32_t)interval; + if (order == TSDB_ORDER_DESC && ret * interval < (smon - emon)) ret += 1; } - - skey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); - ekey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); - - struct tm tm; - time_t t = (time_t)skey; - taosLocalTime(&t, &tm, NULL); - int32_t smon = tm.tm_year * 12 + tm.tm_mon; - - t = (time_t)ekey; - taosLocalTime(&t, &tm, NULL); - int32_t emon = tm.tm_year * 12 + tm.tm_mon; - - if (unit == 'y') { - interval *= 12; - } - - return (emon - smon) / (int32_t)interval; + return ret + 1; } int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 4e0dff9d4f..44d39392a2 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -578,9 +578,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma SColumnInfoData* pCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId); int64_t* tsList = (int64_t*)pCol->pData; TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - numOfRes = taosTimeCountInterval(lastKey, pFillInfo->currentKey, pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); - numOfRes += 1; + numOfRes = taosTimeCountIntervalForFill(lastKey, pFillInfo->currentKey, pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, pFillInfo->interval.precision, pFillInfo->order); ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || @@ -588,9 +587,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma return 0; } - numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); - numOfRes += 1; + numOfRes = taosTimeCountIntervalForFill(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, pFillInfo->interval.precision, pFillInfo->order); } return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index 393102c8ed..b442647ff4 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -137,6 +137,12 @@ class TDTestCase: sql = "select _wstart, _wend, count(ts), sum(c1) from meters where ts > '2018-11-25 00:00:00.000' and ts < '2018-11-26 00:00:00.00' interval(1d) fill(NULL) order by _wstart desc" tdSql.query(sql) tdSql.checkRows(1) + sql = "select _wstart, count(*) from meters where ts > '2018-08-20 00:00:00.000' and ts < '2018-09-30 00:00:00.000' interval(9d) fill(NULL) order by _wstart desc;" + tdSql.query(sql) + tdSql.checkRows(6) + sql = "select _wstart, count(*) from meters where ts > '2018-08-20 00:00:00.000' and ts < '2018-09-30 00:00:00.000' interval(9d) fill(NULL) order by _wstart;" + tdSql.query(sql) + tdSql.checkRows(6) def run(self): self.prepareTestEnv() From 0b18689fa6dc4510b9bfa4c7b6dbf7a851f57d99 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 6 Sep 2023 10:57:49 +0800 Subject: [PATCH 26/96] fix: fix coverity issues --- source/client/src/clientEnv.c | 10 ++- source/client/src/clientMain.c | 14 ++-- source/libs/command/src/command.c | 3 +- source/libs/executor/src/dataInserter.c | 2 +- .../libs/executor/src/dynqueryctrloperator.c | 64 ++++++++++++++++++- .../libs/executor/src/eventwindowoperator.c | 6 +- source/libs/executor/src/groupcacheoperator.c | 19 +++++- source/libs/executor/src/hashjoinoperator.c | 2 + source/libs/executor/src/mergejoinoperator.c | 5 ++ source/libs/executor/src/scanoperator.c | 20 ++++-- source/libs/executor/src/tsort.c | 18 +++++- 11 files changed, 137 insertions(+), 26 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b555f4e683..dbddf9cac6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -380,8 +380,7 @@ void destroySubRequests(SRequestObj *pRequest) { pReqList[++reqIdx] = pTmp; releaseRequest(tmpRefId); } else { - tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId, - pTmp->requestId); + tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); break; } } @@ -398,7 +397,7 @@ void destroySubRequests(SRequestObj *pRequest) { removeRequest(pTmp->self); releaseRequest(pTmp->self); } else { - tscError("0x%" PRIx64 " is not there", tmpRefId); + tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); break; } } @@ -492,8 +491,7 @@ void stopAllQueries(SRequestObj *pRequest) { pReqList[++reqIdx] = pTmp; releaseRequest(tmpRefId); } else { - tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId, - pTmp->requestId); + tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); break; } } @@ -512,7 +510,7 @@ void stopAllQueries(SRequestObj *pRequest) { taosStopQueryImpl(pTmp); releaseRequest(pTmp->self); } else { - tscError("0x%" PRIx64 " is not there", tmpRefId); + tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); break; } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 73b4ec2a74..ac7a6e6646 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -874,8 +874,13 @@ void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResult if (TSDB_CODE_SUCCESS == code) { code = cloneCatalogReq(&pNewWrapper->pCatalogReq, pWrapper->pCatalogReq); } - doAsyncQueryFromAnalyse(pResultMeta, pNewWrapper, code); - nodesDestroyNode(pRoot); + if (TSDB_CODE_SUCCESS == code) { + doAsyncQueryFromAnalyse(pResultMeta, pNewWrapper, code); + nodesDestroyNode(pRoot); + } else { + handleQueryAnslyseRes(pWrapper, pResultMeta, code); + return; + } } void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code) { @@ -1148,8 +1153,7 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { pReqList[++reqIdx] = pTmp; releaseRequest(tmpRefId); } else { - tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId, - pTmp->requestId); + tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); break; } } @@ -1162,7 +1166,7 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { removeRequest(pTmp->self); releaseRequest(pTmp->self); } else { - tscError("0x%" PRIx64 " is not there", tmpRefId); + tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); break; } } diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 991e2013f5..8b868ffde4 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -308,10 +308,11 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch if (retentions) { len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); - taosMemoryFree(retentions); } } + taosMemoryFree(retentions); + (varDataLen(buf2)) = len; colDataSetVal(pCol2, 0, buf2, false); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 38d82f05fb..e47cbb7eba 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -446,7 +446,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat taosThreadMutexInit(&inserter->mutex, NULL); if (NULL == inserter->pDataBlocks) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; + goto _return; } inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols; diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index f2ed4ba618..8fc46e0239 100755 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -151,14 +151,21 @@ static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid, bool needCache, SOperatorParam* pChild) { *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { + freeOperatorParam(pChild, OP_GET_PARAM); return TSDB_CODE_OUT_OF_MEMORY; } if (pChild) { (*ppRes)->pChildren = taosArrayInit(1, POINTER_BYTES); - if (NULL == *ppRes) { + if (NULL == (*ppRes)->pChildren) { + freeOperatorParam(pChild, OP_GET_PARAM); + freeOperatorParam(*ppRes, OP_GET_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild)) { + freeOperatorParam(pChild, OP_GET_PARAM); + freeOperatorParam(*ppRes, OP_GET_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } } else { @@ -167,6 +174,8 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down SGcOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcOperatorParam)); if (NULL == pGc) { + freeOperatorParam(*ppRes, OP_GET_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } @@ -193,6 +202,7 @@ static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_ SGcNotifyOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcNotifyOperatorParam)); if (NULL == pGc) { + freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); return TSDB_CODE_OUT_OF_MEMORY; } @@ -248,6 +258,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d SExchangeOperatorBatchParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam)); if (NULL == pExc) { + taosMemoryFreeClear(*ppRes); return TSDB_CODE_OUT_OF_MEMORY; } @@ -255,6 +266,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d pExc->pBatchs = tSimpleHashInit(tSimpleHashGetSize(pVg), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); if (NULL == pExc->pBatchs) { taosMemoryFree(pExc); + taosMemoryFreeClear(*ppRes); return TSDB_CODE_OUT_OF_MEMORY; } tSimpleHashSetFreeFp(pExc->pBatchs, freeExchangeGetBasicOperatorParam); @@ -288,21 +300,36 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initParam, SOperatorParam* pChild0, SOperatorParam* pChild1) { *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { + freeOperatorParam(pChild0, OP_GET_PARAM); + freeOperatorParam(pChild1, OP_GET_PARAM); return TSDB_CODE_OUT_OF_MEMORY; } (*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES); if (NULL == *ppRes) { + freeOperatorParam(pChild0, OP_GET_PARAM); + freeOperatorParam(pChild1, OP_GET_PARAM); + freeOperatorParam(*ppRes, OP_GET_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) { + freeOperatorParam(pChild0, OP_GET_PARAM); + freeOperatorParam(pChild1, OP_GET_PARAM); + freeOperatorParam(*ppRes, OP_GET_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) { + freeOperatorParam(pChild1, OP_GET_PARAM); + freeOperatorParam(*ppRes, OP_GET_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } SSortMergeJoinOperatorParam* pJoin = taosMemoryMalloc(sizeof(SSortMergeJoinOperatorParam)); if (NULL == pJoin) { + freeOperatorParam(*ppRes, OP_GET_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } @@ -318,16 +345,28 @@ static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initPara static int32_t buildMergeJoinNotifyOperatorParam(SOperatorParam** ppRes, SOperatorParam* pChild0, SOperatorParam* pChild1) { *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { + freeOperatorParam(pChild0, OP_NOTIFY_PARAM); + freeOperatorParam(pChild1, OP_NOTIFY_PARAM); return TSDB_CODE_OUT_OF_MEMORY; } (*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES); if (NULL == *ppRes) { + taosMemoryFreeClear(*ppRes); + freeOperatorParam(pChild0, OP_NOTIFY_PARAM); + freeOperatorParam(pChild1, OP_NOTIFY_PARAM); return TSDB_CODE_OUT_OF_MEMORY; } if (pChild0 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) { + freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); + freeOperatorParam(pChild0, OP_NOTIFY_PARAM); + freeOperatorParam(pChild1, OP_NOTIFY_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } if (pChild1 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) { + freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); + freeOperatorParam(pChild1, OP_NOTIFY_PARAM); + *ppRes = NULL; return TSDB_CODE_OUT_OF_MEMORY; } @@ -420,13 +459,34 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS if (TSDB_CODE_SUCCESS == code) { code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0); + pSrcParam0 = NULL; } if (TSDB_CODE_SUCCESS == code) { code = buildGroupCacheOperatorParam(&pGcParam1, 1, *rightVg, *rightUid, pPost->rightNeedCache, pSrcParam1); + pSrcParam1 = NULL; } if (TSDB_CODE_SUCCESS == code) { code = buildMergeJoinOperatorParam(ppParam, pSrcParam0 ? true : false, pGcParam0, pGcParam1); } + if (TSDB_CODE_SUCCESS != code) { + if (pSrcParam0) { + freeOperatorParam(pSrcParam0, OP_GET_PARAM); + } + if (pSrcParam1) { + freeOperatorParam(pSrcParam1, OP_GET_PARAM); + } + if (pGcParam0) { + freeOperatorParam(pGcParam0, OP_GET_PARAM); + } + if (pGcParam1) { + freeOperatorParam(pGcParam1, OP_GET_PARAM); + } + if (*ppParam) { + freeOperatorParam(*ppParam, OP_GET_PARAM); + *ppParam = NULL; + } + } + return code; } @@ -488,7 +548,7 @@ static void handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCt if (pPost->leftNeedCache) { uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); - if (--(*num) <= 0) { + if (num && --(*num) <= 0) { tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); notifySeqJoinTableCacheEnd(pOperator, pPost, true); } diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index d61034c26e..3cfd0ab582 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -277,7 +277,7 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p SFilterColumnParam param2 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(pInfo->pEndCondInfo, ¶m2); if (code != TSDB_CODE_SUCCESS) { - return code; + goto _return; } int32_t status2 = 0; @@ -331,10 +331,12 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p } } +_return: + colDataDestroy(ps); taosMemoryFree(ps); colDataDestroy(pe); taosMemoryFree(pe); - return TSDB_CODE_SUCCESS; + return code; } diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index ff771ccc8c..d20ad1d67c 100755 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -223,6 +223,9 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S SGroupCacheFileInfo newFile = {0}; taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)); pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); + if (NULL == pTmp) { + return TSDB_CODE_OUT_OF_MEMORY; + } } if (pTmp->deleted) { @@ -287,7 +290,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC if (deleted) { qTrace("FileId:%d-%d-%d already be deleted, skip write", - pCtx->id, pGroup->vgId, pHead->basic.fileId); + pCtx->id, pGroup ? pGroup->vgId : GROUP_CACHE_DEFAULT_VGID, pHead->basic.fileId); int64_t blkId = pHead->basic.blkId; pHead = pHead->next; @@ -337,7 +340,9 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr return TSDB_CODE_OUT_OF_MEMORY; } pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId)); - + if (NULL == pBufInfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t code = TSDB_CODE_SUCCESS; SGcBlkBufInfo* pWriteHead = NULL; @@ -378,6 +383,10 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr static FORCE_INLINE void chkRemoveVgroupCurrFile(SGcFileCacheCtx* pFileCtx, int32_t downstreamIdx, int32_t vgId) { SGroupCacheFileInfo* pFileInfo = taosHashGet(pFileCtx->pCacheFile, &pFileCtx->fileId, sizeof(pFileCtx->fileId)); + if (NULL == pFileInfo) { + return; + } + if (0 == pFileInfo->groupNum) { removeGroupCacheFile(pFileInfo); @@ -711,6 +720,9 @@ static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int newFile.groupNum = 1; taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)); pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); + if (NULL == pTmp) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { pTmp->groupNum++; } @@ -786,6 +798,9 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* } *ppGrp = taosHashGet(pGrpHash, &uid, sizeof(uid)); + if (NULL == *ppGrp) { + return TSDB_CODE_OUT_OF_MEMORY; + } initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache); qError("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache); diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index d57f9f9f90..f382283d27 100755 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -636,12 +636,14 @@ static int32_t addRowToHashImpl(SHJoinOperatorInfo* pJoin, SGroupData* pGroup, S int32_t code = getValBufFromPages(pJoin->pRowBufs, getHJoinValBufSize(pTable, rowIdx), &pTable->valData, pRow); if (code) { + taosMemoryFree(pRow); return code; } if (NULL == pGroup) { pRow->next = NULL; if (tSimpleHashPut(pJoin->pKeyHash, pTable->keyData, keyLen, &group, sizeof(group))) { + taosMemoryFree(pRow); return TSDB_CODE_OUT_OF_MEMORY; } } else { diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 1b286f9bdd..2348a3c97b 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -711,6 +711,11 @@ static bool mergeJoinGetNextTimestamp(SOperatorInfo* pOperator, int64_t* pLeftTs } } } + + if (NULL == pJoinInfo->pLeft || NULL == pJoinInfo->pRight) { + setMergeJoinDone(pOperator); + return false; + } // only the timestamp match support for ordinary table SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e96ed87d3e..b24713fa41 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2883,7 +2883,7 @@ static EDealRes tagScanRewriteTagColumn(SNode** pNode, void* pContext) { } -static void tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aFilterIdxs, void* pVnode, SStorageAPI* pAPI, STagScanInfo* pInfo) { +static int32_t tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aFilterIdxs, void* pVnode, SStorageAPI* pAPI, STagScanInfo* pInfo) { int32_t code = 0; int32_t numOfTables = taosArrayGetSize(aUidTags); @@ -2894,9 +2894,15 @@ static void tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aF SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; SScalarParam output = {0}; - tagScanCreateResultData(&type, numOfTables, &output); + code = tagScanCreateResultData(&type, numOfTables, &output); + if (TSDB_CODE_SUCCESS == code) { + return code; + } - scalarCalculate(pTagCond, pBlockList, &output); + code = scalarCalculate(pTagCond, pBlockList, &output); + if (TSDB_CODE_SUCCESS == code) { + return code; + } bool* result = (bool*)output.columnData->pData; for (int32_t i = 0 ; i < numOfTables; ++i) { @@ -2911,7 +2917,7 @@ static void tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aF blockDataDestroy(pResBlock); taosArrayDestroy(pBlockList); - + return TSDB_CODE_SUCCESS; } static void tagScanFillOneCellWithTag(SOperatorInfo* pOperator, const STUidTagInfo* pUidTagInfo, SExprInfo* pExprInfo, SColumnInfoData* pColInfo, int rowIndex, const SStorageAPI* pAPI, void* pVnode) { @@ -3024,7 +3030,11 @@ static SSDataBlock* doTagScanFromCtbIdx(SOperatorInfo* pOperator) { bool ignoreFilterIdx = true; if (pInfo->pTagCond != NULL) { ignoreFilterIdx = false; - tagScanFilterByTagCond(aUidTags, pInfo->pTagCond, aFilterIdxs, pInfo->readHandle.vnode, pAPI, pInfo); + int32_t code = tagScanFilterByTagCond(aUidTags, pInfo->pTagCond, aFilterIdxs, pInfo->readHandle.vnode, pAPI, pInfo); + if (TSDB_CODE_SUCCESS != code) { + pOperator->pTaskInfo->code = code; + T_LONG_JMP(pOperator->pTaskInfo->env, code); + } } else { ignoreFilterIdx = true; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6c4a780dfb..287c824540 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -904,6 +904,7 @@ static int32_t getPageBufIncForRow(SSDataBlock* blk, int32_t row, int32_t rowIdx } static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockOrderInfo* order, SArray* aExtSrc) { + int32_t code = TSDB_CODE_SUCCESS; int pgHeaderSz = sizeof(int32_t) + sizeof(int32_t) * taosArrayGetSize(pHandle->pDataBlock->pDataBlock); int32_t rowCap = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, pgHeaderSz); blockDataEnsureCapacity(pHandle->pDataBlock, rowCap); @@ -930,7 +931,13 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); SMultiwayMergeTreeInfo* pTree = NULL; - tMergeTreeCreate(&pTree, taosArrayGetSize(aBlk), &sup, blockCompareTsFn); + code = tMergeTreeCreate(&pTree, taosArrayGetSize(aBlk), &sup, blockCompareTsFn); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); + + return code; + } int32_t nRows = 0; int32_t nMergedRows = 0; bool mergeLimitReached = false; @@ -1054,7 +1061,14 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { tSimpleHashClear(mUidBlk); int64_t p = taosGetTimestampUs(); - sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); + code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); + if (code != TSDB_CODE_SUCCESS) { + tSimpleHashCleanup(mUidBlk); + taosArrayDestroy(aBlkSort); + taosArrayDestroy(aExtSrc); + return code; + } + int64_t el = taosGetTimestampUs() - p; pHandle->sortElapsed += el; From c86eeb3938f09b2020c96f56cad33864a78685b0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 10:58:33 +0800 Subject: [PATCH 27/96] fix(stream): add task status check when waiting for the table create. --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 5 ++++- source/dnode/vnode/src/tq/tqSink.c | 14 +++++++++++--- source/dnode/vnode/src/tq/tqStreamTask.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 1 - source/libs/stream/src/streamQueue.c | 10 +++++----- 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3355e771e2..536273c044 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -250,7 +250,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, SRpcMsg* pMsg); +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4aee98148b..5d47237fb4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -890,9 +890,10 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { return streamSendCheckRsp(pTq->pStreamMeta, &req, &rsp, &pMsg->info, taskId); } -int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, SRpcMsg* pMsg) { +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) { char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t vgId = pTq->pStreamMeta->vgId; int32_t code; SStreamTaskCheckRsp rsp; @@ -901,7 +902,9 @@ int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, SRpcMsg* pMsg) { tDecoderInit(&decoder, (uint8_t*)pReq, len); code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); if (code < 0) { + terrno = TSDB_CODE_INVALID_MSG; tDecoderClear(&decoder); + tqError("vgId:%d failed to parse check rsp msg, code:%s", vgId, tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index f7132ff6c4..a738befec9 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -487,10 +487,18 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, pTableSinkInfo->uid = mr.me.uid; metaReaderClear(&mr); } - } else { // not exist, wait and retry + } else { // not exist, wait and retry metaReaderClear(&mr); - taosMsleep(100); - tqDebug("s-task:%s wait for the table:%s ready before insert data", id, dstTableName); + if (streamTaskShouldStop(&pTask->status)) { + tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); + taosArrayDestroy(tbData.aRowP); + taosArrayDestroy(pVals); + + return TSDB_CODE_SUCCESS; + } else { + taosMsleep(100); + tqDebug("s-task:%s wait 100ms for the table:%s ready before insert data", id, dstTableName); + } } } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index eb587b8be2..553ef06974 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -112,7 +112,7 @@ int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq) { int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); if (numOfTasks == 0) { - tqInfo("vgId:%d no stream tasks exist", vgId); + tqDebug("vgId:%d no stream tasks existed to run", vgId); taosWUnLockLatch(&pMeta->lock); return 0; } @@ -150,7 +150,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); if (numOfTasks == 0) { - tqInfo("vgId:%d no stream tasks exist", vgId); + tqDebug("vgId:%d no stream tasks existed to run", vgId); taosWUnLockLatch(&pMeta->lock); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0b7f969ed7..62822021a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -756,7 +756,7 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) case TDMT_VND_STREAM_TASK_CHECK: return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); case TDMT_VND_STREAM_TASK_CHECK_RSP: - return tqProcessStreamTaskCheckRsp(pVnode->pTq, 0, pMsg); + return tqProcessStreamTaskCheckRsp(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE: return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE_RSP: diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 82fa21ea40..8a80d74c63 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1591,7 +1591,6 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t return 0; } int streamStateOpenBackend(void* backend, SStreamState* pState) { - // qInfo("start to open state %p on backend %p 0x%" PRIx64 "-%d", pState, backend, pState->streamId, pState->taskId); taosAcquireRef(streamBackendId, pState->streamBackendRid); SBackendWrapper* handle = backend; SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 34b0a00639..78901a5ab1 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -264,9 +264,9 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pQueue)) { - qError( - "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", - pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); +// qError( +// "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", +// pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); taosFreeQitem(pItem); return -1; @@ -288,8 +288,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { if (streamQueueIsFull(pQueue)) { - qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", - pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); +// qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", +// pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); destroyStreamDataBlock((SStreamDataBlock*)pItem); return -1; } From e607a0a5cbe88aea45945818a60c6968d6862e12 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 11:39:07 +0800 Subject: [PATCH 28/96] fix(stream): add some logs. --- source/dnode/vnode/src/tq/tqRead.c | 6 +++--- source/libs/transport/src/transSvr.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0839a2bfa3..4f5701524d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -491,11 +491,11 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { - tqDebug("block found, ver:%" PRId64 ", uid:%" PRId64", %s", pReader->msg.ver, pSubmitTbData->uid, idstr); + tqDebug("block found, ver:%" PRId64 ", uid:%" PRId64 ", %s", pReader->msg.ver, pSubmitTbData->uid, idstr); return true; } else { - tqInfo("discard submit block, uid:%" PRId64 ", total queried tables:%d continue %s", pSubmitTbData->uid, - taosHashGetSize(pReader->tbIdHash), idstr); + tqDebug("discard submit block, uid:%" PRId64 ", total queried tables:%d continue %s", pSubmitTbData->uid, + taosHashGetSize(pReader->tbIdHash), idstr); } pReader->nextBlk++; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index a53830723c..3117bbf00e 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -395,7 +395,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { if (status == 0) { tTrace("success to dispatch conn to work thread"); } else { - tError("fail to dispatch conn to work thread"); + tError("fail to dispatch conn to work thread, code:%s", uv_strerror(status)); } if (!uv_is_closing((uv_handle_t*)req->data)) { uv_close((uv_handle_t*)req->data, uvFreeCb); From 2cfe615aa2f39c6f73d0cd36ce74009470786522 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:37:11 +0800 Subject: [PATCH 29/96] Update 14-java.mdx --- docs/zh/08-connector/14-java.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 0ff00d1710..64e9a3daed 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -1004,7 +1004,7 @@ TaosConsumer consumer = new TaosConsumer<>(config); - httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。 - messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。 - httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。 - 其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group) + 其他参数请参考：[Consumer 参数列表](../../develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 @@ -1082,7 +1082,7 @@ consumer.unsubscribe(); consumer.close() ``` -详情请参考：[数据订阅](../../../develop/tmq) +详情请参考：[数据订阅](../../develop/tmq) #### 完整示例 @@ -1373,7 +1373,7 @@ public static void main(String[] args) throws Exception { **解决方法**： 更换 taos-jdbcdriver 3.0.2+ 版本。 -其它问题请参考 [FAQ](../../../train-faq/faq) +其它问题请参考 [FAQ](../../train-faq/faq) ## API 参考 From 89a49a40a47b666a4aa0eab74c6107ce143211df Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:39:33 +0800 Subject: [PATCH 30/96] Update 26-rust.mdx --- docs/zh/08-connector/26-rust.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 3e51aa72bb..018552117e 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -352,7 +352,7 @@ client.put(&sml_data)? ### 数据订阅 -TDengine 通过消息队列 [TMQ](../../../taos-sql/tmq/) 启动一个订阅。 +TDengine 通过消息队列 [TMQ](../../taos-sql/tmq/) 启动一个订阅。 #### 创建 Topic @@ -491,7 +491,7 @@ let taos = pool.get()?; ## 常见问题 -请参考 [FAQ](../../../train-faq/faq) +请参考 [FAQ](../../train-faq/faq) ## API 参考 From 875a081adc06bd62a276b6d474802008084a34b6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 6 Sep 2023 14:11:24 +0800 Subject: [PATCH 31/96] fix: typo issue --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b24713fa41..12ec63b4cb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2895,12 +2895,12 @@ static int32_t tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* SScalarParam output = {0}; code = tagScanCreateResultData(&type, numOfTables, &output); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS != code) { return code; } code = scalarCalculate(pTagCond, pBlockList, &output); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS != code) { return code; } From 8783ebcb02b2840371d9ead18ff149f565b0fe40 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 14:34:36 +0800 Subject: [PATCH 32/96] enh(stream): add some logs and record the submit package performance. --- include/libs/stream/tstream.h | 8 ++ source/dnode/vnode/src/tq/tqSink.c | 109 +++++++++++++++++----------- source/libs/stream/src/streamTask.c | 2 +- 3 files changed, 76 insertions(+), 43 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index ea3524d12d..86e79835ac 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -312,10 +312,17 @@ typedef struct STaskSchedInfo { void* pTimer; } STaskSchedInfo; +typedef struct SSinkTaskRecorder { + int64_t numOfPackages; + int64_t numOfRows; +} SSinkTaskRecorder; + typedef struct { + int64_t created; int64_t init; int64_t step1Start; int64_t step2Start; + int64_t sinkStart; } STaskTimestamp; struct SStreamTask { @@ -345,6 +352,7 @@ struct SStreamTask { STaskSinkSma smaSink; STaskSinkFetch fetchSink; }; + SSinkTaskRecorder sinkRecorder; void* launchTaskTimer; SMsgCb* pMsgCb; // msg handle diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index a738befec9..5596c958e9 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -133,7 +133,6 @@ static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { return TSDB_CODE_SUCCESS; } - void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { const SArray* pBlocks = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; @@ -144,6 +143,10 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { int32_t numOfBlocks = taosArrayGetSize(pBlocks); int32_t code = TSDB_CODE_SUCCESS; + if (pTask->tsInfo.sinkStart == 0) { + pTask->tsInfo.sinkStart = taosGetTimestampMs(); + } + tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table", vgId, pTask->id.idStr, numOfBlocks); SArray* tagArray = NULL; @@ -259,10 +262,17 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { continue; } else { + pTask->sinkRecorder.numOfPackages += 1; code = doSinkResultBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask); } } + if ((pTask->sinkRecorder.numOfPackages % 5000) == 0) { + SSinkTaskRecorder* pRec = &pTask->sinkRecorder; + tqInfo("s-task:%s vgId:d write %d blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", pTask->id.idStr, + vgId, pRec->numOfPackages, pRec->numOfRows, (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); + } + tqDebug("vgId:%d, s-task:%s write results completed", vgId, pTask->id.idStr); _end: @@ -399,6 +409,55 @@ static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSi return code; } +static int32_t tqBuildSubmitReq(SSubmitTbData* pTableData, int32_t vgId, void** pMsg, int32_t* msgLen) { + int32_t code = 0; + void* pBuf = NULL; + *msgLen = 0; + + SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; + if (submitReq.aSubmitTbData == NULL) { + tDestroySubmitTbData(pTableData, TSDB_MSG_FLG_ENCODE); + taosArrayDestroy(pTableData->aRowP); + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(submitReq.aSubmitTbData, pTableData); + + // encode + int32_t len = 0; + tEncodeSize(tEncodeSubmitReq, &submitReq, len, code); + + SEncoder encoder; + len += sizeof(SSubmitReq2Msg); + + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + return TSDB_CODE_OUT_OF_MEMORY; + } + + ((SSubmitReq2Msg*)pBuf)->header.vgId = vgId; + ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); + ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); + + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); + if (tEncodeSubmitReq(&encoder, &submitReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req, code:%s, ignore and continue", terrstr()); + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + return code; + } + + tEncoderClear(&encoder); + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + + *msgLen = len; + *pMsg = pBuf; + return TSDB_CODE_SUCCESS; +} + int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, SSDataBlock* pDataBlock, SStreamTask* pTask) { int32_t numOfRows = pDataBlock->info.rows; @@ -406,7 +465,6 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, uint64_t groupId = pDataBlock->info.id.groupId; STSchema* pTSchema = pTask->tbSink.pTSchema; int32_t code = TSDB_CODE_SUCCESS; - void* pBuf = NULL; SArray* pVals = NULL; const char* id = pTask->id.idStr; @@ -573,7 +631,7 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, void* colData = colDataGetData(pColData, j); if (IS_STR_DATA_TYPE(pCol->type)) { // address copy, no value - SValue sv = (SValue){.nData = varDataLen(colData), .pData = varDataVal(colData)}; + SValue sv = (SValue){.nData = varDataLen(colData), .pData = (uint8_t*) varDataVal(colData)}; SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); taosArrayPush(pVals, &cv); } else { @@ -601,53 +659,20 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, taosArrayPush(tbData.aRowP, &pRow); } - SSubmitReq2 submitReq = {0}; - if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { - tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); - - taosArrayDestroy(tbData.aRowP); - taosArrayDestroy(pVals); - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosArrayPush(submitReq.aSubmitTbData, &tbData); - - // encode int32_t len = 0; - tEncodeSize(tEncodeSubmitReq, &submitReq, len, code); - - SEncoder encoder; - len += sizeof(SSubmitReq2Msg); - - pBuf = rpcMallocCont(len); - if (NULL == pBuf) { - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); - taosArrayDestroy(tbData.aRowP); + void* pBuf = NULL; + code = tqBuildSubmitReq(&tbData, vgId, &pBuf, &len); + if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pVals); - } - - ((SSubmitReq2Msg*)pBuf)->header.vgId = vgId; - ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); - ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); - - tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); - if (tEncodeSubmitReq(&encoder, &submitReq) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to encode submit req, code:%s, ignore and continue", terrstr()); - tEncoderClear(&encoder); - rpcFreeCont(pBuf); - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); - return code; } - tEncoderClear(&encoder); - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + pTask->sinkRecorder.numOfRows += numOfRows; - SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len }; + SRpcMsg msg = {.msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len}; code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); - if(code == TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_SUCCESS) { tqDebug("s-task:%s send submit msg to dstTable:%s, numOfRows:%d", id, pTableSinkInfo->name.data, numOfRows); } else { tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 711dbf65e7..7ce5040257 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -375,7 +375,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i return -1; } - pTask->tsInfo.init = taosGetTimestampMs(); + pTask->tsInfo.created = taosGetTimestampMs(); pTask->inputInfo.status = TASK_INPUT_STATUS__NORMAL; pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; pTask->pMeta = pMeta; From 236b363996e49f0b88b8a9c32075b713fbfba129 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 14:37:11 +0800 Subject: [PATCH 33/96] fix(stream): fix syntax error. --- source/dnode/vnode/src/tq/tqSink.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5596c958e9..82703260ed 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -269,8 +269,9 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { if ((pTask->sinkRecorder.numOfPackages % 5000) == 0) { SSinkTaskRecorder* pRec = &pTask->sinkRecorder; - tqInfo("s-task:%s vgId:d write %d blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", pTask->id.idStr, - vgId, pRec->numOfPackages, pRec->numOfRows, (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); + tqInfo("s-task:%s vgId:d write %" PRId64 " blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", + pTask->id.idStr, vgId, pRec->numOfPackages, pRec->numOfRows, + (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); } tqDebug("vgId:%d, s-task:%s write results completed", vgId, pTask->id.idStr); From b382cba58ce6fe17327ef0d063c43e34ef9c5df5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 14:37:58 +0800 Subject: [PATCH 34/96] fix(stream): fix syntax error. --- source/dnode/vnode/src/tq/tqSink.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 82703260ed..2df1dbe24f 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -269,7 +269,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { if ((pTask->sinkRecorder.numOfPackages % 5000) == 0) { SSinkTaskRecorder* pRec = &pTask->sinkRecorder; - tqInfo("s-task:%s vgId:d write %" PRId64 " blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", + tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", pTask->id.idStr, vgId, pRec->numOfPackages, pRec->numOfRows, (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); } From cd88c8ec311fb4772fedcba79e84ebd6d91e5600 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 6 Sep 2023 14:45:53 +0800 Subject: [PATCH 35/96] fix: little test --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 9 ++++++--- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index c9cbd37b09..dc5e3649cc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -982,7 +982,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TS goto _exit; } - if (writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray)) { + for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); if (brinBlk->minTbid.uid != writer->ctx->tbid->uid) { @@ -995,6 +995,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TS writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; + break; } } @@ -1111,7 +1112,7 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA break; } - if (writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray)) { + for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); @@ -1119,6 +1120,7 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; + break; } } @@ -1251,7 +1253,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom break; } - if (writer->ctx->tombBlkArrayIdx < TARRAY2_SIZE(writer->ctx->tombBlkArray)) { + for (; writer->ctx->tombBlkArrayIdx < TARRAY2_SIZE(writer->ctx->tombBlkArray); ++writer->ctx->tombBlkArrayIdx) { const STombBlk *tombBlk = TARRAY2_GET_PTR(writer->ctx->tombBlkArray, writer->ctx->tombBlkArrayIdx); code = tsdbDataFileReadTombBlock(writer->ctx->reader, tombBlk, writer->ctx->tombBlock); @@ -1259,6 +1261,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom writer->ctx->tombBlockIdx = 0; writer->ctx->tombBlkArrayIdx++; + break; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 03a14a7162..37c7e2ffc1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -47,7 +47,7 @@ static int32_t tsdbSttLvlInitEx(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lvl } code = TARRAY2_APPEND(lvl[0]->fobjArr, fobj); - return code; + if (code) return code; } return 0; } From 7257518dcaf5f39cd553413173acf929c5cb03fd Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 5 Sep 2023 19:21:24 +0800 Subject: [PATCH 36/96] fix: tdbBtcDelete release pages before modifying pBtc->pPage --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++++--- source/libs/tdb/src/db/tdbPCache.c | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a48bae002e..7a62b38b16 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -2200,10 +2200,15 @@ int tdbBtcDelete(SBTC *pBtc) { tdbOsFree(pCell); if (pPage->nOverflow > 0) { - tdbDebug("tdb/btc-delete: btree balance after update cell, pPage/nOverflow: %p/%d.", pPage, - pPage->nOverflow); + tdbDebug("tdb/btc-delete: btree balance after update cell, pPage/nOverflow/pgno: %p/%d/%" PRIu32 ".", pPage, + pPage->nOverflow, TDB_PAGE_PGNO(pPage)); - pBtc->iPage = iPage; + tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn); + while (--pBtc->iPage != iPage) { + tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pgStack[pBtc->iPage], pBtc->pTxn); + } + + // pBtc->iPage = iPage; pBtc->pPage = pPage; ret = tdbBtreeBalance(pBtc); if (ret < 0) { diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 262f3d27e6..5b8c407e95 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -295,7 +295,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) } // 1. pPage == NULL - // 2. pPage && pPage->isLocal == 0 && !TDB_TXN_IS_WRITE(pTxn) + // 2. pPage && !pPage->isLocal == 0 && !TDB_TXN_IS_WRITE(pTxn) pPageH = pPage; pPage = NULL; From 060f97ff17ee0cca9dd4a5142ec237706bb4f50d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 6 Sep 2023 16:14:50 +0800 Subject: [PATCH 37/96] install taos-ws-py for ci --- Jenkinsfile2 | 4 +++- tests/ci/Dockerfile | 2 +- tests/parallel_test/run_case.sh | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index f4dcdb242e..c41e739bd3 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -315,7 +315,9 @@ def pre_test_build_win() { python.exe -m pip install --upgrade pip python -m pip uninstall taospy -y python -m pip install taospy==2.7.10 - xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + python -m pip uninstall taos-ws-py -y + python -m pip install taos-ws-py==0.2.8 + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' return 1 } diff --git a/tests/ci/Dockerfile b/tests/ci/Dockerfile index 38cab07792..213570dfb2 100644 --- a/tests/ci/Dockerfile +++ b/tests/ci/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.8 RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro taos-ws-py +RUN pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro RUN apt-get update RUN apt-get install -y psmisc sudo tree libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config build-essential valgrind \ vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 206f99ff3d..94928e74ae 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -81,6 +81,11 @@ pip3 list|grep taospy pip3 uninstall taospy -y pip3 install --default-timeout=120 taospy==2.7.10 +#define taos-ws-py 0.2.8 +pip3 list|grep taos-ws-py +pip3 uninstall taos-ws-py -y +pip3 install --default-timeout=120 taos-ws-py==0.2.8 + $TIMEOUT_CMD $cmd RET=$? echo "cmd exit code: $RET" From 926cc01cff614763f14629393861334db5affdd7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 16:48:43 +0800 Subject: [PATCH 38/96] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqSink.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 2df1dbe24f..ee073e6bc4 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -147,7 +147,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { pTask->tsInfo.sinkStart = taosGetTimestampMs(); } - tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table", vgId, pTask->id.idStr, numOfBlocks); + tqInfo("vgId:%d, s-task:%s write %d stream resBlock(s) into table", vgId, pTask->id.idStr, numOfBlocks); SArray* tagArray = NULL; SArray* pVals = NULL; @@ -401,7 +401,6 @@ static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSi int32_t code = tqPutTableInfo(pSinkTableMap, groupId, pTableSinkInfo); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableSinkInfo); - tqError("s-task:%s failed to put tableSinkInfo in to cache, code:%s", id, tstrerror(code)); } else { tqDebug("s-task:%s new dst table:%s(uid:%" PRIu64 ") added into cache, total:%d", id, pTableSinkInfo->name.data, pTableSinkInfo->uid, tSimpleHashGetSize(pSinkTableMap)); @@ -674,7 +673,7 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); if (code == TSDB_CODE_SUCCESS) { - tqDebug("s-task:%s send submit msg to dstTable:%s, numOfRows:%d", id, pTableSinkInfo->name.data, numOfRows); + tqDebug("s-task:%s send submit msg to dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); } else { tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); } From 1d343fa57e008e1e7ecafd4f27711693d91c4f67 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 6 Sep 2023 17:10:36 +0800 Subject: [PATCH 39/96] more fix --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index dc5e3649cc..7e5eb2c553 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -980,9 +980,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TS writer->ctx->brinBlkArray = NULL; writer->ctx->tbHasOldData = false; goto _exit; - } - - for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + } else { const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); if (brinBlk->minTbid.uid != writer->ctx->tbid->uid) { @@ -995,7 +993,6 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TS writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; - break; } } @@ -1110,9 +1107,7 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA if (writer->ctx->brinBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->brinBlkArray)) { writer->ctx->brinBlkArray = NULL; break; - } - - for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + } else { const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); @@ -1120,7 +1115,6 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; - break; } } @@ -1251,9 +1245,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom if (writer->ctx->tombBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->tombBlkArray)) { writer->ctx->hasOldTomb = false; break; - } - - for (; writer->ctx->tombBlkArrayIdx < TARRAY2_SIZE(writer->ctx->tombBlkArray); ++writer->ctx->tombBlkArrayIdx) { + } else { const STombBlk *tombBlk = TARRAY2_GET_PTR(writer->ctx->tombBlkArray, writer->ctx->tombBlkArrayIdx); code = tsdbDataFileReadTombBlock(writer->ctx->reader, tombBlk, writer->ctx->tombBlock); @@ -1261,7 +1253,6 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom writer->ctx->tombBlockIdx = 0; writer->ctx->tombBlkArrayIdx++; - break; } } From 7782530e3a4b966451430a1c37caff0ea9e60e41 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 6 Sep 2023 17:15:04 +0800 Subject: [PATCH 40/96] test:modify method of stopall function --- tests/pytest/util/dnodes.py | 71 +++++++++++++++++++------------------ tests/system-test/test.py | 2 +- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index cd873ca0f2..c4fc1ce654 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -668,7 +668,7 @@ class TDDnodes: self.testCluster = False self.valgrind = 0 self.asan = False - self.killValgrind = 1 + self.killValgrind = 0 def init(self, path, remoteIP = ""): binPath = self.dnodes[0].getPath() + "/../../../" @@ -775,9 +775,41 @@ class TDDnodes: tdLog.info("execute finished") return + def killProcesser(self, processerName): + if platform.system().lower() == 'windows': + killCmd = ("wmic process where name=\"%s.exe\" call terminate > NUL 2>&1" % processerName) + psCmd = ("wmic process where name=\"%s.exe\" | findstr \"%s.exe\"" % (processerName, processerName)) + else: + killCmd = ( + "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" + % processerName + ) + psCmd = ("ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % processerName) + + processID = "" + + try: + processID = subprocess.check_output(psCmd, shell=True) + while processID: + os.system(killCmd) + time.sleep(1) + try: + processID = subprocess.check_output(psCmd, shell=True) + except Exception as err: + processID = "" + tdLog.debug('**** kill pid warn: {err}') + except Exception as err: + processID = "" + tdLog.debug(f'**** find pid warn: {err}') + + + def stopAll(self): tdLog.info("stop all dnodes, asan:%d" % self.asan) - distro_id = distro.id() + if platform.system().lower() != 'windows': + distro_id = distro.id() + else: + distro_id = "not alpine" if self.asan and distro_id != "alpine": tdLog.info("execute script: %s" % self.stopDnodesPath) os.system(self.stopDnodesPath) @@ -792,7 +824,6 @@ class TDDnodes: if (distro_id == "alpine"): - print(distro_id) psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}' | xargs" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() while(processID): @@ -803,36 +834,9 @@ class TDDnodes: processID = subprocess.check_output( psCmd, shell=True).decode("utf-8").strip() elif platform.system().lower() == 'windows': - psCmd = "for /f %a in ('wmic process where \"name='taosd.exe'\" get processId ^| xargs echo ^| awk '{print $2}' ^&^& echo aa') do @(ps | grep %a | awk '{print $1}' | xargs)" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - while(processID): - print(f"pid of taosd.exe:{processID}") - killCmd = "kill -9 %s > nul 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8").strip() - - psCmd = "for /f %a in ('wmic process where \"name='tmq_sim.exe'\" get processId ^| xargs echo ^| awk '{print $2}' ^&^& echo aa') do @(ps | grep %a | awk '{print $1}' | xargs)" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - while(processID): - print(f"pid of tmq_sim.exe:{processID}") - killCmd = "kill -9 %s > nul 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8").strip() - - psCmd = "for /f %a in ('wmic process where \"name='taosBenchmark.exe'\" get processId ^| xargs echo ^| awk '{print $2}' ^&^& echo aa') do @(ps | grep %a | awk '{print $1}' | xargs)" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - while(processID): - print(f"pid of taosBenchmark.exe:{processID}") - killCmd = "kill -9 %s > nul 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8").strip() - + self.killProcesser("taosd") + self.killProcesser("tmq_sim") + self.killProcesser("taosBenchmark") else: psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}' | xargs" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() @@ -849,7 +853,6 @@ class TDDnodes: time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8").strip() - if self.killValgrind == 1: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}' | xargs" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 1c50e5bbbe..b1625997b4 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -177,7 +177,7 @@ if __name__ == "__main__": sys.exit(0) if key in ['-k', '--killValgrind']: - killValgrind = 0 + killValgrind = 1 if key in ['-e', '--execCmd']: try: From a82027f4925e5ccc662450d1f27efded351ed7fe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 18:58:47 +0800 Subject: [PATCH 41/96] enh(stream): pack multi-datablock into one submit message. --- source/dnode/vnode/src/tq/tqSink.c | 155 +++++++++++++++-------------- 1 file changed, 82 insertions(+), 73 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index ee073e6bc4..512007d578 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -25,9 +25,10 @@ typedef struct STableSinkInfo { } STableSinkInfo; static int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, - SSDataBlock* pDataBlock, SStreamTask* pTask); + SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData); static int32_t doSinkDeleteBlock(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid); +static int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr) { @@ -142,17 +143,25 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { int32_t vgId = TD_VID(pVnode); int32_t numOfBlocks = taosArrayGetSize(pBlocks); int32_t code = TSDB_CODE_SUCCESS; + const char* id = pTask->id.idStr; if (pTask->tsInfo.sinkStart == 0) { pTask->tsInfo.sinkStart = taosGetTimestampMs(); } - tqInfo("vgId:%d, s-task:%s write %d stream resBlock(s) into table", vgId, pTask->id.idStr, numOfBlocks); + tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table", vgId, id, numOfBlocks); + bool hasSubmit = false; SArray* tagArray = NULL; SArray* pVals = NULL; SArray* crTblArray = NULL; + SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; + if (submitReq.aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + for (int32_t i = 0; i < numOfBlocks; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); int32_t rows = pDataBlock->info.rows; @@ -262,19 +271,46 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { continue; } else { + hasSubmit = true; pTask->sinkRecorder.numOfPackages += 1; - code = doSinkResultBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask); + + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; + code = doSinkResultBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; } } - if ((pTask->sinkRecorder.numOfPackages % 5000) == 0) { - SSinkTaskRecorder* pRec = &pTask->sinkRecorder; - tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", - pTask->id.idStr, vgId, pRec->numOfPackages, pRec->numOfRows, - (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); - } + if (hasSubmit) { + int32_t len = 0; + void* pBuf = NULL; + code = tqBuildSubmitReq(&submitReq, vgId, &pBuf, &len); - tqDebug("vgId:%d, s-task:%s write results completed", vgId, pTask->id.idStr); + if (code != TSDB_CODE_SUCCESS) { + tqError("s-task:%s build submit msg failed, vgId:%d, code:%s", id, vgId, tstrerror(code)); + goto _end; + } + + SRpcMsg msg = {.msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len}; + code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); + + if (code == TSDB_CODE_SUCCESS) { + tqDebug("s-task:%s vgId:%d send submit %d blocks(%d rows) into dstTables completed", id, vgId); + } else { + tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); + } + + if ((pTask->sinkRecorder.numOfPackages % 5000) == 0) { + SSinkTaskRecorder* pRec = &pTask->sinkRecorder; + tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", + pTask->id.idStr, vgId, pRec->numOfPackages, pRec->numOfRows, + (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); + } + } else { + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); + } _end: taosArrayDestroy(tagArray); @@ -409,30 +445,21 @@ static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSi return code; } -static int32_t tqBuildSubmitReq(SSubmitTbData* pTableData, int32_t vgId, void** pMsg, int32_t* msgLen) { +int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) { int32_t code = 0; void* pBuf = NULL; *msgLen = 0; - SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; - if (submitReq.aSubmitTbData == NULL) { - tDestroySubmitTbData(pTableData, TSDB_MSG_FLG_ENCODE); - taosArrayDestroy(pTableData->aRowP); - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosArrayPush(submitReq.aSubmitTbData, pTableData); - // encode int32_t len = 0; - tEncodeSize(tEncodeSubmitReq, &submitReq, len, code); + tEncodeSize(tEncodeSubmitReq, pSubmitReq, len, code); SEncoder encoder; len += sizeof(SSubmitReq2Msg); pBuf = rpcMallocCont(len); if (NULL == pBuf) { - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pSubmitReq, TSDB_MSG_FLG_ENCODE); return TSDB_CODE_OUT_OF_MEMORY; } @@ -441,17 +468,17 @@ static int32_t tqBuildSubmitReq(SSubmitTbData* pTableData, int32_t vgId, void** ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); - if (tEncodeSubmitReq(&encoder, &submitReq) < 0) { + if (tEncodeSubmitReq(&encoder, pSubmitReq) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to encode submit req, code:%s, ignore and continue", terrstr()); tEncoderClear(&encoder); rpcFreeCont(pBuf); - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pSubmitReq, TSDB_MSG_FLG_ENCODE); return code; } tEncoderClear(&encoder); - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pSubmitReq, TSDB_MSG_FLG_ENCODE); *msgLen = len; *pMsg = pBuf; @@ -459,7 +486,7 @@ static int32_t tqBuildSubmitReq(SSubmitTbData* pTableData, int32_t vgId, void** } int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, SSDataBlock* pDataBlock, - SStreamTask* pTask) { + SStreamTask* pTask, SSubmitTbData* pTableData) { int32_t numOfRows = pDataBlock->info.rows; int32_t vgId = TD_VID(pVnode); uint64_t groupId = pDataBlock->info.id.groupId; @@ -468,15 +495,14 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, SArray* pVals = NULL; const char* id = pTask->id.idStr; - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - tqDebug("s-task:%s sink data pipeline, build submit msg from %d-th resBlock, including %d rows, dst suid:%" PRId64, + tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, id, blockIndex + 1, numOfRows, suid); - tbData.aRowP = taosArrayInit(numOfRows, sizeof(SRow*)); + pTableData->aRowP = taosArrayInit(numOfRows, sizeof(SRow*)); pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); - if (tbData.aRowP == NULL || pVals == NULL) { - taosArrayDestroy(tbData.aRowP); + if (pTableData->aRowP == NULL || pVals == NULL) { + taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); code = TSDB_CODE_OUT_OF_MEMORY; @@ -517,13 +543,21 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, } if (exist) { - tbData.uid = pTableSinkInfo->uid; + pTableData->uid = pTableSinkInfo->uid; - if (tbData.uid == 0) { + if (pTableData->uid == 0) { tqDebug("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); } while (pTableSinkInfo->uid == 0) { + if (streamTaskShouldStop(&pTask->status)) { + tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); + taosArrayDestroy(pTableData->aRowP); + taosArrayDestroy(pVals); + + return TSDB_CODE_SUCCESS; + } + // wait for the table to be created SMetaReader mr = {0}; metaReaderDoInit(&mr, pVnode->pMeta, 0); @@ -534,29 +568,21 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, if (!isValid) { // not valid table, ignore it metaReaderClear(&mr); - taosArrayDestroy(tbData.aRowP); + taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return TSDB_CODE_SUCCESS; } else { tqDebug("s-task:%s set uid:%"PRIu64" for dstTable:%s from meta", id, mr.me.uid, pTableSinkInfo->name.data); - tbData.uid = mr.me.uid; + pTableData->uid = mr.me.uid; pTableSinkInfo->uid = mr.me.uid; metaReaderClear(&mr); } } else { // not exist, wait and retry metaReaderClear(&mr); - if (streamTaskShouldStop(&pTask->status)) { - tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); - taosArrayDestroy(tbData.aRowP); - taosArrayDestroy(pVals); - - return TSDB_CODE_SUCCESS; - } else { - taosMsleep(100); - tqDebug("s-task:%s wait 100ms for the table:%s ready before insert data", id, dstTableName); - } + taosMsleep(100); + tqDebug("s-task:%s wait 100ms for the table:%s ready before insert data", id, dstTableName); } } @@ -576,12 +602,12 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); - tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; - tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock); - if (tbData.pCreateTbReq == NULL) { + pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock); + if (pTableData->pCreateTbReq == NULL) { tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); - taosArrayDestroy(tbData.aRowP); + taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return terrno; @@ -593,14 +619,14 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, if (!isValid) { metaReaderClear(&mr); taosMemoryFree(pTableSinkInfo); - taosArrayDestroy(tbData.aRowP); + taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return TSDB_CODE_SUCCESS; } else { - tbData.uid = mr.me.uid; + pTableData->uid = mr.me.uid; metaReaderClear(&mr); - doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, tbData.uid, id); + doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, pTableData->uid, id); } } } @@ -648,35 +674,18 @@ int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, SRow* pRow = NULL; code = tRowBuild(pVals, (STSchema*)pTSchema, &pRow); if (code != TSDB_CODE_SUCCESS) { - tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(pTableData, TSDB_MSG_FLG_ENCODE); - taosArrayDestroy(tbData.aRowP); + taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return code; } ASSERT(pRow); - taosArrayPush(tbData.aRowP, &pRow); + taosArrayPush(pTableData->aRowP, &pRow); } - int32_t len = 0; - void* pBuf = NULL; - code = tqBuildSubmitReq(&tbData, vgId, &pBuf, &len); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pVals); - return code; - } - - pTask->sinkRecorder.numOfRows += numOfRows; - - SRpcMsg msg = {.msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len}; - code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); - - if (code == TSDB_CODE_SUCCESS) { - tqDebug("s-task:%s send submit msg to dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); - } else { - tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); - } + tqDebug("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); taosArrayDestroy(pVals); return code; From 5ba6a68e7a9303de72afc5637b7e4832211f57be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 19:00:39 +0800 Subject: [PATCH 42/96] fix(stream): fix syntax error. --- source/dnode/vnode/src/tq/tqSink.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 512007d578..6347b9cadd 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -294,9 +294,8 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { SRpcMsg msg = {.msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len}; code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); - if (code == TSDB_CODE_SUCCESS) { - tqDebug("s-task:%s vgId:%d send submit %d blocks(%d rows) into dstTables completed", id, vgId); + tqDebug("s-task:%s vgId:%d send submit %d blocks into dstTables completed", id, vgId, numOfBlocks); } else { tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); } From 8edf86b3bb8a823dc699403972c1c78f3209a4f3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 19:08:09 +0800 Subject: [PATCH 43/96] fix(stream): add some logs. --- include/libs/stream/tstream.h | 3 ++- source/dnode/vnode/src/tq/tqSink.c | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 86e79835ac..d36231c018 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -313,7 +313,8 @@ typedef struct STaskSchedInfo { } STaskSchedInfo; typedef struct SSinkTaskRecorder { - int64_t numOfPackages; + int64_t numOfSubmit; + int64_t numOfBlocks; int64_t numOfRows; } SSinkTaskRecorder; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6347b9cadd..45991d83c8 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -272,7 +272,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { continue; } else { hasSubmit = true; - pTask->sinkRecorder.numOfPackages += 1; + pTask->sinkRecorder.numOfBlocks += 1; SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; code = doSinkResultBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); @@ -300,10 +300,13 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); } - if ((pTask->sinkRecorder.numOfPackages % 5000) == 0) { + pTask->sinkRecorder.numOfSubmit += 1; + + if ((pTask->sinkRecorder.numOfSubmit % 5000) == 0) { SSinkTaskRecorder* pRec = &pTask->sinkRecorder; - tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) into dst table, duration:%.2fSec.", - pTask->id.idStr, vgId, pRec->numOfPackages, pRec->numOfRows, + tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) in %" PRId64 + " submit into dst table, duration:%.2f Sec.", + pTask->id.idStr, vgId, pRec->numOfBlocks, pRec->numOfRows, pRec->numOfSubmit, (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); } } else { From 2b84e0b02e3f192d008ef28752013c9435df619e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Sep 2023 19:39:00 +0800 Subject: [PATCH 44/96] refactor: check if the put data into queue is success or failed. --- source/dnode/vnode/src/tq/tqSink.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 45991d83c8..449d460b02 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -24,7 +24,7 @@ typedef struct STableSinkInfo { tstr name; } STableSinkInfo; -static int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, +static int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData); static int32_t doSinkDeleteBlock(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid); @@ -275,7 +275,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { pTask->sinkRecorder.numOfBlocks += 1; SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - code = doSinkResultBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); + code = doBuildSubmitFromResBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); taosArrayPush(submitReq.aSubmitTbData, &tbData); pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; @@ -297,6 +297,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { if (code == TSDB_CODE_SUCCESS) { tqDebug("s-task:%s vgId:%d send submit %d blocks into dstTables completed", id, vgId, numOfBlocks); } else { + ASSERT(0); tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); } @@ -487,7 +488,7 @@ int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int return TSDB_CODE_SUCCESS; } -int32_t doSinkResultBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, SSDataBlock* pDataBlock, +int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData) { int32_t numOfRows = pDataBlock->info.rows; int32_t vgId = TD_VID(pVnode); From b240e91c911f9ce0b4c57f99bd08bbd8d6038ceb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Sep 2023 09:58:48 +0800 Subject: [PATCH 45/96] enh(stream): add token bucket to limit the rate of sink operation. --- include/libs/stream/tstream.h | 8 +++++ source/dnode/vnode/src/tq/tqSink.c | 6 ++-- source/libs/stream/inc/streamInt.h | 2 ++ source/libs/stream/src/streamQueue.c | 49 ++++++++++++++++++++++++++++ source/libs/stream/src/streamTask.c | 1 + 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index d36231c018..79d427e5c7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -326,6 +326,13 @@ typedef struct { int64_t sinkStart; } STaskTimestamp; +typedef struct STokenBucket { + int32_t capacity; // total capacity + int64_t fillTimestamp;// fill timestamp + int32_t numOfToken; // total available tokens + int32_t rate; // number of token per second +} STokenBucket; + struct SStreamTask { int64_t ver; SStreamTaskId id; @@ -354,6 +361,7 @@ struct SStreamTask { STaskSinkFetch fetchSink; }; SSinkTaskRecorder sinkRecorder; + STokenBucket tokenBucket; void* launchTaskTimer; SMsgCb* pMsgCb; // msg handle diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 449d460b02..1e714a8cad 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -25,10 +25,12 @@ typedef struct STableSinkInfo { } STableSinkInfo; static int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, - SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData); + SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData); static int32_t doSinkDeleteBlock(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid); static int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); +static void fillBucket(STokenBucket* pBucket); +static bool hasAvailableToken(STokenBucket* pBucket); int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr) { @@ -692,4 +694,4 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF taosArrayDestroy(pVals); return code; -} \ No newline at end of file +} diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index bb81582a2d..bbb7595e5a 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -77,6 +77,8 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); int32_t streamTaskFillHistoryFinished(SStreamTask* pTask); int32_t streamTransferStateToStreamTask(SStreamTask* pTask); +int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t cap, int32_t rate); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 78901a5ab1..0a0573cc90 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -29,6 +29,8 @@ typedef struct SQueueReader { int32_t waitDuration; // maximum wait time to format several block into a batch to process, unit: ms } SQueueReader; +static bool streamTaskHasAvailableToken(STokenBucket* pBucket); + static void streamQueueCleanup(SStreamQueue* pQueue) { void* qItem = NULL; while ((qItem = streamQueueNextItem(pQueue)) != NULL) { @@ -175,6 +177,14 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu return TSDB_CODE_SUCCESS; } + STokenBucket* pBucket = &pTask->tokenBucket; + bool has = streamTaskHasAvailableToken(pBucket); + if (!has) { // no available token in th bucket, ignore this execution + qInfo("s-task:%s no available token for sink, capacity:%d, rate:%d token/sec, quit", pTask->id.idStr, + pBucket->capacity, pBucket->rate); + return TSDB_CODE_SUCCESS; + } + SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputInfo.queue); if (qItem == NULL) { qDebug("===stream===break batchSize:%d, %s", *numOfBlocks, id); @@ -320,3 +330,42 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) return 0; } + +int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t cap, int32_t rate) { + if (cap < 100 || rate < 50 || pBucket == NULL) { + qError("failed to init sink task bucket, cap:%d, rate:%d", cap, rate); + return TSDB_CODE_INVALID_PARA; + } + + pBucket->capacity = cap; + pBucket->rate = rate; + pBucket->numOfToken = cap; + pBucket->fillTimestamp = taosGetTimestampMs(); + return TSDB_CODE_SUCCESS; +} + +static void fillBucket(STokenBucket* pBucket) { + int64_t now = taosGetTimestampMs(); + int64_t delta = now - pBucket->fillTimestamp; + + int32_t inc = (delta / 1000.0) * pBucket->rate; + if (inc > 0) { + if ((pBucket->numOfToken + inc) < pBucket->capacity) { + pBucket->numOfToken += inc; + } else { + pBucket->numOfToken = pBucket->capacity; + } + + pBucket->fillTimestamp = now; + } +} + +bool streamTaskHasAvailableToken(STokenBucket* pBucket) { + fillBucket(pBucket); + bool hasToken = pBucket->numOfToken > 0; + if (hasToken) { + return true; + } else { + return false; + } +} \ No newline at end of file diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 7ce5040257..24b8933387 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -385,6 +385,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->dataRange.range.minVer = ver; pTask->pMsgCb = pMsgCb; + streamTaskInitTokenBucket(&pTask->tokenBucket, 400, 200); taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); From 956047a441be733f435724ce23e4d6fc465f9fee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Sep 2023 10:02:13 +0800 Subject: [PATCH 46/96] fix(stream): count down the token in bucket. --- source/libs/stream/src/streamQueue.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 0a0573cc90..202afa5854 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -362,8 +362,9 @@ static void fillBucket(STokenBucket* pBucket) { bool streamTaskHasAvailableToken(STokenBucket* pBucket) { fillBucket(pBucket); - bool hasToken = pBucket->numOfToken > 0; + bool hasToken = (--pBucket->numOfToken) > 0; if (hasToken) { + qInfo("remain token:%d", pBucket->numOfToken); return true; } else { return false; From e407d7dc6bbd2a5752c578e9d3af1b83559e7c86 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Sep 2023 10:09:05 +0800 Subject: [PATCH 47/96] fix(stream): update log. --- source/libs/stream/src/streamQueue.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 202afa5854..acd34132d5 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -180,8 +180,8 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu STokenBucket* pBucket = &pTask->tokenBucket; bool has = streamTaskHasAvailableToken(pBucket); if (!has) { // no available token in th bucket, ignore this execution - qInfo("s-task:%s no available token for sink, capacity:%d, rate:%d token/sec, quit", pTask->id.idStr, - pBucket->capacity, pBucket->rate); +// qInfo("s-task:%s no available token for sink, capacity:%d, rate:%d token/sec, quit", pTask->id.idStr, +// pBucket->capacity, pBucket->rate); return TSDB_CODE_SUCCESS; } @@ -357,6 +357,7 @@ static void fillBucket(STokenBucket* pBucket) { } pBucket->fillTimestamp = now; + qDebug("new token available, ts:%"PRId64, now); } } From dc821057e5275a673d10864f4238128385066393 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Sep 2023 10:33:38 +0800 Subject: [PATCH 48/96] fix(stream): update the log. --- source/libs/stream/src/streamQueue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index acd34132d5..f2b50a024f 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -357,7 +357,7 @@ static void fillBucket(STokenBucket* pBucket) { } pBucket->fillTimestamp = now; - qDebug("new token available, ts:%"PRId64, now); + qInfo("new token available, ts:%"PRId64, now); } } @@ -365,7 +365,7 @@ bool streamTaskHasAvailableToken(STokenBucket* pBucket) { fillBucket(pBucket); bool hasToken = (--pBucket->numOfToken) > 0; if (hasToken) { - qInfo("remain token:%d", pBucket->numOfToken); + qDebug("remain token:%d", pBucket->numOfToken); return true; } else { return false; From 82222f1ab8fb06ab1fc0c345419921938262ee37 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Sep 2023 11:43:43 +0800 Subject: [PATCH 49/96] other: disable detail display. --- cmake/cmake.define | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index edc5dd601a..53026df4ef 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.0) -set(CMAKE_VERBOSE_MAKEFILE ON) +set(CMAKE_VERBOSE_MAKEFILE FALSE) set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory From b317c8ebff51c998f9d6a2ac3a0cf929de37df3d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Sep 2023 12:46:15 +0800 Subject: [PATCH 50/96] fix(stream): disable merge submit blocks. --- include/libs/stream/tstream.h | 1 - source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 454 +++++++++++------- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/stream/src/streamRecover.c | 19 +- tests/system-test/8-stream/scalar_function.py | 3 +- 7 files changed, 300 insertions(+), 186 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 79d427e5c7..fc825b54dd 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -700,7 +700,6 @@ int32_t streamSourceScanHistoryData(SStreamTask* pTask); int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); // agg level -int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask); int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, SRpcHandleInfo* pRpcInfo); int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5d47237fb4..df645db7f1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1126,7 +1126,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); // set the fill-history task to be normal - if (pTask->info.fillHistory == 1) { + if (pTask->info.fillHistory == 1 && !streamTaskShouldStop(&pTask->status)) { streamSetStatusNormal(pTask); } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 1e714a8cad..2fc0f41a3b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -26,7 +26,7 @@ typedef struct STableSinkInfo { static int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData); -static int32_t doSinkDeleteBlock(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, +static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid); static int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); static void fillBucket(STokenBucket* pBucket); @@ -136,6 +136,149 @@ static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { return TSDB_CODE_SUCCESS; } +static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, + int64_t suid) { + tqDebug("s-task:%s build create table msg", pTask->id.idStr); + + STSchema* pTSchema = pTask->tbSink.pTSchema; + int32_t rows = pDataBlock->info.rows; + SArray* tagArray = NULL; + int32_t code = 0; + + SVCreateTbBatchReq reqs = {0}; + + SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); + if (NULL == reqs.pArray) { + goto _end; + } + + for (int32_t rowId = 0; rowId < rows; rowId++) { + SVCreateTbReq* pCreateTbReq = &((SVCreateTbReq){0}); + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); // taosStrdup(stbFullName); + + // set tag content + int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); + if (size == 2) { + tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + tdDestroySVCreateTbReq(pCreateTbReq); + goto _end; + } + + STagVal tagVal = { + .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; + + taosArrayPush(tagArray, &tagVal); + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + } else { + tagArray = taosArrayInit(size - 1, sizeof(STagVal)); + if (!tagArray) { + tdDestroySVCreateTbReq(pCreateTbReq); + goto _end; + } + + for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { + SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); + + STagVal tagVal = {.cid = pTSchema->numOfCols + step, .type = pTagData->info.type}; + void* pData = colDataGetData(pTagData, rowId); + if (colDataIsNull_s(pTagData, rowId)) { + continue; + } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { + tagVal.nData = varDataLen(pData); + tagVal.pData = (uint8_t*)varDataVal(pData); + } else { + memcpy(&tagVal.i64, pData, pTagData->info.bytes); + } + taosArrayPush(tagArray, &tagVal); + } + } + pCreateTbReq->ctb.tagNum = TMAX(size - UD_TAG_COLUMN_INDEX, 1); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + tagArray = taosArrayDestroy(tagArray); + if (pTag == NULL) { + tdDestroySVCreateTbReq(pCreateTbReq); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set table name + if (!pDataBlock->info.parTbName[0]) { + SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + + void* pGpIdData = colDataGetData(pGpIdColInfo, rowId); + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData); + } else { + pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName); + } + + taosArrayPush(reqs.pArray, pCreateTbReq); + tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); + } + + reqs.nReqs = taosArrayGetSize(reqs.pArray); + code = tqPutReqToQueue(pVnode, &reqs); + if (code != TSDB_CODE_SUCCESS) { + tqError("s-task:%s failed to send create table msg", pTask->id.idStr); + } + + _end: + taosArrayDestroy(tagArray); + taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); + return code; +} + +static int32_t doBuildSubmitAndSendMsg(SVnode* pVnode, SStreamTask* pTask, int32_t numOfBlocks, SSubmitReq2* pReq) { + const char* id = pTask->id.idStr; + int32_t vgId = TD_VID(pVnode); + int32_t len = 0; + void* pBuf = NULL; + + int32_t code = tqBuildSubmitReq(pReq, vgId, &pBuf, &len); + if (code != TSDB_CODE_SUCCESS) { + tqError("s-task:%s build submit msg failed, vgId:%d, code:%s", id, vgId, tstrerror(code)); + return code; + } + + SRpcMsg msg = {.msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len}; + code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); + if (code == TSDB_CODE_SUCCESS) { + tqDebug("s-task:%s vgId:%d send submit %d blocks into dstTables completed", id, vgId, numOfBlocks); + } else { + tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); + } + + pTask->sinkRecorder.numOfSubmit += 1; + + if ((pTask->sinkRecorder.numOfSubmit % 5000) == 0) { + SSinkTaskRecorder* pRec = &pTask->sinkRecorder; + double el = (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0; + tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) in %" PRId64 + " submit into dst table, duration:%.2f Sec.", + pTask->id.idStr, vgId, pRec->numOfBlocks, pRec->numOfRows, pRec->numOfSubmit, el); + } + + return TSDB_CODE_SUCCESS; +} + void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { const SArray* pBlocks = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; @@ -151,180 +294,141 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { pTask->tsInfo.sinkStart = taosGetTimestampMs(); } - tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table", vgId, id, numOfBlocks); - - bool hasSubmit = false; - SArray* tagArray = NULL; - SArray* pVals = NULL; - SArray* crTblArray = NULL; - - SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; - if (submitReq.aSubmitTbData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - - for (int32_t i = 0; i < numOfBlocks; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - code = doSinkDeleteBlock(pVnode, stbFullName, pDataBlock, pTask, suid); - } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { - tqDebug("s-task:%s build create table msg", pTask->id.idStr); - - SVCreateTbBatchReq reqs = {0}; - crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); - if (NULL == reqs.pArray) { - goto _end; - } - - for (int32_t rowId = 0; rowId < rows; rowId++) { - SVCreateTbReq* pCreateTbReq = &((SVCreateTbReq){0}); - - // set const - pCreateTbReq->flags = 0; - pCreateTbReq->type = TSDB_CHILD_TABLE; - pCreateTbReq->ctb.suid = suid; - - // set super table name - SName name = {0}; - tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); // taosStrdup(stbFullName); - - // set tag content - int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); - if (size == 2) { - tagArray = taosArrayInit(1, sizeof(STagVal)); - - if (!tagArray) { - tdDestroySVCreateTbReq(pCreateTbReq); - goto _end; - } - - STagVal tagVal = { - .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; - - taosArrayPush(tagArray, &tagVal); - - // set tag name - SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; - taosArrayPush(tagName, tagNameStr); - pCreateTbReq->ctb.tagName = tagName; - } else { - tagArray = taosArrayInit(size - 1, sizeof(STagVal)); - if (!tagArray) { - tdDestroySVCreateTbReq(pCreateTbReq); - goto _end; - } - - for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { - SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); - - STagVal tagVal = {.cid = pTSchema->numOfCols + step, .type = pTagData->info.type}; - void* pData = colDataGetData(pTagData, rowId); - if (colDataIsNull_s(pTagData, rowId)) { - continue; - } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { - tagVal.nData = varDataLen(pData); - tagVal.pData = (uint8_t*) varDataVal(pData); - } else { - memcpy(&tagVal.i64, pData, pTagData->info.bytes); - } - taosArrayPush(tagArray, &tagVal); - } - - } - pCreateTbReq->ctb.tagNum = TMAX(size - UD_TAG_COLUMN_INDEX, 1); - - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - tagArray = taosArrayDestroy(tagArray); - if (pTag == NULL) { - tdDestroySVCreateTbReq(pCreateTbReq); - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - - pCreateTbReq->ctb.pTag = (uint8_t*)pTag; - - // set table name - if (!pDataBlock->info.parTbName[0]) { - SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); - void* pGpIdData = colDataGetData(pGpIdColInfo, rowId); - pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData); - } else { - pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName); - } - - taosArrayPush(reqs.pArray, pCreateTbReq); - tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); - } - - reqs.nReqs = taosArrayGetSize(reqs.pArray); - if (tqPutReqToQueue(pVnode, &reqs) != TSDB_CODE_SUCCESS) { - goto _end; - } - - tagArray = taosArrayDestroy(tagArray); - taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); - crTblArray = NULL; - } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { - continue; - } else { - hasSubmit = true; - pTask->sinkRecorder.numOfBlocks += 1; - - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - code = doBuildSubmitFromResBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); - - taosArrayPush(submitReq.aSubmitTbData, &tbData); - pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; + bool isMixBlocks = true; + for(int32_t i = 0; i < numOfBlocks; ++i) { + SSDataBlock* p = taosArrayGet(pBlocks, i); + if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) { + isMixBlocks = true; + break; } } - if (hasSubmit) { - int32_t len = 0; - void* pBuf = NULL; - code = tqBuildSubmitReq(&submitReq, vgId, &pBuf, &len); + if (isMixBlocks) { + tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id, + numOfBlocks); - if (code != TSDB_CODE_SUCCESS) { - tqError("s-task:%s build submit msg failed, vgId:%d, code:%s", id, vgId, tstrerror(code)); - goto _end; - } + for(int32_t i = 0; i < numOfBlocks; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + code = doBuildAndSendDeleteMsg(pVnode, stbFullName, pDataBlock, pTask, suid); + } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); + } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { + continue; + } else { + pTask->sinkRecorder.numOfBlocks += 1; - SRpcMsg msg = {.msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len}; - code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); - if (code == TSDB_CODE_SUCCESS) { - tqDebug("s-task:%s vgId:%d send submit %d blocks into dstTables completed", id, vgId, numOfBlocks); - } else { - ASSERT(0); - tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); - } + SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; + if (submitReq.aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + tqError("s-task:%s vgId:%d failed to prepare submit msg in sink task, code:%s", id, vgId, tstrerror(code)); + return; + } - pTask->sinkRecorder.numOfSubmit += 1; + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; + code = doBuildSubmitFromResBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); + taosArrayPush(submitReq.aSubmitTbData, &tbData); - if ((pTask->sinkRecorder.numOfSubmit % 5000) == 0) { - SSinkTaskRecorder* pRec = &pTask->sinkRecorder; - tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) in %" PRId64 - " submit into dst table, duration:%.2f Sec.", - pTask->id.idStr, vgId, pRec->numOfBlocks, pRec->numOfRows, pRec->numOfSubmit, - (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0); + code = doBuildSubmitAndSendMsg(pVnode, pTask, numOfBlocks, &submitReq); + } } } else { - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); - tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); + tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, merge submit msg", vgId, id, numOfBlocks); + SHashObj* pTableIndexMap = taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; + if (submitReq.aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + tqError("s-task:%s vgId:%d failed to prepare submit msg in sink task, code:%s", id, vgId, tstrerror(code)); + taosHashCleanup(pTableIndexMap); + return; + } + + bool hasSubmit = false; + + for (int32_t i = 0; i < numOfBlocks; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + code = doBuildAndSendDeleteMsg(pVnode, stbFullName, pDataBlock, pTask, suid); + } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); + } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { + continue; + } else { + hasSubmit = true; + pTask->sinkRecorder.numOfBlocks += 1; + + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; + code = doBuildSubmitFromResBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); + + int32_t* index = taosHashGet(pTableIndexMap, &tbData.uid, sizeof(tbData.uid)); + if (index == NULL) { // no data yet, append it + taosArrayPush(submitReq.aSubmitTbData, &tbData); + + int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; + taosHashPut(pTableIndexMap, &tbData.uid, sizeof(tbData.uid), &size, sizeof(size)); + } else { + SSubmitTbData* pExisted = taosArrayGet(submitReq.aSubmitTbData, *index); + // merge the new submit table block with the existed blocks + // if ts in the new data block overlap with existed one, replace it + int32_t oldLen = taosArrayGetSize(pExisted->aRowP); + int32_t newLen = taosArrayGetSize(tbData.aRowP); + + int32_t j = 0, k = 0; + SArray* pFinal = taosArrayInit(oldLen + newLen, POINTER_BYTES); + while (j < newLen && k < oldLen) { + SRow* pNewRow = taosArrayGetP(tbData.aRowP, j); + SRow* pOldRow = taosArrayGetP(pExisted->aRowP, k); + if (pNewRow->ts <= pOldRow->ts) { + taosArrayPush(pFinal, &pNewRow); + if (pNewRow->ts < pOldRow->ts) { + j += 1; + } else { + j += 1; + k += 1; + } + } else { + taosArrayPush(pFinal, &pOldRow); + k += 1; + } + } + + while (j < newLen) { + SRow* pRow = taosArrayGetP(tbData.aRowP, j++); + taosArrayPush(pFinal, &pRow); + } + + while (k < oldLen) { + SRow* pRow = taosArrayGetP(pExisted->aRowP, k++); + taosArrayPush(pFinal, &pRow); + } + + taosArrayDestroy(tbData.aRowP); + taosArrayDestroy(pExisted->aRowP); + pExisted->aRowP = pFinal; + + tqDebug("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id, + (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (tbData.pCreateTbReq != NULL)); + } + + pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; + } + } + + taosHashCleanup(pTableIndexMap); + + if (hasSubmit) { + doBuildSubmitAndSendMsg(pVnode, pTask, numOfBlocks, &submitReq); + } else { + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); + } } -_end: - taosArrayDestroy(tagArray); - taosArrayDestroy(pVals); - taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); // TODO: change } -int32_t doSinkDeleteBlock(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, +int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid) { SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))}; @@ -490,8 +594,19 @@ int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int return TSDB_CODE_SUCCESS; } -int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, SSDataBlock* pDataBlock, - SStreamTask* pTask, SSubmitTbData* pTableData) { +static int32_t tsAscendingSortFn(const void* p1, const void* p2) { + SRow* pRow1 = *(SRow**) p1; + SRow* pRow2 = *(SRow**) p2; + + if (pRow1->ts == pRow2->ts) { + return 0; + } else { + return pRow1->ts > pRow2->ts? 1:-1; + } +} + +int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, + SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData) { int32_t numOfRows = pDataBlock->info.rows; int32_t vgId = TD_VID(pVnode); uint64_t groupId = pDataBlock->info.id.groupId; @@ -507,7 +622,7 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); if (pTableData->aRowP == NULL || pVals == NULL) { - taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); code = TSDB_CODE_OUT_OF_MEMORY; @@ -557,7 +672,7 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF while (pTableSinkInfo->uid == 0) { if (streamTaskShouldStop(&pTask->status)) { tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); - taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return TSDB_CODE_SUCCESS; @@ -573,7 +688,7 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF if (!isValid) { // not valid table, ignore it metaReaderClear(&mr); - taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return TSDB_CODE_SUCCESS; @@ -612,7 +727,7 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF if (pTableData->pCreateTbReq == NULL) { tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); - taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return terrno; @@ -624,7 +739,7 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF if (!isValid) { metaReaderClear(&mr); taosMemoryFree(pTableSinkInfo); - taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return TSDB_CODE_SUCCESS; } else { @@ -681,7 +796,7 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF if (code != TSDB_CODE_SUCCESS) { tDestroySubmitTbData(pTableData, TSDB_MSG_FLG_ENCODE); - taosArrayDestroy(pTableData->aRowP); + pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return code; } @@ -690,6 +805,7 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF taosArrayPush(pTableData->aRowP, &pRow); } + taosArraySort(pTableData->aRowP, tsAscendingSortFn); tqDebug("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); taosArrayDestroy(pVals); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 62822021a6..8b094132f3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1443,6 +1443,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in SColData *pColData = (SColData *)taosArrayGet(pSubmitTbData->aCol, 0); TSKEY *aKey = (TSKEY *)(pColData->pData); + vDebug("vgId:%d submit %d rows data, uid:%"PRId64, TD_VID(pVnode), pColData->nVal, pSubmitTbData->uid); for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) { if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) { @@ -1459,7 +1460,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in for (int32_t iRow = 0; iRow < nRow; ++iRow) { if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) { code = TSDB_CODE_INVALID_MSG; - vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver); goto _exit; } } @@ -1564,6 +1565,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in } else { // create table failed if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { code = terrno; + vError("vgId:%d failed to create table:%s, code:%s", TD_VID(pVnode), pSubmitTbData->pCreateTbReq->name, + tstrerror(terrno)); goto _exit; } terrno = 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 12ec63b4cb..474128007a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2257,7 +2257,7 @@ FETCH_NEXT_BLOCK: int32_t current = pInfo->validBlockIndex++; SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); - qDebug("set %d/%d as the input submit block, %s", current, totalBlocks, id); + qDebug("set %d/%d as the input submit block, %s", current + 1, totalBlocks, id); if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id); continue; diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 743d87e938..5c03e3a95e 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -30,6 +30,13 @@ static void streamTaskSetRangeStreamCalc(SStreamTask* pTask); static int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); static void streamTaskSetReady(SStreamTask* pTask, int32_t numOfReqs) { + if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { + pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->pUpstreamInfoList); + qDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", + pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, + streamGetTaskStatusStr(pTask->status.taskStatus)); + } + ASSERT(pTask->status.downstreamReady == 0); pTask->status.downstreamReady = 1; @@ -97,11 +104,8 @@ int32_t streamTaskLaunchScanHistory(SStreamTask* pTask) { streamSetParamForScanHistory(pTask); streamTaskEnablePause(pTask); } - - streamTaskScanHistoryPrepare(pTask); } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) { qDebug("s-task:%s sink task do nothing to handle scan-history", pTask->id.idStr); - streamTaskScanHistoryPrepare(pTask); } return 0; } @@ -402,15 +406,6 @@ int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } -// agg -int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask) { - pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->pUpstreamInfoList); - qDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", - pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, - streamGetTaskStatusStr(pTask->status.taskStatus)); - return 0; -} - int32_t streamAggUpstreamScanHistoryFinish(SStreamTask* pTask) { void* exec = pTask->exec.pExecutor; if (pTask->info.fillHistory && qRestoreStreamOperatorOption(exec) < 0) { diff --git a/tests/system-test/8-stream/scalar_function.py b/tests/system-test/8-stream/scalar_function.py index 56537e2f54..3bc44a7dc7 100644 --- a/tests/system-test/8-stream/scalar_function.py +++ b/tests/system-test/8-stream/scalar_function.py @@ -6,7 +6,8 @@ from util.cases import * from util.common import * class TDTestCase: - updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + updatecfgDict = {'vdebugFlag': 143, 'qdebugflag':135, 'tqdebugflag':135, 'udebugflag':135, 'rpcdebugflag':135, + 'asynclog': 0} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) From 328377c25fc2884aa38fbe427ca6e99628c26f01 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Sep 2023 23:04:39 +0800 Subject: [PATCH 51/96] refactor: add some logs. --- source/libs/stream/src/streamQueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index f2b50a024f..609a67fb95 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -357,7 +357,7 @@ static void fillBucket(STokenBucket* pBucket) { } pBucket->fillTimestamp = now; - qInfo("new token available, ts:%"PRId64, now); + qInfo("new token available, current:%d, inc:%d ts:%"PRId64, pBucket->numOfToken, inc, now); } } From 35b97cbd19edac6746c28e7fa658ff44bf42947b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Sep 2023 23:50:41 +0800 Subject: [PATCH 52/96] fix(stream): set correct token in bucket. --- source/libs/stream/src/streamDispatch.c | 13 ++++++++++--- source/libs/stream/src/streamExec.c | 2 +- source/libs/stream/src/streamQueue.c | 8 ++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 916cc6e9ee..d04f55628c 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1001,10 +1001,17 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i // so the TASK_INPUT_STATUS_BLOCKED is rsp if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { pTask->inputInfo.status = TASK_INPUT_STATUS__BLOCKED; // block the input of current task, to push pressure to upstream - pTask->msgInfo.blockingTs = taosGetTimestampMs(); // record the blocking start time + double el = 0; + if (pTask->msgInfo.blockingTs == 0) { + pTask->msgInfo.blockingTs = taosGetTimestampMs(); // record the blocking start time + } else { + el = (taosGetTimestampMs() - pTask->msgInfo.blockingTs) / 1000.0; + } + int8_t ref = atomic_add_fetch_8(&pTask->status.timerActive, 1); - qError("s-task:%s inputQ of downstream task:0x%x is full, time:%" PRId64 " wait for %dms and retry dispatch data, ref:%d", - id, pRsp->downstreamTaskId, pTask->msgInfo.blockingTs, DISPATCH_RETRY_INTERVAL_MS, ref); + qError("s-task:%s inputQ of downstream task:0x%x is full, time:%" PRId64 + " wait for %dms and retry dispatch data, total wait:%.2fSec ref:%d", + id, pRsp->downstreamTaskId, pTask->msgInfo.blockingTs, DISPATCH_RETRY_INTERVAL_MS, el, ref); streamRetryDispatchStreamBlock(pTask, DISPATCH_RETRY_INTERVAL_MS); } else { // pipeline send data in output queue // this message has been sent successfully, let's try next one. diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index ff667fa778..158727efea 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -84,7 +84,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i } if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) { - qWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", pTask->id.idStr); + qWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry exec task", pTask->id.idStr); taosMsleep(1000); continue; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 609a67fb95..b65063f49a 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -347,6 +347,7 @@ int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t cap, int32_t ra static void fillBucket(STokenBucket* pBucket) { int64_t now = taosGetTimestampMs(); int64_t delta = now - pBucket->fillTimestamp; + ASSERT(pBucket->numOfToken >= 0); int32_t inc = (delta / 1000.0) * pBucket->rate; if (inc > 0) { @@ -363,10 +364,9 @@ static void fillBucket(STokenBucket* pBucket) { bool streamTaskHasAvailableToken(STokenBucket* pBucket) { fillBucket(pBucket); - bool hasToken = (--pBucket->numOfToken) > 0; - if (hasToken) { - qDebug("remain token:%d", pBucket->numOfToken); - return true; + if (pBucket->numOfToken > 0) { + qDebug("remain token:%d", pBucket->numOfToken-1); + return --pBucket->numOfToken; } else { return false; } From bf79c2c9939a01bf3669690a461acb962989dfc0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Sep 2023 00:11:13 +0800 Subject: [PATCH 53/96] fix(stream): limit the sink rate --- source/libs/stream/src/streamTask.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 24b8933387..58f7721c92 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -385,7 +385,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->dataRange.range.minVer = ver; pTask->pMsgCb = pMsgCb; - streamTaskInitTokenBucket(&pTask->tokenBucket, 400, 200); + streamTaskInitTokenBucket(&pTask->tokenBucket, 300, 200); taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); From e4918e85eb49cf6ac2cda3724e3ce5b757770dab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Sep 2023 00:22:22 +0800 Subject: [PATCH 54/96] fix(stream): limit the sink rate --- source/libs/stream/src/streamTask.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 58f7721c92..050fa39397 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -385,7 +385,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->dataRange.range.minVer = ver; pTask->pMsgCb = pMsgCb; - streamTaskInitTokenBucket(&pTask->tokenBucket, 300, 200); + streamTaskInitTokenBucket(&pTask->tokenBucket, 200, 100); taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); From 47fd14444705680e5ed95604b6273fe3990568c1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Sep 2023 11:19:51 +0800 Subject: [PATCH 55/96] other: add some logs. --- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +++++ source/libs/stream/src/streamTask.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8b094132f3..6ff33c4f6e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1446,6 +1446,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in vDebug("vgId:%d submit %d rows data, uid:%"PRId64, TD_VID(pVnode), pColData->nVal, pSubmitTbData->uid); for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) { + vDebug("vgId:%d uid:%"PRId64" ts:%"PRId64, TD_VID(pVnode), pSubmitTbData->uid, aKey[iRow]); + if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) { code = TSDB_CODE_INVALID_MSG; vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); @@ -1457,7 +1459,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP); SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + vDebug("vgId:%d submit %d rows data, uid:%"PRId64, TD_VID(pVnode), nRow, pSubmitTbData->uid); for (int32_t iRow = 0; iRow < nRow; ++iRow) { + vDebug("vgId:%d uid:%"PRId64" ts:%"PRId64, TD_VID(pVnode), pSubmitTbData->uid, aRow[iRow]->ts); + if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) { code = TSDB_CODE_INVALID_MSG; vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 050fa39397..162ae3f097 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -385,7 +385,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->dataRange.range.minVer = ver; pTask->pMsgCb = pMsgCb; - streamTaskInitTokenBucket(&pTask->tokenBucket, 200, 100); + streamTaskInitTokenBucket(&pTask->tokenBucket, 150, 100); taosThreadMutexInit(&pTask->lock, NULL); streamTaskOpenAllUpstreamInput(pTask); From 6f5c855a4baef69c4162ffe4f0e7b0f8afb90d88 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Sep 2023 15:08:36 +0800 Subject: [PATCH 56/96] fix(stream): fix msg lost bug. --- source/dnode/vnode/src/tq/tqSink.c | 2 +- source/dnode/vnode/src/tq/tqStreamTask.c | 22 ++++++++++------------ source/libs/stream/src/streamExec.c | 4 ++-- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 2fc0f41a3b..88bdc8a4d9 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -329,7 +329,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { code = doBuildSubmitFromResBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); taosArrayPush(submitReq.aSubmitTbData, &tbData); - code = doBuildSubmitAndSendMsg(pVnode, pTask, numOfBlocks, &submitReq); + code = doBuildSubmitAndSendMsg(pVnode, pTask, 1, &submitReq); } } } else { diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 553ef06974..4a2d5a2ea6 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -396,25 +396,23 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { int32_t numOfItems = streamTaskGetInputQItems(pTask); int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX; + taosThreadMutexLock(&pTask->lock); + + pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + if (pTask->status.taskStatus != TASK_STATUS__NORMAL) { + tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pStatus); + taosThreadMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pStreamMeta, pTask); + continue; + } + SStreamQueueItem* pItem = NULL; code = extractMsgFromWal(pTask->exec.pWalReader, (void**)&pItem, maxVer, pTask->id.idStr); if ((code != TSDB_CODE_SUCCESS || pItem == NULL) && (numOfItems == 0)) { // failed, continue handleFillhistoryScanComplete(pTask, walReaderGetCurrentVer(pTask->exec.pWalReader)); streamMetaReleaseTask(pStreamMeta, pTask); - continue; - } - - taosThreadMutexLock(&pTask->lock); - pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); - - if (pTask->status.taskStatus != TASK_STATUS__NORMAL) { - tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pStatus); taosThreadMutexUnlock(&pTask->lock); - streamMetaReleaseTask(pStreamMeta, pTask); - if (pItem != NULL) { - streamFreeQitem(pItem); - } continue; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 158727efea..6f0357324a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -566,8 +566,8 @@ int32_t streamExecForAll(SStreamTask* pTask) { ASSERT(pTask->chkInfo.checkpointVer <= pTask->chkInfo.currentVer && ver >= pTask->chkInfo.checkpointVer); if (ver != pTask->chkInfo.checkpointVer) { - qDebug("s-task:%s update checkpointVer(unsaved) from %" PRId64 " to %" PRId64, pTask->id.idStr, - pTask->chkInfo.checkpointVer, ver); + qDebug("s-task:%s update checkpointVer(unsaved) from %" PRId64 " to %" PRId64 " , currentVer:%" PRId64, + pTask->id.idStr, pTask->chkInfo.checkpointVer, ver, pTask->chkInfo.currentVer); pTask->chkInfo.checkpointVer = ver; } From aa8909b267a649c743a3aebad39e659324cc1608 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Sep 2023 15:18:54 +0800 Subject: [PATCH 57/96] other: add some logs. --- include/libs/wal/wal.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqMeta.c | 4 ++-- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/sync/src/syncRaftLog.c | 2 +- source/libs/wal/src/walRead.c | 34 ++++++++++++++++-------------- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index cfe70a186c..a56a5567eb 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -192,7 +192,7 @@ int32_t walApplyVer(SWal *, int64_t ver); // int32_t walDataCorrupted(SWal*); // wal reader -SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); +SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id); void walCloseReader(SWalReader *pRead); void walReadReset(SWalReader *pReader); int32_t walReadVer(SWalReader *pRead, int64_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index df645db7f1..af08a0dfc4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -819,7 +819,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files - pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond); + pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId); } // reset the task status from unfinished transaction diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 85151c6e19..154ac1e8c1 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -312,14 +312,14 @@ static int buildHandle(STQ* pTq, STqHandle* handle){ return -1; } } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) { - handle->pWalReader = walOpenReader(pVnode->pWal, NULL); + handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); handle->execHandle.pTqReader = tqReaderOpen(pVnode); buildSnapContext(reader.vnode, reader.version, 0, handle->execHandle.subType, handle->fetchMeta, (SSnapContext**)(&reader.sContext)); handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - handle->pWalReader = walOpenReader(pVnode->pWal, NULL); + handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 4f5701524d..cf5d160d75 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -250,7 +250,7 @@ STqReader* tqReaderOpen(SVnode* pVnode) { return NULL; } - pReader->pWalReader = walOpenReader(pVnode->pWal, NULL); + pReader->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); if (pReader->pWalReader == NULL) { taosMemoryFree(pReader); return NULL; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 9299651999..b167f2ecb6 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -59,7 +59,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { ASSERT(pData->pWal != NULL); taosThreadMutexInit(&(pData->mutex), NULL); - pData->pWalHandle = walOpenReader(pData->pWal, NULL); + pData->pWalHandle = walOpenReader(pData->pWal, NULL, 0); ASSERT(pData->pWalHandle != NULL); pLogStore->syncLogUpdateCommitIndex = raftLogUpdateCommitIndex; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index d9e43e4324..9db81111f9 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -16,7 +16,7 @@ #include "taoserror.h" #include "walInt.h" -SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { +SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) { SWalReader *pReader = taosMemoryCalloc(1, sizeof(SWalReader)); if (pReader == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -24,7 +24,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { } pReader->pWal = pWal; - pReader->readerId = tGenIdPI64(); + pReader->readerId = (id != 0)? id:tGenIdPI64(); pReader->pIdxFile = NULL; pReader->pLogFile = NULL; pReader->curVersion = -1; @@ -257,9 +257,9 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { bool seeked = false; wDebug("vgId:%d, try to fetch ver %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64, + ", applied ver:%" PRId64", %"PRIx64, pRead->pWal->cfg.vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, - pRead->pWal->vers.appliedVer); + pRead->pWal->vers.appliedVer, pRead->readerId); // TODO: valid ver if (ver > pRead->pWal->vers.commitVer) { @@ -297,7 +297,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { code = walValidHeadCksum(pRead->pHead); if (code != 0) { - wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId, ver); + wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, %"PRIx64, pRead->pWal->cfg.vgId, ver, + pRead->readerId); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -324,11 +325,13 @@ int32_t walSkipFetchBody(SWalReader *pRead) { int32_t walFetchBody(SWalReader *pRead) { SWalCont *pReadHead = &pRead->pHead->head; int64_t ver = pReadHead->version; + int32_t vgId = pRead->pWal->cfg.vgId; + int64_t id = pRead->readerId; wDebug("vgId:%d, fetch body %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64, - pRead->pWal->cfg.vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, - pRead->pWal->vers.appliedVer); + ", applied ver:%" PRId64 ", %" PRIx64, + vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, + pRead->pWal->vers.appliedVer, id); if (pRead->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); @@ -344,26 +347,25 @@ int32_t walFetchBody(SWalReader *pRead) { if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", - pRead->pWal->cfg.vgId, pReadHead->version, ver, tstrerror(terrno)); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, %"PRIx64, + vgId, pReadHead->version, ver, tstrerror(terrno), pRead->readerId); } else { - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted", - pRead->pWal->cfg.vgId, pReadHead->version, ver); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted, %"PRIx64, + vgId, pReadHead->version, ver, pRead->readerId); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } return -1; } if (pReadHead->version != ver) { - wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, - pReadHead->version, ver); + wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64", %"PRIx64, vgId, + pReadHead->version, ver, id); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } if (walValidBodyCksum(pRead->pHead) != 0) { - wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, - ver); + wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, %" PRIx64, vgId, ver, id); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } From 9a30573b8bdb62fb7830f86b057531491f408457 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Sep 2023 16:12:08 +0800 Subject: [PATCH 58/96] fix(stream): disable scan wal when halt is set. --- source/dnode/vnode/src/tq/tq.c | 7 +++++-- source/libs/wal/src/walRead.c | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index af08a0dfc4..ece29a492e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1090,14 +1090,17 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { } // now we can stop the stream task execution - streamTaskHalt(pStreamTask); + int64_t latestVer = 0; + taosThreadMutexLock(&pStreamTask->lock); + streamTaskHalt(pStreamTask); tqDebug("s-task:%s level:%d sched-status:%d is halt by fill-history task:%s", pStreamTask->id.idStr, pStreamTask->info.taskLevel, pStreamTask->status.schedStatus, id); + latestVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); + taosThreadMutexUnlock(&pStreamTask->lock); // if it's an source task, extract the last version in wal. pRange = &pTask->dataRange.range; - int64_t latestVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); done = streamHistoryTaskSetVerRangeStep2(pTask, latestVer); if (done) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9db81111f9..a45b1eb85d 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -308,9 +308,9 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { int32_t walSkipFetchBody(SWalReader *pRead) { wDebug("vgId:%d, skip fetch body %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64, + ", applied ver:%" PRId64", 0x%"PRIx64, pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, - pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer); + pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId); int64_t code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); if (code < 0) { @@ -329,7 +329,7 @@ int32_t walFetchBody(SWalReader *pRead) { int64_t id = pRead->readerId; wDebug("vgId:%d, fetch body %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64 ", %" PRIx64, + ", applied ver:%" PRId64 ", 0x%" PRIx64, vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, id); @@ -358,14 +358,14 @@ int32_t walFetchBody(SWalReader *pRead) { } if (pReadHead->version != ver) { - wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64", %"PRIx64, vgId, + wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64", 0x%"PRIx64, vgId, pReadHead->version, ver, id); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } if (walValidBodyCksum(pRead->pHead) != 0) { - wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, %" PRIx64, vgId, ver, id); + wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } From 85a5c45098a1fba40b19e4b339fdaa30d9285af7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Sep 2023 00:01:25 +0800 Subject: [PATCH 59/96] fix(stream): limit the already processed data. --- include/libs/stream/tstream.h | 2 +- source/dnode/snode/src/snode.c | 14 ++++++------- source/dnode/vnode/src/tq/tq.c | 18 ++++++++--------- source/dnode/vnode/src/tq/tqRead.c | 13 ++++++------ source/dnode/vnode/src/tq/tqStreamTask.c | 24 +++++++++++------------ source/libs/stream/src/streamCheckpoint.c | 2 +- source/libs/stream/src/streamExec.c | 4 ++-- source/libs/stream/src/streamRecover.c | 3 ++- source/libs/stream/src/streamTask.c | 2 +- source/libs/wal/src/walRead.c | 13 ++++++------ 10 files changed, 49 insertions(+), 46 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index fc825b54dd..30c60bcf0d 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -260,7 +260,7 @@ typedef struct SStreamTaskId { typedef struct SCheckpointInfo { int64_t checkpointId; int64_t checkpointVer; // latest checkpointId version - int64_t currentVer; // current offset in WAL, not serialize it + int64_t nextProcessVer; // current offset in WAL, not serialize it } SCheckpointInfo; typedef struct SStreamStatus { diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index c1a59416f6..2b1885fb0e 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -86,18 +86,18 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { SCheckpointInfo* pChkInfo = &pTask->chkInfo; // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { - pTask->chkInfo.currentVer = pTask->chkInfo.checkpointVer + 1; - qInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr, - pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->currentVer); + pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; + qInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, + pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } else { - if (pTask->chkInfo.currentVer == -1) { - pTask->chkInfo.currentVer = 0; + if (pTask->chkInfo.nextProcessVer == -1) { + pTask->chkInfo.nextProcessVer = 0; } } - qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " currentVer:%" PRId64 + qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", - SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->currentVer, + SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->info.fillHistory, pTask->info.triggerParam); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ece29a492e..3c4ad50a20 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -834,14 +834,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { - pTask->chkInfo.currentVer = pTask->chkInfo.checkpointVer + 1; + pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr, - pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->currentVer); + pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " currentVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", - vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->currentVer, + vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->info.fillHistory, pTask->info.triggerParam); @@ -1121,7 +1121,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { int64_t dstVer = pTask->dataRange.range.minVer - 1; - pTask->chkInfo.currentVer = dstVer; + pTask->chkInfo.nextProcessVer = dstVer; walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer); tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer, pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE); @@ -1154,7 +1154,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { tqDebug( "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, start " "ver:%" PRId64 ", window:%" PRId64 " - %" PRId64, - id, pTask->chkInfo.currentVer, pWindow->skey, pWindow->ekey); + id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); } code = streamTaskScanHistoryDataComplete(pTask); @@ -1289,8 +1289,8 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { // even in halt status, the data in inputQ must be processed int8_t st = pTask->status.taskStatus; if (st == TASK_STATUS__NORMAL || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK) { - tqDebug("vgId:%d s-task:%s start to process block from inputQ, last chk point:%" PRId64, vgId, pTask->id.idStr, - pTask->chkInfo.currentVer); + tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, + pTask->chkInfo.nextProcessVer); streamProcessRunReq(pTask); } else { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); @@ -1429,10 +1429,10 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); } else { // from the previous paused version and go on tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); } if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index cf5d160d75..cadbc70c6f 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -187,25 +187,26 @@ end: int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId) { int32_t code = -1; int32_t vgId = TD_VID(pTq->pVnode); + int64_t id = pHandle->pWalReader->readerId; int64_t offset = *fetchOffset; int64_t lastVer = walGetLastVer(pHandle->pWalReader->pWal); int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal); int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal); - wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64, - vgId, offset, lastVer, committedVer, appliedVer); + wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64", 0x%"PRIx64, + vgId, offset, lastVer, committedVer, appliedVer, id); while (offset <= appliedVer) { if (walFetchHead(pHandle->pWalReader, offset) < 0) { tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 - ", no more log to return, reqId:0x%" PRIx64, - pHandle->consumerId, pHandle->epoch, vgId, offset, reqId); + ", no more log to return, reqId:0x%" PRIx64 " 0x%" PRIx64, + pHandle->consumerId, pHandle->epoch, vgId, offset, reqId, id); goto END; } - tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64, vgId, - pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId); + tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64" 0x%"PRIx64, vgId, + pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id); if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) { code = walFetchBody(pHandle->pWalReader); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 4a2d5a2ea6..955614297b 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -256,36 +256,36 @@ int32_t tqStartStreamTasks(STQ* pTq) { int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId) { // seek the stored version and extract data from WAL int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); - if (pTask->chkInfo.currentVer < firstVer) { + if (pTask->chkInfo.nextProcessVer < firstVer) { tqWarn("vgId:%d s-task:%s ver:%" PRId64 " earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, - vgId, pTask->id.idStr, pTask->chkInfo.currentVer, firstVer, firstVer); + vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, firstVer, firstVer); - pTask->chkInfo.currentVer = firstVer; + pTask->chkInfo.nextProcessVer = firstVer; // todo need retry if failed - int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); if (code != TSDB_CODE_SUCCESS) { return code; } // append the data for the stream - tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); + tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer); } else { int64_t currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); if (currentVer == -1) { // we only seek the read for the first time - int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit return code; } // append the data for the stream tqDebug("vgId:%d s-task:%s wal reader initial seek to ver:%" PRId64, vgId, pTask->id.idStr, - pTask->chkInfo.currentVer); + pTask->chkInfo.nextProcessVer); } } int64_t skipToVer = walReaderGetSkipToVersion(pTask->exec.pWalReader); - if (skipToVer != 0 && skipToVer > pTask->chkInfo.currentVer) { + if (skipToVer != 0 && skipToVer > pTask->chkInfo.nextProcessVer) { int32_t code = walReaderSeekVer(pTask->exec.pWalReader, skipToVer); if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit return code; @@ -304,7 +304,7 @@ void handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) { if ((pTask->info.fillHistory == 1) && ver > pTask->dataRange.range.maxVer) { if (!pTask->status.appendTranstateBlock) { - qWarn("s-task:%s fill-history scan WAL, currentVer:%" PRId64 " reach the maximum ver:%" PRId64 + qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the maximum ver:%" PRId64 ", not scan wal anymore, add transfer-state block into inputQ", id, ver, maxVer); @@ -313,7 +313,7 @@ void handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) { /*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask); /*int32_t code = */ streamSchedExec(pTask); } else { - qWarn("s-task:%s fill-history scan WAL, currentVer:%" PRId64 " reach the maximum ver:%" PRId64 ", not scan wal", + qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the maximum ver:%" PRId64 ", not scan wal", id, ver, maxVer); } } @@ -421,12 +421,12 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { code = streamTaskPutDataIntoInputQ(pTask, pItem); if (code == TSDB_CODE_SUCCESS) { int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader); - pTask->chkInfo.currentVer = ver; + pTask->chkInfo.nextProcessVer = ver; handleFillhistoryScanComplete(pTask, ver); tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, ver); } else { tqError("s-task:%s append input queue failed, too many in inputQ, ver:%" PRId64, pTask->id.idStr, - pTask->chkInfo.currentVer); + pTask->chkInfo.nextProcessVer); } } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index baf319d014..361602fac9 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -287,7 +287,7 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { streamTaskOpenAllUpstreamInput(p); // open inputQ for all upstream tasks qDebug("vgId:%d s-task:%s level:%d commit task status after checkpoint completed, checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s", - pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.currentVer, + pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, streamGetTaskStatusStr(prev)); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6f0357324a..8d282696c1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -563,11 +563,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { SIZE_IN_MB(resSize), totalBlocks); // update the currentVer if processing the submit blocks. - ASSERT(pTask->chkInfo.checkpointVer <= pTask->chkInfo.currentVer && ver >= pTask->chkInfo.checkpointVer); + ASSERT(pTask->chkInfo.checkpointVer <= pTask->chkInfo.nextProcessVer && ver >= pTask->chkInfo.checkpointVer); if (ver != pTask->chkInfo.checkpointVer) { qDebug("s-task:%s update checkpointVer(unsaved) from %" PRId64 " to %" PRId64 " , currentVer:%" PRId64, - pTask->id.idStr, pTask->chkInfo.checkpointVer, ver, pTask->chkInfo.currentVer); + pTask->id.idStr, pTask->chkInfo.checkpointVer, ver, pTask->chkInfo.nextProcessVer); pTask->chkInfo.checkpointVer = ver; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 5c03e3a95e..cf04659ac1 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -504,7 +504,8 @@ int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) { static void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) { pHTask->dataRange.range.minVer = 0; - pHTask->dataRange.range.maxVer = pTask->chkInfo.currentVer; + // the query version range should be limited to the already processed data + pHTask->dataRange.range.maxVer = pTask->chkInfo.nextProcessVer - 1; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { qDebug("s-task:%s set the launch condition for fill-history s-task:%s, window:%" PRId64 " - %" PRId64 diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 162ae3f097..ea4c2e71bc 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -380,7 +380,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; pTask->pMeta = pMeta; - pTask->chkInfo.currentVer = ver; + pTask->chkInfo.nextProcessVer = ver; pTask->dataRange.range.maxVer = ver; pTask->dataRange.range.minVer = ver; pTask->pMsgCb = pMsgCb; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index a45b1eb85d..2eee04a27a 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -75,6 +75,7 @@ int32_t walNextValidMsg(SWalReader *pReader) { terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } + while (fetchVer <= appliedVer) { if (walFetchHead(pReader, fetchVer) < 0) { return -1; @@ -257,7 +258,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { bool seeked = false; wDebug("vgId:%d, try to fetch ver %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64", %"PRIx64, + ", applied ver:%" PRId64", 0x%"PRIx64, pRead->pWal->cfg.vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId); @@ -297,7 +298,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { code = walValidHeadCksum(pRead->pHead); if (code != 0) { - wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, %"PRIx64, pRead->pWal->cfg.vgId, ver, + wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, 0x%"PRIx64, pRead->pWal->cfg.vgId, ver, pRead->readerId); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -347,11 +348,11 @@ int32_t walFetchBody(SWalReader *pRead) { if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, %"PRIx64, - vgId, pReadHead->version, ver, tstrerror(terrno), pRead->readerId); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64, + vgId, pReadHead->version, ver, tstrerror(terrno), id); } else { - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted, %"PRIx64, - vgId, pReadHead->version, ver, pRead->readerId); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted, 0x%"PRIx64, + vgId, pReadHead->version, ver, id); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } return -1; From 4ffb4a5b1f4a380fbede56fabcca2ce987a85921 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Sep 2023 11:02:32 +0800 Subject: [PATCH 60/96] refactor: disable some logs. --- source/libs/stream/src/streamQueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index b65063f49a..19c75bcb03 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -358,7 +358,7 @@ static void fillBucket(STokenBucket* pBucket) { } pBucket->fillTimestamp = now; - qInfo("new token available, current:%d, inc:%d ts:%"PRId64, pBucket->numOfToken, inc, now); + qDebug("new token available, current:%d, inc:%d ts:%"PRId64, pBucket->numOfToken, inc, now); } } From 10271aee26c337bc97444a7148148c36a1a9a4c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Sep 2023 14:12:03 +0800 Subject: [PATCH 61/96] fix(stream): fix syntax error in unit test. --- source/libs/wal/test/walMetaTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 0784db917a..70d8921be3 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -326,7 +326,7 @@ TEST_F(WalCleanDeleteEnv, roll) { TEST_F(WalKeepEnv, readHandleRead) { walResetEnv(); int code; - SWalReader* pRead = walOpenReader(pWal, NULL); + SWalReader* pRead = walOpenReader(pWal, NULL, 0); ASSERT(pRead != NULL); int i; @@ -387,7 +387,7 @@ TEST_F(WalRetentionEnv, repairMeta1) { ASSERT_EQ(pWal->vers.lastVer, 99); - SWalReader* pRead = walOpenReader(pWal, NULL); + SWalReader* pRead = walOpenReader(pWal, NULL, 0); ASSERT(pRead != NULL); for (int i = 0; i < 1000; i++) { From a9321248a959cdad8f98d227fc78c7daf63b26fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Sep 2023 18:34:03 +0800 Subject: [PATCH 62/96] fix(stream): set the correct flag when there is one token existed. --- source/libs/stream/src/streamQueue.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 19c75bcb03..a667ec2371 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -365,8 +365,9 @@ static void fillBucket(STokenBucket* pBucket) { bool streamTaskHasAvailableToken(STokenBucket* pBucket) { fillBucket(pBucket); if (pBucket->numOfToken > 0) { - qDebug("remain token:%d", pBucket->numOfToken-1); - return --pBucket->numOfToken; +// qDebug("current token:%d", pBucket->numOfToken); + --pBucket->numOfToken; + return true; } else { return false; } From 50ef500f5152920503e82b2ecfcd59daa7305148 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 10:09:31 +0800 Subject: [PATCH 63/96] refactor(stream): do sink module refactor. --- source/common/src/tdatablock.c | 16 +- source/dnode/vnode/src/tq/tqSink.c | 467 ++++++++++++++------------- source/libs/stream/src/streamQueue.c | 1 - 3 files changed, 256 insertions(+), 228 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f7bb6f85e2..b5239c14b5 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2139,22 +2139,16 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha return TSDB_CODE_FAILED; } - SSmlKv pTag = {.key = "group_id", - .keyLen = sizeof("group_id") - 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .u = groupId, - .length = sizeof(uint64_t)}; + int8_t type = TSDB_DATA_TYPE_UBIGINT; + const char* name = "group_id"; + int32_t len = strlen(name) - 1; + SSmlKv pTag = { .key = name, .keyLen = len, .type = type, .u = groupId, .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); RandTableName rname = { - .tags = tags, - .stbFullName = stbFullName, - .stbFullNameLen = strlen(stbFullName), - .ctbShortName = cname, - }; + .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; buildChildTableName(&rname); - taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 88bdc8a4d9..c84ab831bd 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -24,13 +24,13 @@ typedef struct STableSinkInfo { tstr name; } STableSinkInfo; -static int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, - SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData); +static int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock, + SSubmitTbData* pTableData); static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid); +static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, + SSubmitTbData* pTableData); static int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); -static void fillBucket(STokenBucket* pBucket); -static bool hasAvailableToken(STokenBucket* pBucket); int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr) { @@ -246,11 +246,12 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S return code; } -static int32_t doBuildSubmitAndSendMsg(SVnode* pVnode, SStreamTask* pTask, int32_t numOfBlocks, SSubmitReq2* pReq) { +static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks) { const char* id = pTask->id.idStr; int32_t vgId = TD_VID(pVnode); int32_t len = 0; void* pBuf = NULL; + int32_t numOfFinalBlocks = taosArrayGetSize(pReq->aSubmitTbData); int32_t code = tqBuildSubmitReq(pReq, vgId, &pBuf, &len); if (code != TSDB_CODE_SUCCESS) { @@ -261,7 +262,8 @@ static int32_t doBuildSubmitAndSendMsg(SVnode* pVnode, SStreamTask* pTask, int32 SRpcMsg msg = {.msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len}; code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); if (code == TSDB_CODE_SUCCESS) { - tqDebug("s-task:%s vgId:%d send submit %d blocks into dstTables completed", id, vgId, numOfBlocks); + tqDebug("s-task:%s vgId:%d comp %d blocks into %d and send to dstTable(s) completed", id, vgId, numOfBlocks, + numOfFinalBlocks); } else { tqError("s-task:%s failed to put into write-queue since %s", id, terrstr()); } @@ -272,13 +274,61 @@ static int32_t doBuildSubmitAndSendMsg(SVnode* pVnode, SStreamTask* pTask, int32 SSinkTaskRecorder* pRec = &pTask->sinkRecorder; double el = (taosGetTimestampMs() - pTask->tsInfo.sinkStart) / 1000.0; tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) in %" PRId64 - " submit into dst table, duration:%.2f Sec.", + " submit into dst table, duration:%.2f Sec.", pTask->id.idStr, vgId, pRec->numOfBlocks, pRec->numOfRows, pRec->numOfSubmit, el); } return TSDB_CODE_SUCCESS; } +// merge the new submit table block with the existed blocks +// if ts in the new data block overlap with existed one, replace it +static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id) { + int32_t oldLen = taosArrayGetSize(pExisted->aRowP); + int32_t newLen = taosArrayGetSize(pNew->aRowP); + + int32_t j = 0, k = 0; + SArray* pFinal = taosArrayInit(oldLen + newLen, POINTER_BYTES); + if (pFinal == NULL) { + tqError("s-task:%s failed to prepare merge result datablock, code:%s", id, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return TSDB_CODE_OUT_OF_MEMORY; + } + + while (j < newLen && k < oldLen) { + SRow* pNewRow = taosArrayGetP(pNew->aRowP, j); + SRow* pOldRow = taosArrayGetP(pExisted->aRowP, k); + if (pNewRow->ts <= pOldRow->ts) { + taosArrayPush(pFinal, &pNewRow); + j += 1; + + if (pNewRow->ts == pOldRow->ts) { + k += 1; + } + } else { + taosArrayPush(pFinal, &pOldRow); + k += 1; + } + } + + while (j < newLen) { + SRow* pRow = taosArrayGetP(pNew->aRowP, j++); + taosArrayPush(pFinal, &pRow); + } + + while (k < oldLen) { + SRow* pRow = taosArrayGetP(pExisted->aRowP, k++); + taosArrayPush(pFinal, &pRow); + } + + taosArrayDestroy(pNew->aRowP); + taosArrayDestroy(pExisted->aRowP); + pExisted->aRowP = pFinal; + + tqDebug("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id, + (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (pNew->pCreateTbReq != NULL)); + return TSDB_CODE_SUCCESS; +} + void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { const SArray* pBlocks = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; @@ -294,16 +344,16 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { pTask->tsInfo.sinkStart = taosGetTimestampMs(); } - bool isMixBlocks = true; + bool onlySubmitData = true; for(int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* p = taosArrayGet(pBlocks, i); if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) { - isMixBlocks = true; + onlySubmitData = false; break; } } - if (isMixBlocks) { + if (!onlySubmitData) { tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id, numOfBlocks); @@ -326,10 +376,18 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { } SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - code = doBuildSubmitFromResBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); - taosArrayPush(submitReq.aSubmitTbData, &tbData); + code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } - code = doBuildSubmitAndSendMsg(pVnode, pTask, 1, &submitReq); + code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1); } } } else { @@ -345,91 +403,63 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { } bool hasSubmit = false; - for (int32_t i = 0; i < numOfBlocks; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - code = doBuildAndSendDeleteMsg(pVnode, stbFullName, pDataBlock, pTask, suid); - } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { - code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); - } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { + if (pDataBlock->info.type == STREAM_CHECKPOINT) { continue; - } else { - hasSubmit = true; - pTask->sinkRecorder.numOfBlocks += 1; + } - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - code = doBuildSubmitFromResBlock(pVnode, i, stbFullName, suid, pDataBlock, pTask, &tbData); + hasSubmit = true; + pTask->sinkRecorder.numOfBlocks += 1; + uint64_t groupId = pDataBlock->info.id.groupId; - int32_t* index = taosHashGet(pTableIndexMap, &tbData.uid, sizeof(tbData.uid)); - if (index == NULL) { // no data yet, append it - taosArrayPush(submitReq.aSubmitTbData, &tbData); + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; - taosHashPut(pTableIndexMap, &tbData.uid, sizeof(tbData.uid), &size, sizeof(size)); - } else { - SSubmitTbData* pExisted = taosArrayGet(submitReq.aSubmitTbData, *index); - // merge the new submit table block with the existed blocks - // if ts in the new data block overlap with existed one, replace it - int32_t oldLen = taosArrayGetSize(pExisted->aRowP); - int32_t newLen = taosArrayGetSize(tbData.aRowP); - - int32_t j = 0, k = 0; - SArray* pFinal = taosArrayInit(oldLen + newLen, POINTER_BYTES); - while (j < newLen && k < oldLen) { - SRow* pNewRow = taosArrayGetP(tbData.aRowP, j); - SRow* pOldRow = taosArrayGetP(pExisted->aRowP, k); - if (pNewRow->ts <= pOldRow->ts) { - taosArrayPush(pFinal, &pNewRow); - if (pNewRow->ts < pOldRow->ts) { - j += 1; - } else { - j += 1; - k += 1; - } - } else { - taosArrayPush(pFinal, &pOldRow); - k += 1; - } - } - - while (j < newLen) { - SRow* pRow = taosArrayGetP(tbData.aRowP, j++); - taosArrayPush(pFinal, &pRow); - } - - while (k < oldLen) { - SRow* pRow = taosArrayGetP(pExisted->aRowP, k++); - taosArrayPush(pFinal, &pRow); - } - - taosArrayDestroy(tbData.aRowP); - taosArrayDestroy(pExisted->aRowP); - pExisted->aRowP = pFinal; - - tqDebug("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id, - (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (tbData.pCreateTbReq != NULL)); + int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId)); + if (index == NULL) { // no data yet, append it + code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; } - pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; + code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + + int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; + taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); + } else { + code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + SSubmitTbData* pExisted = taosArrayGet(submitReq.aSubmitTbData, *index); + code = doMergeExistedRows(pExisted, &tbData, id); + if (code != TSDB_CODE_SUCCESS) { + continue; + } } + + pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; } taosHashCleanup(pTableIndexMap); if (hasSubmit) { - doBuildSubmitAndSendMsg(pVnode, pTask, numOfBlocks, &submitReq); + doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); } else { tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); } } - - // TODO: change } int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, - int64_t suid) { + int64_t suid) { SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))}; int32_t code = tqBuildDeleteReq(stbFullName, pDataBlock, &deleteReq, pTask->id.idStr); @@ -605,153 +635,21 @@ static int32_t tsAscendingSortFn(const void* p1, const void* p2) { } } -int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbFullName, int64_t suid, - SSDataBlock* pDataBlock, SStreamTask* pTask, SSubmitTbData* pTableData) { - int32_t numOfRows = pDataBlock->info.rows; - int32_t vgId = TD_VID(pVnode); - uint64_t groupId = pDataBlock->info.id.groupId; - STSchema* pTSchema = pTask->tbSink.pTSchema; - int32_t code = TSDB_CODE_SUCCESS; - SArray* pVals = NULL; - const char* id = pTask->id.idStr; - - tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, - id, blockIndex + 1, numOfRows, suid); +static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) { + int32_t numOfRows = pDataBlock->info.rows; + int32_t code = TSDB_CODE_SUCCESS; + SArray* pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); pTableData->aRowP = taosArrayInit(numOfRows, sizeof(SRow*)); - pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); if (pTableData->aRowP == NULL || pVals == NULL) { pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); - code = TSDB_CODE_OUT_OF_MEMORY; - tqError("s-task:%s vgId:%d failed to prepare write stream res blocks, code:%s", id, vgId, tstrerror(code)); + tqError("s-task:%s failed to prepare write stream res blocks, code:%s", id, tstrerror(code)); return code; } - STableSinkInfo* pTableSinkInfo = NULL; - bool exist = tqGetTableInfo(pTask->tbSink.pTblInfo, groupId, &pTableSinkInfo); - - char* dstTableName = pDataBlock->info.parTbName; - if (exist) { - if (dstTableName[0] == 0) { - tstrncpy(dstTableName, pTableSinkInfo->name.data, pTableSinkInfo->name.len + 1); - tqDebug("s-task:%s vgId:%d, gropuId:%" PRIu64 " datablock table name is null, set name:%s", id, vgId, groupId, - dstTableName); - } else { - if (pTableSinkInfo->uid != 0) { - tqDebug("s-task:%s write %d rows into groupId:%" PRIu64 " dstTable:%s(uid:%" PRIu64 ")", id, numOfRows, groupId, - dstTableName, pTableSinkInfo->uid); - } else { - tqDebug("s-task:%s write %d rows into groupId:%" PRIu64 " dstTable:%s(not set uid yet for the secondary block)", - id, numOfRows, groupId, dstTableName); - } - } - } else { // not exist - if (dstTableName[0] == 0) { - memset(dstTableName, 0, TSDB_TABLE_NAME_LEN); - buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); - } - - int32_t nameLen = strlen(dstTableName); - pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen); - - pTableSinkInfo->name.len = nameLen; - memcpy(pTableSinkInfo->name.data, dstTableName, nameLen); - tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); - } - - if (exist) { - pTableData->uid = pTableSinkInfo->uid; - - if (pTableData->uid == 0) { - tqDebug("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); - } - - while (pTableSinkInfo->uid == 0) { - if (streamTaskShouldStop(&pTask->status)) { - tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); - taosArrayDestroy(pVals); - - return TSDB_CODE_SUCCESS; - } - - // wait for the table to be created - SMetaReader mr = {0}; - metaReaderDoInit(&mr, pVnode->pMeta, 0); - - code = metaGetTableEntryByName(&mr, dstTableName); - if (code == 0) { // table alreay exists, check its type and uid - bool isValid = isValidDestChildTable(&mr, vgId, dstTableName, suid); - if (!isValid) { // not valid table, ignore it - metaReaderClear(&mr); - - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); - taosArrayDestroy(pVals); - - return TSDB_CODE_SUCCESS; - } else { - tqDebug("s-task:%s set uid:%"PRIu64" for dstTable:%s from meta", id, mr.me.uid, pTableSinkInfo->name.data); - - pTableData->uid = mr.me.uid; - pTableSinkInfo->uid = mr.me.uid; - metaReaderClear(&mr); - } - } else { // not exist, wait and retry - metaReaderClear(&mr); - taosMsleep(100); - tqDebug("s-task:%s wait 100ms for the table:%s ready before insert data", id, dstTableName); - } - } - - } else { - // todo: this check is not safe, and results in losing of submit message from WAL. - // The auto-create option will always set to be open for those submit messages, which arrive during the period - // the creating of the destination table, due to the absence of the user-specified table in TSDB. When scanning - // data from WAL, those submit messages, with auto-created table option, will be discarded expect the first, for - // those mismatched table uids. Only the FIRST table has the correct table uid, and those remain all have - // randomly generated false table uid in the WAL. - SMetaReader mr = {0}; - metaReaderDoInit(&mr, pVnode->pMeta, 0); - - // table not in cache, let's try the extract it from tsdb meta - if (metaGetTableEntryByName(&mr, dstTableName) < 0) { - metaReaderClear(&mr); - - tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); - - pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; - pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock); - if (pTableData->pCreateTbReq == NULL) { - tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); - - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); - taosArrayDestroy(pVals); - - return terrno; - } - - doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, 0, id); - } else { - bool isValid = isValidDestChildTable(&mr, vgId, dstTableName, suid); - if (!isValid) { - metaReaderClear(&mr); - taosMemoryFree(pTableSinkInfo); - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); - taosArrayDestroy(pVals); - return TSDB_CODE_SUCCESS; - } else { - pTableData->uid = mr.me.uid; - metaReaderClear(&mr); - - doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, pTableData->uid, id); - } - } - } - - // rows for (int32_t j = 0; j < numOfRows; j++) { taosArrayClear(pVals); @@ -795,7 +693,6 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF code = tRowBuild(pVals, (STSchema*)pTSchema, &pRow); if (code != TSDB_CODE_SUCCESS) { tDestroySubmitTbData(pTableData, TSDB_MSG_FLG_ENCODE); - pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pVals); return code; @@ -805,9 +702,147 @@ int32_t doBuildSubmitFromResBlock(SVnode* pVnode, int32_t blockIndex, char* stbF taosArrayPush(pTableData->aRowP, &pRow); } + taosArrayDestroy(pVals); + return TSDB_CODE_SUCCESS; +} + +int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, + SSubmitTbData* pTableData) { + uint64_t groupId = pDataBlock->info.id.groupId; + char* dstTableName = pDataBlock->info.parTbName; + int32_t numOfRows = pDataBlock->info.rows; + const char* id = pTask->id.idStr; + int64_t suid = pTask->tbSink.stbUid; + STSchema* pTSchema = pTask->tbSink.pTSchema; + int32_t vgId = TD_VID(pVnode); + STableSinkInfo* pTableSinkInfo = NULL; + + bool alreadyCached = tqGetTableInfo(pTask->tbSink.pTblInfo, groupId, &pTableSinkInfo); + if (alreadyCached) { + if (dstTableName[0] == 0) { // data block does not set the destination table name + tstrncpy(dstTableName, pTableSinkInfo->name.data, pTableSinkInfo->name.len + 1); + tqDebug("s-task:%s vgId:%d, gropuId:%" PRIu64 " datablock table name is null, set name:%s", id, vgId, groupId, + dstTableName); + } else { + if (pTableSinkInfo->uid != 0) { + tqDebug("s-task:%s write %d rows into groupId:%" PRIu64 " dstTable:%s(uid:%" PRIu64 ")", id, numOfRows, groupId, + dstTableName, pTableSinkInfo->uid); + } else { + tqDebug("s-task:%s write %d rows into groupId:%" PRIu64 " dstTable:%s(not set uid yet for the secondary block)", + id, numOfRows, groupId, dstTableName); + } + } + } else { // this groupId has not been kept in cache yet + if (dstTableName[0] == 0) { + memset(dstTableName, 0, TSDB_TABLE_NAME_LEN); + buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); + } + + int32_t nameLen = strlen(dstTableName); + pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen); + + pTableSinkInfo->name.len = nameLen; + memcpy(pTableSinkInfo->name.data, dstTableName, nameLen); + tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); + } + + if (alreadyCached) { + pTableData->uid = pTableSinkInfo->uid; + + if (pTableData->uid == 0) { + tqDebug("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); + } + + while (pTableSinkInfo->uid == 0) { + if (streamTaskShouldStop(&pTask->status)) { + tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); + return TSDB_CODE_SUCCESS; + } + + // wait for the table to be created + SMetaReader mr = {0}; + metaReaderDoInit(&mr, pVnode->pMeta, 0); + + int32_t code = metaGetTableEntryByName(&mr, dstTableName); + if (code == 0) { // table already exists, check its type and uid + bool isValid = isValidDestChildTable(&mr, vgId, dstTableName, suid); + if (!isValid) { // not valid table, ignore it + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; + } else { + tqDebug("s-task:%s set uid:%" PRIu64 " for dstTable:%s from meta", id, mr.me.uid, pTableSinkInfo->name.data); + + pTableData->uid = mr.me.uid; + pTableSinkInfo->uid = mr.me.uid; + metaReaderClear(&mr); + } + } else { // not exist, wait and retry + metaReaderClear(&mr); + taosMsleep(100); + tqDebug("s-task:%s wait 100ms for the table:%s ready before insert data", id, dstTableName); + } + } + + } else { + // todo: this check is not safe, and results in losing of submit message from WAL. + // The auto-create option will always set to be open for those submit messages, which arrive during the period + // the creating of the destination table, due to the absence of the user-specified table in TSDB. When scanning + // data from WAL, those submit messages, with auto-created table option, will be discarded expect the first, for + // those mismatched table uids. Only the FIRST table has the correct table uid, and those remain all have + // randomly generated false table uid in the WAL. + SMetaReader mr = {0}; + metaReaderDoInit(&mr, pVnode->pMeta, 0); + + // table not in cache, let's try the extract it from tsdb meta + if (metaGetTableEntryByName(&mr, dstTableName) < 0) { + metaReaderClear(&mr); + + tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); + + pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock); + if (pTableData->pCreateTbReq == NULL) { + tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); + return terrno; + } + + doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, 0, id); + } else { + bool isValid = isValidDestChildTable(&mr, vgId, dstTableName, suid); + if (!isValid) { + metaReaderClear(&mr); + taosMemoryFree(pTableSinkInfo); + tqError("s-task:%s vgId:%d table:%s already exists, but not child table, stream results is discarded", id, vgId, + dstTableName); + return TSDB_CODE_PAR_DUPLICATED_COLUMN; + } else { + pTableData->uid = mr.me.uid; + metaReaderClear(&mr); + doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, pTableData->uid, id); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock, + SSubmitTbData* pTableData) { + int32_t numOfRows = pDataBlock->info.rows; + const char* id = pTask->id.idStr; + + tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, + id, blockIndex + 1, numOfRows, pTask->tbSink.stbUid); + char* dstTableName = pDataBlock->info.parTbName; + + // convert all rows + int32_t code = doConvertRows(pTableData, pTask->tbSink.pTSchema, pDataBlock, id); + if (code != TSDB_CODE_SUCCESS) { + tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno)); + return code; + } + taosArraySort(pTableData->aRowP, tsAscendingSortFn); tqDebug("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); - - taosArrayDestroy(pVals); return code; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index a667ec2371..62873f37bc 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -365,7 +365,6 @@ static void fillBucket(STokenBucket* pBucket) { bool streamTaskHasAvailableToken(STokenBucket* pBucket) { fillBucket(pBucket); if (pBucket->numOfToken > 0) { -// qDebug("current token:%d", pBucket->numOfToken); --pBucket->numOfToken; return true; } else { From 0980e021e57202935545910fe2a6d52b2e3937ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 13:37:19 +0800 Subject: [PATCH 64/96] fix(stream): set the correct start version for the step 2 stage in fill history. --- source/dnode/vnode/src/tq/tq.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3c4ad50a20..0b50315636 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -839,7 +839,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } - tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " currentVer:%" PRId64 + tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), @@ -1119,8 +1119,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { pTask->tsInfo.step2Start = taosGetTimestampMs(); streamSetParamForStreamScannerStep2(pTask, pRange, pWindow); - int64_t dstVer = pTask->dataRange.range.minVer - 1; - + int64_t dstVer = pTask->dataRange.range.minVer; pTask->chkInfo.nextProcessVer = dstVer; walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer); tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer, From c729fc989fb8e956004d1e972297cfff705d65a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 13:49:43 +0800 Subject: [PATCH 65/96] fix(stream): set the init start timestamp. --- source/dnode/vnode/src/tq/tq.c | 3 +++ source/dnode/vnode/src/tq/tqStreamTask.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0b50315636..3d1827e8bc 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -983,6 +983,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms bool restored = pTq->pVnode->restored; if (p != NULL && restored) { + p->tsInfo.init = taosGetTimestampMs(); + tqDebug("s-task:%s set the init ts:%"PRId64, p->id.idStr, p->tsInfo.init); + streamTaskCheckDownstream(p); } else if (!restored) { tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 955614297b..9bae6d6e33 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -94,6 +94,9 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) { continue; } + pTask->tsInfo.init = taosGetTimestampMs(); + tqDebug("s-task:%s set the init ts:%"PRId64, pTask->id.idStr, pTask->tsInfo.init); + streamSetStatusNormal(pTask); streamTaskCheckDownstream(pTask); From 3b79ab32668fead0751653b71760581684ca71ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 13:50:55 +0800 Subject: [PATCH 66/96] fix(stream): reset the init ts. --- source/libs/stream/src/streamTask.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index ea4c2e71bc..663deca171 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -503,8 +503,9 @@ int32_t streamTaskStop(SStreamTask* pTask) { taosMsleep(100); } + pTask->tsInfo.init = 0; int64_t el = taosGetTimestampMs() - st; - qDebug("vgId:%d s-task:%s is closed in %" PRId64 " ms", pMeta->vgId, pTask->id.idStr, el); + qDebug("vgId:%d s-task:%s is closed in %" PRId64 " ms, and reset init ts", pMeta->vgId, pTask->id.idStr, el); return 0; } From 45880f70d4cc5748db15d70e3abc7f9dc1d1c774 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 13:57:33 +0800 Subject: [PATCH 67/96] fix(stream): fix display error in logs. --- source/libs/stream/src/streamRecover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index cf04659ac1..0693495b14 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -41,8 +41,8 @@ static void streamTaskSetReady(SStreamTask* pTask, int32_t numOfReqs) { pTask->status.downstreamReady = 1; int64_t el = (taosGetTimestampMs() - pTask->tsInfo.init); - qDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%dms, task status:%s", - pTask->id.idStr, numOfReqs, (int32_t) el, streamGetTaskStatusStr(pTask->status.taskStatus)); + qDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%"PRId64"ms, task status:%s", + pTask->id.idStr, numOfReqs, el, streamGetTaskStatusStr(pTask->status.taskStatus)); } int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { From 5ab739ae466e8fdbbe1d2f0598e6eaf45b63b4a2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 15:29:12 +0800 Subject: [PATCH 68/96] fix(stream): fix memory leak. --- source/dnode/vnode/src/tq/tqSink.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index c84ab831bd..a5a685580d 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -24,12 +24,12 @@ typedef struct STableSinkInfo { tstr name; } STableSinkInfo; +static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, + SSubmitTbData* pTableData); static int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock, SSubmitTbData* pTableData); static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid); -static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, - SSubmitTbData* pTableData); static int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, @@ -303,6 +303,7 @@ static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* if (pNewRow->ts == pOldRow->ts) { k += 1; + tRowDestroy(pOldRow); } } else { taosArrayPush(pFinal, &pOldRow); @@ -803,6 +804,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock); if (pTableData->pCreateTbReq == NULL) { tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); + taosMemoryFree(pTableSinkInfo); return terrno; } @@ -827,7 +829,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat } int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock, - SSubmitTbData* pTableData) { + SSubmitTbData* pTableData) { int32_t numOfRows = pDataBlock->info.rows; const char* id = pTask->id.idStr; From 24524e1a8d15a74d545a3698ee25c54c8aa78f3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 19:10:54 +0800 Subject: [PATCH 69/96] fix(stream): fix the invalid length of tag column name. --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b5239c14b5..53646b84b3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2141,7 +2141,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha int8_t type = TSDB_DATA_TYPE_UBIGINT; const char* name = "group_id"; - int32_t len = strlen(name) - 1; + int32_t len = strlen(name); SSmlKv pTag = { .key = name, .keyLen = len, .type = type, .u = groupId, .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); From e8f7146a2ce1c42467e686242a2113e0fb08c55f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 22:19:10 +0800 Subject: [PATCH 70/96] fix(tsdb): check the schema before merge rows in buffer, and do some refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 281 ++++++++++++------------ 1 file changed, 136 insertions(+), 145 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index fa3e00ce0f..beaeefc048 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1504,6 +1504,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } + + int32_t code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to init row merger, code:%s", tstrerror(code)); + return code; + } } int64_t minKey = 0; @@ -1535,15 +1541,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - // todo remove init - bool init = false; - // ASC: file block ---> last block -----> imem -----> mem // DESC: mem -----> imem -----> last block -----> file block if (pReader->info.order == TSDB_ORDER_ASC) { if (minKey == key) { - init = true; - int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1552,47 +1554,44 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - if (init) { - tsdbRowMergerAdd(pMerger, fRow1, NULL); - } else { - init = true; - int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == k.ts) { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return terrno; - } - if (init) { - tsdbRowMergerAdd(pMerger, pRow, pSchema); - } else { - init = true; - int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; + STSchema* pTSchema = NULL; + if (pRow->type == TSDBROW_ROW_FMT) { + pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pTSchema == NULL) { + return terrno; } } - int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } } } else { if (minKey == k.ts) { - init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return terrno; + STSchema* pTSchema = NULL; + if (pRow->type == TSDBROW_ROW_FMT) { + pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pTSchema == NULL) { + return terrno; + } } - int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1605,28 +1604,18 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - if (init) { - tsdbRowMergerAdd(pMerger, fRow1, NULL); - } else { - init = true; - int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == key) { - if (init) { - tsdbRowMergerAdd(pMerger, &fRow, NULL); - } else { - init = true; - int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } @@ -1674,7 +1663,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1699,7 +1688,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1742,6 +1731,12 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (ps == NULL) { return terrno; } + + int32_t code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to init row merger, code:%s", tstrerror(code)); + return code; + } } if (hasDataInFileBlock(pBlockData, pDumpInfo)) { @@ -1768,7 +1763,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader } // the following for key == tsLast SRow* pTSRow = NULL; - int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1776,7 +1771,10 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); @@ -1816,14 +1814,21 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return code; + + STSchema* pSchema = NULL; + if (pRow->type == TSDBROW_ROW_FMT) { + pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } } - STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - if (piSchema == NULL) { - return code; + STSchema* piSchema = NULL; + if (piRow->type == TSDBROW_ROW_FMT) { + piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return code; + } } // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized @@ -1833,6 +1838,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to init row merger, code:%s", tstrerror(code)); + return code; + } } int64_t minKey = 0; @@ -1872,15 +1883,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } } - bool init = false; - // ASC: file block -----> last block -----> imem -----> mem // DESC: mem -----> imem -----> last block -----> file block if (ASCENDING_TRAVERSE(pReader->info.order)) { if (minKey == key) { - init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1890,14 +1898,9 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - if (init) { - tsdbRowMergerAdd(pMerger, pRow1, NULL); - } else { - init = true; - code = tsdbRowMergerAdd(pMerger, pRow1, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, @@ -1905,14 +1908,9 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == ik.ts) { - if (init) { - tsdbRowMergerAdd(pMerger, piRow, piSchema); - } else { - init = true; - code = tsdbRowMergerAdd(pMerger, piRow, piSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; } code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); @@ -1922,15 +1920,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == k.ts) { - if (init) { - tsdbRowMergerAdd(pMerger, pRow, pSchema); - } else { - // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerAdd(pMerger, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; } + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1938,7 +1932,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } } else { if (minKey == k.ts) { - init = true; code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1951,15 +1944,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == ik.ts) { - if (init) { - tsdbRowMergerAdd(pMerger, piRow, piSchema); - } else { - init = true; - code = tsdbRowMergerAdd(pMerger, piRow, piSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; } + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1968,29 +1957,22 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - if (init) { - tsdbRowMergerAdd(pMerger, pRow1, NULL); - } else { - init = true; - code = tsdbRowMergerAdd(pMerger, pRow1, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - if (!init) { - code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - tsdbRowMergerAdd(pMerger, &fRow, NULL); + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } } @@ -2184,6 +2166,12 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc if (ps == NULL) { return terrno; } + + code = tsdbRowMergerInit(pMerger, ps); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to init row merger, code:%s", tstrerror(code)); + return code; + } } if (copied) { @@ -2193,7 +2181,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SRow* pTSRow = NULL; - code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3328,16 +3316,15 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } + STSchema* pTSchema = NULL; if (pRow->type == TSDBROW_ROW_FMT) { - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); if (pTSchema == NULL) { return terrno; } - - tsdbRowMergerAdd(pMerger, pRow, pTSchema); - } else { // column format - tsdbRowMergerAdd(pMerger, pRow, NULL); } + + tsdbRowMergerAdd(pMerger, pRow, pTSchema); } return TSDB_CODE_SUCCESS; @@ -3473,31 +3460,30 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, int32_t code = 0; // start to merge duplicated rows - if (current.type == TSDBROW_ROW_FMT) { - // get the correct schema for data in memory - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + STSchema* pTSchema = NULL; + if (current.type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory + pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); if (pTSchema == NULL) { return terrno; } + } - code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + STSchema* pTSchema1 = NULL; + if (pNextRow->type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory + pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); if (pTSchema1 == NULL) { return terrno; } + } - tsdbRowMergerAdd(&pReader->status.merger, pNextRow, pTSchema1); - } else { // let's merge rows in file block - code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pReader->info.pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - tsdbRowMergerAdd(&pReader->status.merger, pNextRow, NULL); + code = tsdbRowMergerAdd(&pReader->status.merger, pNextRow, pTSchema1); + if (code != TSDB_CODE_SUCCESS) { + return code; } code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, pReader); @@ -3523,14 +3509,21 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return terrno; + + STSchema* pSchema = NULL; + if (pRow->type == TSDBROW_ROW_FMT) { + pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } } - STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - if (piSchema == NULL) { - return terrno; + STSchema* piSchema = NULL; + if (piRow->type == TSDBROW_ROW_FMT) { + piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return terrno; + } } if (ASCENDING_TRAVERSE(pReader->info.order)) { // ascending order imem --> mem @@ -4898,10 +4891,10 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pMem = pTsdb->mem; pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); if (pSnap->pNode == NULL) { - taosThreadRwlockUnlock(&pTsdb->rwLock); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + pSnap->pNode->pQHandle = pReader; pSnap->pNode->reseek = reseek; @@ -4912,10 +4905,10 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pIMem = pTsdb->imem; pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); if (pSnap->pINode == NULL) { - taosThreadRwlockUnlock(&pTsdb->rwLock); code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + pSnap->pINode->pQHandle = pReader; pSnap->pINode->reseek = reseek; @@ -4924,18 +4917,16 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs // fs code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pSnap->pfSetArray); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; + if (code == TSDB_CODE_SUCCESS) { + tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), pReader->idStr); } - // unlock +_exit: taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d take read snapshot failed, %s code:%s", TD_VID(pTsdb->pVnode), pReader->idStr, tstrerror(code)); -_exit: - if (code) { *ppSnap = NULL; if (pSnap) { if (pSnap->pNode) taosMemoryFree(pSnap->pNode); From a5b93aaf97e831f810216468cfc8b4d466e9b2df Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Sep 2023 23:55:34 +0800 Subject: [PATCH 71/96] fix(tsdb): fix invalid read. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index beaeefc048..e4aba7011c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4918,14 +4918,14 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs // fs code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pSnap->pfSetArray); if (code == TSDB_CODE_SUCCESS) { - tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), pReader->idStr); + tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); } _exit: taosThreadRwlockUnlock(&pTsdb->rwLock); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d take read snapshot failed, %s code:%s", TD_VID(pTsdb->pVnode), pReader->idStr, tstrerror(code)); + tsdbError("vgId:%d take read snapshot failed, code:%s", TD_VID(pTsdb->pVnode), tstrerror(code)); *ppSnap = NULL; if (pSnap) { From f602aa965f74a1548e2297445baaf9efe829f473 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 09:16:18 +0800 Subject: [PATCH 72/96] fix(stream): fix coverity scan issues. --- source/dnode/mnode/impl/src/mndStream.c | 3 ++- source/dnode/vnode/src/tq/tqUtil.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 7 ++++++- source/libs/parser/src/parTranslater.c | 2 -- source/libs/stream/src/stream.c | 1 - source/libs/stream/src/streamCheckpoint.c | 8 ++++++-- source/libs/stream/src/streamMeta.c | 4 ++++ source/libs/stream/src/streamQueue.c | 12 ++++++++++-- 8 files changed, 30 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 66acbcc05b..049b4e737a 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1922,6 +1922,7 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans) { SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); if (pCommitRaw == NULL) { mError("failed to encode stream since %s", terrstr()); + mndTransDrop(pTrans); return -1; } @@ -1988,6 +1989,7 @@ static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgr if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pBuf); taosWUnLockLatch(&pStream->lock); + mndTransDrop(pTrans); return -1; } } @@ -1998,7 +2000,6 @@ static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgr int32_t code = mndPersistTransLog(pStream, pTrans); if (code != TSDB_CODE_SUCCESS) { sdbRelease(pMnode->pSdb, pStream); - mndTransDrop(pTrans); return -1; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index f1b154a4ac..52862ea67b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -437,6 +437,8 @@ int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStream taosArrayDestroy(pRes->uidList); *pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); if (*pRefBlock == NULL) { + blockDataCleanup(pDelBlock); + taosMemoryFree(pDelBlock); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 1f3c8b54ec..74eb1c7302 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -273,7 +273,12 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); tBrinBlockClear(&pIter->block); - tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); + int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); + return NULL; + } + pIter->recordIndex = -1; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8008f4397e..cae8c5d5e8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7007,7 +7007,6 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta int32_t code = nodesListStrictAppend(pParamterList, (SNode*)col); if (code) { - nodesDestroyNode((SNode*)col); nodesDestroyList(pParamterList); return code; } @@ -7025,7 +7024,6 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta } code = nodesListStrictAppend(pProjectionList, pFunc); if (code) { - nodesDestroyNode(pFunc); nodesDestroyList(pProjectionList); return code; } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 1f93498557..32b36f8848 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -83,7 +83,6 @@ static void streamSchedByTimer(void* param, void* tmrId) { atomic_store_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE); pTrigger->pBlock->info.type = STREAM_GET_ALL; if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger) < 0) { - taosFreeQitem(pTrigger); taosTmrReset(streamSchedByTimer, (int32_t)pTask->info.triggerParam, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); return; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 361602fac9..cc93d25fd5 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -125,7 +125,6 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint taosMemoryFree(pBlock); if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pChkpoint) < 0) { - taosFreeQitem(pChkpoint); return TSDB_CODE_OUT_OF_MEMORY; } @@ -271,7 +270,12 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { keys[0] = pId->streamId; keys[1] = pId->taskId; - SStreamTask* p = *(SStreamTask**)taosHashGet(pMeta->pTasks, keys, sizeof(keys)); + SStreamTask** ppTask = taosHashGet(pMeta->pTasks, keys, sizeof(keys)); + if (ppTask == NULL) { + continue; + } + + SStreamTask* p = *ppTask; if (p->info.fillHistory == 1) { continue; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 5bc21286d7..a11f4a8b26 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -418,6 +418,10 @@ int32_t streamMetaGetNumOfStreamTasks(SStreamMeta* pMeta) { int64_t keys[2] = {pId->streamId, pId->taskId}; SStreamTask** p = taosHashGet(pMeta->pTasks, keys, sizeof(keys)); + if (p == NULL) { + continue; + } + if ((*p)->info.fillHistory == 0) { num += 1; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 62873f37bc..bd64e0779a 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -312,12 +312,20 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) } } else if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER || type == STREAM_INPUT__TRANS_STATE) { - taosWriteQitem(pQueue, pItem); + int32_t code = taosWriteQitem(pQueue, pItem); + if (code != TSDB_CODE_SUCCESS) { + taosFreeQitem(pItem); + return code; + } qDebug("s-task:%s level:%d %s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, pTask->info.taskLevel, streamGetBlockTypeStr(type), total, size); } else if (type == STREAM_INPUT__GET_RES) { // use the default memory limit, refactor later. - taosWriteQitem(pQueue, pItem); + int32_t code = taosWriteQitem(pQueue, pItem); + if (code != TSDB_CODE_SUCCESS) { + taosFreeQitem(pItem); + return code; + } qDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); } else { ASSERT(0); From a003db7e1a203f8345e7b0973f24aaa11c725c20 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 12 Sep 2023 10:44:11 +0800 Subject: [PATCH 73/96] reset file handles limit on windows --- source/common/src/tglobal.c | 1 + source/os/src/osFile.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8971c1312c..90812a66b2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1601,6 +1601,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (taosSetS3Cfg(tsCfg) != 0) return -1; } taosSetSystemCfg(tsCfg); + if (taosSetFileHandlesLimit() != 0) return -1; cfgDumpCfg(tsCfg, tsc, false); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index db4807db43..1ce9a348b3 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -954,3 +954,12 @@ cmp_end: return ret; } + +int32_t taosSetFileHandlesLimit() { +#ifdef WINDOWS + const int max_handles = 8192; + int res = _setmaxstdio(max_handles); + return res == max_handles ? 0 : -1; +#endif + return 0; +} From 93159729f6ff566073ab5b037a19d528360ba36c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 11:45:51 +0800 Subject: [PATCH 74/96] fix(stream):check error when vnode is closing. --- include/util/taoserror.h | 2 +- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 402 ++++++++++++++++------------- source/util/src/terror.c | 3 +- 5 files changed, 221 insertions(+), 190 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4a3f0d3a70..613e33fec7 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -788,7 +788,7 @@ int32_t* taosGetErrno(); // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) -#define TSDB_CODE_STREAM_BACKPRESSURE_OUT_OF_QUEUE TAOS_DEF_ERROR_CODE(0, 0x4101) +#define TSDB_CODE_STREAM_EXEC_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x4102) // TDLite #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 14ca7f3b02..c6a424666c 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -155,7 +155,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr); -void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data); +void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t fVer); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3d1827e8bc..e2888e8e9a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -798,7 +798,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; + pTask->tbSink.tbSinkFunc = tqSinkDataIntoDstTable; int32_t ver1 = 1; SMetaInfo info = {0}; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index a5a685580d..7f52750348 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -29,8 +29,18 @@ static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBloc static int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock, SSubmitTbData* pTableData); static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, - int64_t suid); + int64_t suid); static int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); +static int32_t tsAscendingSortFn(const void* p1, const void* p2); +static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id); +static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo, + const char* dstTableName, int64_t* uid); +static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id); +static SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, + SSDataBlock* pDataBlock); +static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); +static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id); +static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks); int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr) { @@ -115,14 +125,6 @@ static bool tqGetTableInfo(SSHashObj* pTableInfoMap,uint64_t groupId, STableSink return false; } -static int32_t tqPutTableInfo(SSHashObj* tblInfo ,uint64_t groupId, STableSinkInfo* pTbl) { - if (tSimpleHashGetSize(tblInfo) > MAX_CACHE_TABLE_INFO_NUM) { - return TSDB_CODE_FAILED; - } - - return tSimpleHashPut(tblInfo, &groupId, sizeof(uint64_t), &pTbl, POINTER_BYTES); -} - static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { void* buf = NULL; int32_t tlen = 0; @@ -246,7 +248,7 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S return code; } -static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks) { +int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks) { const char* id = pTask->id.idStr; int32_t vgId = TD_VID(pVnode); int32_t len = 0; @@ -283,7 +285,7 @@ static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubm // merge the new submit table block with the existed blocks // if ts in the new data block overlap with existed one, replace it -static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id) { +int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id) { int32_t oldLen = taosArrayGetSize(pExisted->aRowP); int32_t newLen = taosArrayGetSize(pNew->aRowP); @@ -330,135 +332,6 @@ static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* return TSDB_CODE_SUCCESS; } -void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, void* data) { - const SArray* pBlocks = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - int64_t suid = pTask->tbSink.stbUid; - char* stbFullName = pTask->tbSink.stbFullName; - STSchema* pTSchema = pTask->tbSink.pTSchema; - int32_t vgId = TD_VID(pVnode); - int32_t numOfBlocks = taosArrayGetSize(pBlocks); - int32_t code = TSDB_CODE_SUCCESS; - const char* id = pTask->id.idStr; - - if (pTask->tsInfo.sinkStart == 0) { - pTask->tsInfo.sinkStart = taosGetTimestampMs(); - } - - bool onlySubmitData = true; - for(int32_t i = 0; i < numOfBlocks; ++i) { - SSDataBlock* p = taosArrayGet(pBlocks, i); - if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) { - onlySubmitData = false; - break; - } - } - - if (!onlySubmitData) { - tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id, - numOfBlocks); - - for(int32_t i = 0; i < numOfBlocks; ++i) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - code = doBuildAndSendDeleteMsg(pVnode, stbFullName, pDataBlock, pTask, suid); - } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { - code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); - } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { - continue; - } else { - pTask->sinkRecorder.numOfBlocks += 1; - - SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; - if (submitReq.aSubmitTbData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - tqError("s-task:%s vgId:%d failed to prepare submit msg in sink task, code:%s", id, vgId, tstrerror(code)); - return; - } - - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); - if (code != TSDB_CODE_SUCCESS) { - continue; - } - - code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); - if (code != TSDB_CODE_SUCCESS) { - continue; - } - - taosArrayPush(submitReq.aSubmitTbData, &tbData); - code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1); - } - } - } else { - tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, merge submit msg", vgId, id, numOfBlocks); - SHashObj* pTableIndexMap = taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - - SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; - if (submitReq.aSubmitTbData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - tqError("s-task:%s vgId:%d failed to prepare submit msg in sink task, code:%s", id, vgId, tstrerror(code)); - taosHashCleanup(pTableIndexMap); - return; - } - - bool hasSubmit = false; - for (int32_t i = 0; i < numOfBlocks; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_CHECKPOINT) { - continue; - } - - hasSubmit = true; - pTask->sinkRecorder.numOfBlocks += 1; - uint64_t groupId = pDataBlock->info.id.groupId; - - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; - - int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId)); - if (index == NULL) { // no data yet, append it - code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); - if (code != TSDB_CODE_SUCCESS) { - continue; - } - - code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); - if (code != TSDB_CODE_SUCCESS) { - continue; - } - - taosArrayPush(submitReq.aSubmitTbData, &tbData); - - int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; - taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); - } else { - code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); - if (code != TSDB_CODE_SUCCESS) { - continue; - } - - SSubmitTbData* pExisted = taosArrayGet(submitReq.aSubmitTbData, *index); - code = doMergeExistedRows(pExisted, &tbData, id); - if (code != TSDB_CODE_SUCCESS) { - continue; - } - } - - pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; - } - - taosHashCleanup(pTableIndexMap); - - if (hasSubmit) { - doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); - } else { - tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); - tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); - } - } -} - int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid) { SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))}; @@ -498,22 +371,25 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* return TSDB_CODE_SUCCESS; } -static bool isValidDestChildTable(SMetaReader* pReader, int32_t vgId, char* ctbName, int64_t suid) { +bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid) { if (pReader->me.type != TSDB_CHILD_TABLE) { tqError("vgId:%d, failed to write into %s, since table type:%d incorrect", vgId, ctbName, pReader->me.type); + terrno = TSDB_CODE_TDB_INVALID_TABLE_TYPE; return false; } if (pReader->me.ctbEntry.suid != suid) { tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid:%" PRId64 ", actual:%" PRId64, vgId, ctbName, suid, pReader->me.ctbEntry.suid); + terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; return false; } + terrno = 0; return true; } -static SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock) { +SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock) { char* ctbName = pDataBlock->info.parTbName; SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)); @@ -570,11 +446,12 @@ static SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, i return pCreateTbReq; } -static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, uint64_t uid, - const char* id) { - pTableSinkInfo->uid = uid; +int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) { + if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) { + return TSDB_CODE_FAILED; + } - int32_t code = tqPutTableInfo(pSinkTableMap, groupId, pTableSinkInfo); + int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableSinkInfo); } else { @@ -625,7 +502,7 @@ int32_t tqBuildSubmitReq(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int return TSDB_CODE_SUCCESS; } -static int32_t tsAscendingSortFn(const void* p1, const void* p2) { +int32_t tsAscendingSortFn(const void* p1, const void* p2) { SRow* pRow1 = *(SRow**) p1; SRow* pRow2 = *(SRow**) p2; @@ -636,7 +513,7 @@ static int32_t tsAscendingSortFn(const void* p1, const void* p2) { } } -static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) { +int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) { int32_t numOfRows = pDataBlock->info.rows; int32_t code = TSDB_CODE_SUCCESS; @@ -707,6 +584,46 @@ static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDa return TSDB_CODE_SUCCESS; } +int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo, + const char* dstTableName, int64_t* uid) { + int32_t vgId = TD_VID(pVnode); + int64_t suid = pTask->tbSink.stbUid; + const char* id = pTask->id.idStr; + + while (pTableSinkInfo->uid == 0) { + if (streamTaskShouldStop(&pTask->status)) { + tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); + return TSDB_CODE_STREAM_EXEC_CANCELLED; + } + + // wait for the table to be created + SMetaReader mr = {0}; + metaReaderDoInit(&mr, pVnode->pMeta, 0); + + int32_t code = metaGetTableEntryByName(&mr, dstTableName); + if (code == 0) { // table already exists, check its type and uid + bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid); + if (isValid) { // not valid table, ignore it + tqDebug("s-task:%s set uid:%" PRIu64 " for dstTable:%s from meta", id, mr.me.uid, pTableSinkInfo->name.data); + ASSERT(terrno == 0); + + // set the destination table uid + (*uid) = mr.me.uid; + pTableSinkInfo->uid = mr.me.uid; + } + + metaReaderClear(&mr); + return terrno; + } else { // not exist, wait and retry + metaReaderClear(&mr); + taosMsleep(100); + tqDebug("s-task:%s wait 100ms for the table:%s ready before insert data", id, dstTableName); + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, SSubmitTbData* pTableData) { uint64_t groupId = pDataBlock->info.id.groupId; @@ -719,6 +636,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat STableSinkInfo* pTableSinkInfo = NULL; bool alreadyCached = tqGetTableInfo(pTask->tbSink.pTblInfo, groupId, &pTableSinkInfo); + if (alreadyCached) { if (dstTableName[0] == 0) { // data block does not set the destination table name tstrncpy(dstTableName, pTableSinkInfo->name.data, pTableSinkInfo->name.len + 1); @@ -741,6 +659,9 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat int32_t nameLen = strlen(dstTableName); pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen); + if (pTableSinkInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pTableSinkInfo->name.len = nameLen; memcpy(pTableSinkInfo->name.data, dstTableName, nameLen); @@ -752,45 +673,16 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat if (pTableData->uid == 0) { tqDebug("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); + return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid); + } else { + tqDebug("s-task:%s set the dstTable uid from cache:%"PRId64, id, pTableData->uid); } - - while (pTableSinkInfo->uid == 0) { - if (streamTaskShouldStop(&pTask->status)) { - tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); - return TSDB_CODE_SUCCESS; - } - - // wait for the table to be created - SMetaReader mr = {0}; - metaReaderDoInit(&mr, pVnode->pMeta, 0); - - int32_t code = metaGetTableEntryByName(&mr, dstTableName); - if (code == 0) { // table already exists, check its type and uid - bool isValid = isValidDestChildTable(&mr, vgId, dstTableName, suid); - if (!isValid) { // not valid table, ignore it - metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; - } else { - tqDebug("s-task:%s set uid:%" PRIu64 " for dstTable:%s from meta", id, mr.me.uid, pTableSinkInfo->name.data); - - pTableData->uid = mr.me.uid; - pTableSinkInfo->uid = mr.me.uid; - metaReaderClear(&mr); - } - } else { // not exist, wait and retry - metaReaderClear(&mr); - taosMsleep(100); - tqDebug("s-task:%s wait 100ms for the table:%s ready before insert data", id, dstTableName); - } - } - } else { - // todo: this check is not safe, and results in losing of submit message from WAL. // The auto-create option will always set to be open for those submit messages, which arrive during the period // the creating of the destination table, due to the absence of the user-specified table in TSDB. When scanning // data from WAL, those submit messages, with auto-created table option, will be discarded expect the first, for // those mismatched table uids. Only the FIRST table has the correct table uid, and those remain all have - // randomly generated false table uid in the WAL. + // randomly generated, but false table uid in the WAL. SMetaReader mr = {0}; metaReaderDoInit(&mr, pVnode->pMeta, 0); @@ -808,19 +700,22 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat return terrno; } - doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, 0, id); + pTableSinkInfo->uid = 0; + doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, id); } else { - bool isValid = isValidDestChildTable(&mr, vgId, dstTableName, suid); + bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid); if (!isValid) { metaReaderClear(&mr); taosMemoryFree(pTableSinkInfo); tqError("s-task:%s vgId:%d table:%s already exists, but not child table, stream results is discarded", id, vgId, dstTableName); - return TSDB_CODE_PAR_DUPLICATED_COLUMN; + return terrno; } else { pTableData->uid = mr.me.uid; + pTableSinkInfo->uid = mr.me.uid; + metaReaderClear(&mr); - doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, pTableData->uid, id); + doPutIntoCache(pTask->tbSink.pTblInfo, pTableSinkInfo, groupId, id); } } } @@ -848,3 +743,140 @@ int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlo tqDebug("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); return code; } + +void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + int32_t vgId = TD_VID(pVnode); + int32_t numOfBlocks = taosArrayGetSize(pBlocks); + int32_t code = TSDB_CODE_SUCCESS; + const char* id = pTask->id.idStr; + + if (pTask->tsInfo.sinkStart == 0) { + pTask->tsInfo.sinkStart = taosGetTimestampMs(); + } + + bool onlySubmitData = true; + for(int32_t i = 0; i < numOfBlocks; ++i) { + SSDataBlock* p = taosArrayGet(pBlocks, i); + if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) { + onlySubmitData = false; + break; + } + } + + if (!onlySubmitData) { + tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, has delete block, submit one-by-one", vgId, id, + numOfBlocks); + + for(int32_t i = 0; i < numOfBlocks; ++i) { + if (streamTaskShouldStop(&pTask->status)) { + return; + } + + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + code = doBuildAndSendDeleteMsg(pVnode, stbFullName, pDataBlock, pTask, suid); + } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + code = doBuildAndSendCreateTableMsg(pVnode, stbFullName, pDataBlock, pTask, suid); + } else if (pDataBlock->info.type == STREAM_CHECKPOINT) { + continue; + } else { + pTask->sinkRecorder.numOfBlocks += 1; + + SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; + if (submitReq.aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + tqError("s-task:%s vgId:%d failed to prepare submit msg in sink task, code:%s", id, vgId, tstrerror(code)); + return; + } + + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; + code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1); + } + } + } else { + tqDebug("vgId:%d, s-task:%s write %d stream resBlock(s) into table, merge submit msg", vgId, id, numOfBlocks); + SHashObj* pTableIndexMap = taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + SSubmitReq2 submitReq = {.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData))}; + if (submitReq.aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + tqError("s-task:%s vgId:%d failed to prepare submit msg in sink task, code:%s", id, vgId, tstrerror(code)); + taosHashCleanup(pTableIndexMap); + return; + } + + bool hasSubmit = false; + for (int32_t i = 0; i < numOfBlocks; i++) { + if (streamTaskShouldStop(&pTask->status)) { + return; + } + + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_CHECKPOINT) { + continue; + } + + hasSubmit = true; + pTask->sinkRecorder.numOfBlocks += 1; + uint64_t groupId = pDataBlock->info.id.groupId; + + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version}; + + int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId)); + if (index == NULL) { // no data yet, append it + code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + + int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; + taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); + } else { + code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + + SSubmitTbData* pExisted = taosArrayGet(submitReq.aSubmitTbData, *index); + code = doMergeExistedRows(pExisted, &tbData, id); + if (code != TSDB_CODE_SUCCESS) { + continue; + } + } + + pTask->sinkRecorder.numOfRows += pDataBlock->info.rows; + } + + taosHashCleanup(pTableIndexMap); + + if (hasSubmit) { + doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); + } else { + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); + } + } +} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 278b23b5a9..461ccc5dce 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -651,8 +651,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SAME_COMMITTED_VALUE, "Same committed valu // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_BACKPRESSURE_OUT_OF_QUEUE,"Out of memory in stream queue") - +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_EXEC_CANCELLED, "Stream task exec cancelled") // TDLite TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags") TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open directory") From 30db04ddb55a176a4f67d4f39dee156699409394 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 14:02:31 +0800 Subject: [PATCH 75/96] fix(test): update test case. --- tests/system-test/7-tmq/tmqMaxGroupIds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqMaxGroupIds.py b/tests/system-test/7-tmq/tmqMaxGroupIds.py index 3bdba8bb6c..5049ee5bd7 100644 --- a/tests/system-test/7-tmq/tmqMaxGroupIds.py +++ b/tests/system-test/7-tmq/tmqMaxGroupIds.py @@ -206,7 +206,7 @@ class TDTestCase: while (1): tdSql.query('show subscriptions;') subscribeNum = tdSql.queryRows - tdLog.info(" get subscriptions count: %d"%(subscribeNum)) + tdLog.info(" get subscriptions count: %d, expected:%d"%(subscribeNum, expectSubscribeNum)) if subscribeNum == expectSubscribeNum: flag = 1 break From e5120fade2919f7ddc3074363ae0eb797508b0f8 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 12 Sep 2023 14:12:21 +0800 Subject: [PATCH 76/96] fix read line on windows --- source/os/src/osFile.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index db4807db43..e68e28af6f 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -841,14 +841,33 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { return -1; } #ifdef WINDOWS - *ptrBuf = taosMemoryMalloc(1024); + size_t bufferSize = 512; + *ptrBuf = taosMemoryMalloc(bufferSize); if (*ptrBuf == NULL) return -1; - if (fgets(*ptrBuf, 1023, pFile->fp) == NULL) { - taosMemoryFreeClear(*ptrBuf); - return -1; + + size_t bytesRead = 0; + size_t totalBytesRead = 0; + + while (1) { + char *result = fgets(*ptrBuf + totalBytesRead, bufferSize - totalBytesRead, pFile->fp); + if (result == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + bytesRead = strlen(*ptrBuf + totalBytesRead); + totalBytesRead += bytesRead; + + if (totalBytesRead < bufferSize - 1 || (*ptrBuf)[totalBytesRead - 1] == '\n') { + break; + } + + bufferSize += 512; + *ptrBuf = taosMemoryRealloc(*ptrBuf, bufferSize); + if (*ptrBuf == NULL) return -1; } - (*ptrBuf)[1023] = 0; - return strlen(*ptrBuf); + + (*ptrBuf)[totalBytesRead] = '\0'; + return totalBytesRead; #else size_t len = 0; return getline(ptrBuf, &len, pFile->fp); From 8787e0d58c3ade7111857e6591a731fd3648a762 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 12 Sep 2023 14:29:42 +0800 Subject: [PATCH 77/96] build failed on linux --- include/os/osFile.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/os/osFile.h b/include/os/osFile.h index da1f8f8b57..63483dc906 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -112,6 +112,8 @@ int32_t taosGetErrorFile(TdFilePtr pFile); int32_t taosCompressFile(char *srcFileName, char *destFileName); +int32_t taosSetFileHandlesLimit(); + #ifdef __cplusplus } #endif From 3e48f3639ced6f09bd5710037aef92bb59db744d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 12 Sep 2023 14:46:32 +0800 Subject: [PATCH 78/96] fix transfer crash --- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 36 ++++++++++---------- source/libs/stream/src/streamMeta.c | 3 ++ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 2d58a10e51..bf81570975 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -165,7 +165,6 @@ struct SStreamTaskWriter { STQ* pTq; int64_t sver; int64_t ever; - TXN* txn; }; int32_t streamTaskSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskWriter** ppWriter) { @@ -182,12 +181,6 @@ int32_t streamTaskSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTa pWriter->sver = sver; pWriter->ever = ever; - if (tdbBegin(pTq->pStreamMeta->db, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - code = -1; - taosMemoryFree(pWriter); - goto _err; - } - *ppWriter = pWriter; tqDebug("vgId:%d, vnode stream-task snapshot writer opened", TD_VID(pTq->pVnode)); return code; @@ -203,30 +196,33 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { int32_t code = 0; STQ* pTq = pWriter->pTq; + taosWLockLatch(&pTq->pStreamMeta->lock); tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode)); if (rollback) { - tdbAbort(pWriter->pTq->pStreamMeta->db, pWriter->txn); + tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); } else { - code = tdbCommit(pWriter->pTq->pStreamMeta->db, pWriter->txn); + code = tdbCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pStreamMeta->db, pWriter->txn); + code = tdbPostCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); if (code) goto _err; } + if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + code = -1; + taosMemoryFree(pWriter); + goto _err; + } + + taosWUnLockLatch(&pTq->pStreamMeta->lock); + taosMemoryFree(pWriter); - - // restore from metastore - // if (tqMetaRestoreHandle(pTq) < 0) { - // goto _err; - // } - return code; _err: tqError("vgId:%d, vnode stream-task snapshot writer failed to close since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code)); + taosWUnLockLatch(&pTq->pStreamMeta->lock); return code; - return 0; } int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t nData) { @@ -247,11 +243,15 @@ int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t } tDecoderClear(&decoder); // tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) + + taosWLockLatch(&pTq->pStreamMeta->lock); int64_t key[2] = {task.streamId, task.taskId}; if (tdbTbUpsert(pTq->pStreamMeta->pTaskDb, key, sizeof(int64_t) << 1, (uint8_t*)pData + sizeof(SSnapDataHdr), - nData - sizeof(SSnapDataHdr), pWriter->txn) < 0) { + nData - sizeof(SSnapDataHdr), pTq->pStreamMeta->txn) < 0) { + taosWUnLockLatch(&pTq->pStreamMeta->lock); return -1; } + taosWUnLockLatch(&pTq->pStreamMeta->lock); } else if (pHdr->type == SNAP_DATA_STREAM_TASK_CHECKPOINT) { // do nothing } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 5bc21286d7..16d01b06a4 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -539,10 +539,13 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } int32_t streamMetaBegin(SStreamMeta* pMeta) { + taosWLockLatch(&pMeta->lock); if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + taosWUnLockLatch(&pMeta->lock); return -1; } + taosWUnLockLatch(&pMeta->lock); return 0; } From ccb75a9c11d6ea5ec7f1b4e6ad2673c65f1b7050 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 12 Sep 2023 14:46:56 +0800 Subject: [PATCH 79/96] fix transfer crash --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 733f073ce9..f19068ea88 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -425,20 +425,6 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } - if (pWriter->pStreamTaskWriter) { - code = streamTaskSnapWriterClose(pWriter->pStreamTaskWriter, rollback); - if (code) goto _exit; - } - - if (pWriter->pStreamStateWriter) { - code = streamStateSnapWriterClose(pWriter->pStreamStateWriter, rollback); - if (code) goto _exit; - - code = streamStateRebuildFromSnap(pWriter->pStreamStateWriter, 0); - pWriter->pStreamStateWriter = NULL; - if (code) goto _exit; - } - if (pWriter->pRsmaSnapWriter) { code = rsmaSnapWriterClose(&pWriter->pRsmaSnapWriter, rollback); if (code) goto _exit; From 78a5680cf492fcd0515137aea947f0a2624569dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 15:59:09 +0800 Subject: [PATCH 80/96] fix(stream): add some logs. --- source/libs/stream/src/stream.c | 11 +++++++++-- source/libs/stream/src/streamQueue.c | 10 +++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 32b36f8848..75ac86b462 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -223,9 +223,16 @@ int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock destroyStreamDataBlock(pBlock); } else { ASSERT(type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH); - code = taosWriteQitem(pTask->outputInfo.queue->pQueue, pBlock); + STaosQueue* pQueue = pTask->outputInfo.queue->pQueue; + code = taosWriteQitem(pQueue, pBlock); + + int32_t total = taosQueueItemSize(pQueue); + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); if (code != 0) { - qError("s-task:%s failed to put res into outputQ", pTask->id.idStr); + qError("s-task:%s failed to put res into outputQ, outputQ items:%d, size:%.2fMiB code:%s, result lost", + pTask->id.idStr, total, size, tstrerror(code)); + } else { + qInfo("s-task:%s data put into outputQ, outputQ items:%d, size:%.2fMiB", pTask->id.idStr, total, size); } streamDispatchStreamBlock(pTask); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index bd64e0779a..510c8dc9b1 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -274,9 +274,9 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pQueue)) { -// qError( -// "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", -// pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); + qTrace( + "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", + pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); taosFreeQitem(pItem); return -1; @@ -298,8 +298,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { if (streamQueueIsFull(pQueue)) { -// qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", -// pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); + qTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", + pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); destroyStreamDataBlock((SStreamDataBlock*)pItem); return -1; } From 8e064cdd03164c8d19d35bf1586dc4aeeb36e52d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 12 Sep 2023 16:41:14 +0800 Subject: [PATCH 81/96] fix merge error --- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 61fc3c7ae9..4a1b3961cd 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -51,7 +51,7 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS SStreamSnapReader* pSnapReader = NULL; - if (streamSnapReaderOpen(pTq, sver, chkpId, pTq->path, &pSnapReader) == 0) { + if (streamSnapReaderOpen(meta, sver, chkpId, pTq->path, &pSnapReader) == 0) { pReader->complete = 1; } else { code = -1; From e457d94ac6a97c261262d0f23b9acccb7e760af2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 18:09:02 +0800 Subject: [PATCH 82/96] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 6 ++-- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 2 +- source/dnode/vnode/src/tq/tqStreamTask.c | 2 +- source/libs/stream/src/stream.c | 2 +- source/libs/stream/src/streamData.c | 38 +++++------------------- source/libs/stream/src/streamDispatch.c | 5 ++-- source/libs/stream/src/streamExec.c | 18 ++++------- source/libs/stream/src/streamQueue.c | 21 +++++++++++-- source/libs/stream/src/streamRecover.c | 4 --- 10 files changed, 41 insertions(+), 59 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 30c60bcf0d..ac067823f8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -111,14 +111,12 @@ typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); typedef struct { int8_t type; int64_t ver; - int32_t* dataRef; SPackedData submit; } SStreamDataSubmit; typedef struct { int8_t type; int64_t ver; - SArray* dataRefs; // SArray SArray* submits; // SArray } SStreamMergedSubmit; @@ -672,7 +670,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask); int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask); int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); -int32_t streamTaskGetInputQItems(const SStreamTask* pTask); +int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); // common int32_t streamRestoreParam(SStreamTask* pTask); @@ -696,7 +694,7 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); // source level int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); -int32_t streamSourceScanHistoryData(SStreamTask* pTask); +int32_t streamScanHistoryData(SStreamTask* pTask); int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); // agg level diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e2888e8e9a..540eaed9ba 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1048,7 +1048,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { ASSERT(pTask->status.pauseAllowed == true); } - streamSourceScanHistoryData(pTask); + streamScanHistoryData(pTask); if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { double el = (taosGetTimestampMs() - pTask->tsInfo.step1Start) / 1000.0; tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 7f52750348..0925573248 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -537,7 +537,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock if (k == 0) { SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); void* colData = colDataGetData(pColData, j); - tqDebug("s-task:%s tq sink pipe2, row %d, col %d ts %" PRId64, id, j, k, *(int64_t*)colData); + tqDebug("s-task:%s sink row %d, col %d ts %" PRId64, id, j, k, *(int64_t*)colData); } if (IS_SET_NULL(pCol)) { diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 9bae6d6e33..14135b14dc 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -396,7 +396,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - int32_t numOfItems = streamTaskGetInputQItems(pTask); + int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputInfo.queue); int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX; taosThreadMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 75ac86b462..102787f019 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -177,7 +177,7 @@ static int32_t streamTaskAppendInputBlocks(SStreamTask* pTask, const SStreamDisp } int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { - SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); + SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SStreamDataBlock)); int8_t status = TASK_INPUT_STATUS__NORMAL; // enqueue diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index ea3e4f5985..00bf631d74 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -115,28 +115,16 @@ SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type) { return NULL; } - pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); - if (pDataSubmit->dataRef == NULL) { - taosFreeQitem(pDataSubmit); - return NULL; - } - pDataSubmit->ver = pData->ver; pDataSubmit->submit = *pData; - *pDataSubmit->dataRef = 1; // initialize the reference count to be 1 pDataSubmit->type = type; return pDataSubmit; } void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) { - int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); - ASSERT(ref >= 0 && pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT); - - if (ref == 0) { - taosMemoryFree(pDataSubmit->submit.msgStr); - taosMemoryFree(pDataSubmit->dataRef); - } + ASSERT(pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT); + taosMemoryFree(pDataSubmit->submit.msgStr); } SStreamMergedSubmit* streamMergedSubmitNew() { @@ -146,11 +134,8 @@ SStreamMergedSubmit* streamMergedSubmitNew() { } pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); - pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - - if (pMerged->dataRefs == NULL || pMerged->submits == NULL) { + if (pMerged->submits == NULL) { taosArrayDestroy(pMerged->submits); - taosArrayDestroy(pMerged->dataRefs); taosFreeQitem(pMerged); return NULL; } @@ -160,9 +145,10 @@ SStreamMergedSubmit* streamMergedSubmitNew() { } int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { - taosArrayPush(pMerged->dataRefs, &pSubmit->dataRef); taosArrayPush(pMerged->submits, &pSubmit->submit); - pMerged->ver = pSubmit->ver; + if (pSubmit->ver > pMerged->ver) { + pMerged->ver = pSubmit->ver; + } return 0; } @@ -222,18 +208,10 @@ void streamFreeQitem(SStreamQueueItem* data) { int32_t sz = taosArrayGetSize(pMerge->submits); for (int32_t i = 0; i < sz; i++) { - int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i); - int32_t ref = atomic_sub_fetch_32(pRef, 1); - ASSERT(ref >= 0); - - if (ref == 0) { - SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); - taosMemoryFree(pSubmit->msgStr); - taosMemoryFree(pRef); - } + SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); + taosMemoryFree(pSubmit->msgStr); } taosArrayDestroy(pMerge->submits); - taosArrayDestroy(pMerge->dataRefs); taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { SStreamRefDataBlock* pRefBlock = (SStreamRefDataBlock*)data; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index d04f55628c..89f067baeb 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -498,9 +498,10 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH)); const char* id = pTask->id.idStr; - int32_t numOfElems = taosQueueItemSize(pTask->outputInfo.queue->pQueue); + int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputInfo.queue); + int32_t size = SIZE_IN_MB(taosQueueMemorySize(pTask->outputInfo.queue->pQueue)); if (numOfElems > 0) { - qDebug("s-task:%s try to dispatch intermediate block to downstream, elem in outputQ:%d", id, numOfElems); + qDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d, size:%.2fMiB", id, numOfElems, size); } // to make sure only one dispatch is running diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 8d282696c1..9595d3bf02 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -164,11 +164,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i return code; } -int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize) { +int32_t streamScanHistoryData(SStreamTask* pTask) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); + int32_t code = TSDB_CODE_SUCCESS; void* exec = pTask->exec.pExecutor; bool finished = false; + int32_t outputBatchSize = 100; qSetStreamOpOpen(exec); @@ -217,8 +219,8 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize) { block.info.childId = pTask->info.selfChildId; taosArrayPush(pRes, &block); - if ((++numOfBlocks) >= batchSize) { - qDebug("s-task:%s scan exec numOfBlocks:%d, output limit:%d reached", pTask->id.idStr, numOfBlocks, batchSize); + if ((++numOfBlocks) >= outputBatchSize) { + qDebug("s-task:%s scan exec numOfBlocks:%d, output limit:%d reached", pTask->id.idStr, numOfBlocks, outputBatchSize); break; } } @@ -248,13 +250,6 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize) { return 0; } -int32_t streamTaskGetInputQItems(const SStreamTask* pTask) { - int32_t numOfItems1 = taosQueueItemSize(pTask->inputInfo.queue->pQueue); - int32_t numOfItems2 = taosQallItemSize(pTask->inputInfo.queue->qall); - - return numOfItems1 + numOfItems2; -} - // wait for the stream task to be idle static void waitForTaskIdle(SStreamTask* pTask, SStreamTask* pStreamTask) { const char* id = pTask->id.idStr; @@ -576,7 +571,6 @@ int32_t streamExecForAll(SStreamTask* pTask) { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. if (type == STREAM_INPUT__CHECKPOINT) { -// ASSERT(pTask->status.taskStatus == TASK_STATUS__CK); qDebug("s-task:%s checkpoint block received, set the status:%s", pTask->id.idStr, streamGetTaskStatusStr(pTask->status.taskStatus)); streamTaskBuildCheckpoint(pTask); @@ -608,8 +602,6 @@ int32_t streamTryExec(SStreamTask* pTask) { return -1; } -// streamTaskBuildCheckpoint(pTask); - atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); qDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 510c8dc9b1..a38095ea58 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -165,6 +165,13 @@ bool streamQueueIsFull(const STaosQueue* pQueue) { return (isFull || size >= STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE); } +int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue) { + int32_t numOfItems1 = taosQueueItemSize(pQueue->pQueue); + int32_t numOfItems2 = taosQallItemSize(pQueue->qall); + + return numOfItems1 + numOfItems2; +} + int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks) { int32_t retryTimes = 0; int32_t MAX_RETRY_TIMES = 5; @@ -269,11 +276,11 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) int8_t type = pItem->type; STaosQueue* pQueue = pTask->inputInfo.queue->pQueue; int32_t total = taosQueueItemSize(pQueue) + 1; - double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pQueue)) { + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); qTrace( "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); @@ -292,24 +299,30 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) return code; } + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); + // use the local variable to avoid the pItem be freed by other threads, since it has been put into queue already. qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, msgLen, ver, total, size + SIZE_IN_MB(msgLen)); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { if (streamQueueIsFull(pQueue)) { + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); + qTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); destroyStreamDataBlock((SStreamDataBlock*)pItem); return -1; } - qDebug("s-task:%s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, total, size); int32_t code = taosWriteQitem(pQueue, pItem); if (code != TSDB_CODE_SUCCESS) { destroyStreamDataBlock((SStreamDataBlock*)pItem); return code; } + + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); + qDebug("s-task:%s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, total, size); } else if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER || type == STREAM_INPUT__TRANS_STATE) { int32_t code = taosWriteQitem(pQueue, pItem); @@ -317,6 +330,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) taosFreeQitem(pItem); return code; } + + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); qDebug("s-task:%s level:%d %s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, pTask->info.taskLevel, streamGetBlockTypeStr(type), total, size); } else if (type == STREAM_INPUT__GET_RES) { @@ -326,6 +341,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) taosFreeQitem(pItem); return code; } + + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); qDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); } else { ASSERT(0); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 0693495b14..54d5957900 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -371,10 +371,6 @@ int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8 return 0; } -int32_t streamSourceScanHistoryData(SStreamTask* pTask) { - return streamScanExec(pTask, 100); -} - int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { SStreamDataBlock* pTranstate = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); if (pTranstate == NULL) { From 66676b977e5ad745fe114bf4f33030ebce1ab14a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 18:16:07 +0800 Subject: [PATCH 83/96] fix(stream): set correct number items in outputQ --- source/libs/stream/src/stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 102787f019..13ec509f63 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -226,7 +226,7 @@ int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock STaosQueue* pQueue = pTask->outputInfo.queue->pQueue; code = taosWriteQitem(pQueue, pBlock); - int32_t total = taosQueueItemSize(pQueue); + int32_t total = streamQueueGetNumOfItems(pTask->outputInfo.queue); double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); if (code != 0) { qError("s-task:%s failed to put res into outputQ, outputQ items:%d, size:%.2fMiB code:%s, result lost", From 51359ad99d5d29169f7e172fbd6fc32d3537ddc4 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 12 Sep 2023 18:24:50 +0800 Subject: [PATCH 84/96] free old pointer as realloc failed. --- source/os/src/osFile.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index e68e28af6f..1d680f214a 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -862,8 +862,13 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { } bufferSize += 512; - *ptrBuf = taosMemoryRealloc(*ptrBuf, bufferSize); - if (*ptrBuf == NULL) return -1; + void* newBuf = taosMemoryRealloc(*ptrBuf, bufferSize); + if (newBuf == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + + *ptrBuf = newBuf; } (*ptrBuf)[totalBytesRead] = '\0'; From 13c20facd01499309ed656f8e559267aff31590f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 18:37:30 +0800 Subject: [PATCH 85/96] fix(stream): fix syntax error. --- source/libs/stream/src/streamDispatch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 89f067baeb..e283b43ac6 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -499,7 +499,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { const char* id = pTask->id.idStr; int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputInfo.queue); - int32_t size = SIZE_IN_MB(taosQueueMemorySize(pTask->outputInfo.queue->pQueue)); + double size = SIZE_IN_MB(taosQueueMemorySize(pTask->outputInfo.queue->pQueue)); if (numOfElems > 0) { qDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d, size:%.2fMiB", id, numOfElems, size); } From d6f4c40e547d1dcbe48ddd09ca150b1c48b0ef51 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 12 Sep 2023 21:03:35 +0800 Subject: [PATCH 86/96] fix mem leak --- source/dnode/mnode/impl/src/mndUser.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ff5bb90c81..b7c2a24ba6 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1121,6 +1121,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate goto _OVER; } ipWhiteMgtUpdate(userObj.user, userObj.pIpWhiteList); + taosMemoryFree(userObj.pIpWhiteList); + mndTransDrop(pTrans); return 0; _OVER: From f6a097d96f9d0d6d357c12e1e5ea2c601a29752d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 12 Sep 2023 21:57:46 +0800 Subject: [PATCH 87/96] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 5 +- source/dnode/vnode/src/tq/tqStreamTask.c | 2 +- source/libs/stream/src/stream.c | 30 ---------- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 71 ++++++++++++++++-------- source/libs/stream/src/streamQueue.c | 64 ++++++++++++++++----- 6 files changed, 103 insertions(+), 71 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index ac067823f8..3eb624f932 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -434,8 +434,9 @@ int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo) int32_t tDecodeStreamTaskId(SDecoder* pDecoder, SStreamTaskId* pTaskId); int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem); +int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock); int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask); -bool streamQueueIsFull(const STaosQueue* pQueue); +bool streamQueueIsFull(const STaosQueue* pQueue, bool inputQ); typedef struct { SMsgHead head; @@ -645,12 +646,10 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t void streamTaskInputFail(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); -int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); bool streamTaskShouldStop(const SStreamStatus* pStatus); bool streamTaskShouldPause(const SStreamStatus* pStatus); bool streamTaskIsIdle(const SStreamTask* pTask); -int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize); void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 14135b14dc..1ac2ddb9cb 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -374,7 +374,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - if (streamQueueIsFull(pTask->inputInfo.queue->pQueue)) { + if (streamQueueIsFull(pTask->inputInfo.queue->pQueue, true)) { tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr); streamMetaReleaseTask(pStreamMeta, pTask); continue; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 13ec509f63..d1bf6a91c5 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -212,36 +212,6 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } -int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { - int32_t code = 0; - int32_t type = pTask->outputInfo.type; - if (type == TASK_OUTPUT__TABLE) { - pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, pBlock->blocks); - destroyStreamDataBlock(pBlock); - } else if (type == TASK_OUTPUT__SMA) { - pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); - destroyStreamDataBlock(pBlock); - } else { - ASSERT(type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH); - STaosQueue* pQueue = pTask->outputInfo.queue->pQueue; - code = taosWriteQitem(pQueue, pBlock); - - int32_t total = streamQueueGetNumOfItems(pTask->outputInfo.queue); - double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); - if (code != 0) { - qError("s-task:%s failed to put res into outputQ, outputQ items:%d, size:%.2fMiB code:%s, result lost", - pTask->id.idStr, total, size, tstrerror(code)); - } else { - qInfo("s-task:%s data put into outputQ, outputQ items:%d, size:%.2fMiB", pTask->id.idStr, total, size); - } - - streamDispatchStreamBlock(pTask); - return code; - } - - return 0; -} - int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { qDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64, pTask->id.idStr, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e283b43ac6..4d5234a68c 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -499,8 +499,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { const char* id = pTask->id.idStr; int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputInfo.queue); - double size = SIZE_IN_MB(taosQueueMemorySize(pTask->outputInfo.queue->pQueue)); if (numOfElems > 0) { + double size = SIZE_IN_MB(taosQueueMemorySize(pTask->outputInfo.queue->pQueue)); qDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d, size:%.2fMiB", id, numOfElems, size); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 9595d3bf02..f03a6a32d4 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -32,33 +32,58 @@ bool streamTaskShouldPause(const SStreamStatus* pStatus) { return (status == TASK_STATUS__PAUSE); } +static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBlock) { + int32_t code = 0; + int32_t type = pTask->outputInfo.type; + if (type == TASK_OUTPUT__TABLE) { + pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, pBlock->blocks); + destroyStreamDataBlock(pBlock); + } else if (type == TASK_OUTPUT__SMA) { + pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); + destroyStreamDataBlock(pBlock); + } else { + ASSERT(type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH); + code = streamTaskPutDataIntoOutputQ(pTask, pBlock); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + streamDispatchStreamBlock(pTask); + return code; + } + + return 0; +} + static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* pRes, int32_t size, int64_t* totalSize, int32_t* totalBlocks) { int32_t numOfBlocks = taosArrayGetSize(pRes); - if (numOfBlocks > 0) { - SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes); - if (pStreamBlocks == NULL) { - qError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno)); - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return -1; - } - - qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, - SIZE_IN_MB(size)); - - int32_t code = streamTaskOutputResultBlock(pTask, pStreamBlocks); - if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { // back pressure and record position - destroyStreamDataBlock(pStreamBlocks); - return -1; - } - - *totalSize += size; - *totalBlocks += numOfBlocks; - } else { + if (numOfBlocks == 0) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes); + if (pStreamBlocks == NULL) { + qError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno)); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return TSDB_CODE_OUT_OF_MEMORY; + } + + qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, + SIZE_IN_MB(size)); + + int32_t code = doOutputResultBlockImpl(pTask, pStreamBlocks); + if (code != TSDB_CODE_SUCCESS) { // back pressure and record position + //code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY + destroyStreamDataBlock(pStreamBlocks); + return code; + } + + *totalSize += size; + *totalBlocks += numOfBlocks; + + return code; } static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, @@ -236,7 +261,7 @@ int32_t streamScanHistoryData(SStreamTask* pTask) { qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - code = streamTaskOutputResultBlock(pTask, qRes); + code = doOutputResultBlockImpl(pTask, qRes); if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosFreeQitem(qRes); @@ -536,7 +561,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (type == STREAM_INPUT__DATA_BLOCK) { qDebug("s-task:%s sink task start to sink %d blocks", id, numOfBlocks); - streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput); + doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); continue; } } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index a38095ea58..29ca351a6b 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -15,10 +15,11 @@ #include "streamInt.h" -#define MAX_STREAM_EXEC_BATCH_NUM 32 -#define MIN_STREAM_EXEC_BATCH_NUM 4 -#define STREAM_TASK_INPUT_QUEUE_CAPACITY 20480 -#define STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE (30) +#define MAX_STREAM_EXEC_BATCH_NUM 32 +#define MIN_STREAM_EXEC_BATCH_NUM 4 +#define STREAM_TASK_QUEUE_CAPACITY 20480 +#define STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE (30) +#define STREAM_TASK_OUTPUT_QUEUE_CAPACITY_IN_SIZE (50) // todo refactor: // read data from input queue @@ -159,10 +160,15 @@ SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue) { } #endif -bool streamQueueIsFull(const STaosQueue* pQueue) { - bool isFull = taosQueueItemSize((STaosQueue*) pQueue) >= STREAM_TASK_INPUT_QUEUE_CAPACITY; - double size = SIZE_IN_MB(taosQueueMemorySize((STaosQueue*) pQueue)); - return (isFull || size >= STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE); +bool streamQueueIsFull(const STaosQueue* pQueue, bool inputQ) { + bool isFull = taosQueueItemSize((STaosQueue*)pQueue) >= STREAM_TASK_QUEUE_CAPACITY; + if (isFull) { + return true; + } + + int32_t threahold = (inputQ) ? STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE : STREAM_TASK_OUTPUT_QUEUE_CAPACITY_IN_SIZE; + double size = SIZE_IN_MB(taosQueueMemorySize((STaosQueue*)pQueue)); + return (size >= threahold); } int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue) { @@ -275,15 +281,15 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) { int8_t type = pItem->type; STaosQueue* pQueue = pTask->inputInfo.queue->pQueue; - int32_t total = taosQueueItemSize(pQueue) + 1; + int32_t total = streamQueueGetNumOfItems(pTask->inputInfo.queue) + 1; if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; - if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pQueue)) { + if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pQueue, true)) { double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); qTrace( "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", - pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); + pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); taosFreeQitem(pItem); return -1; @@ -306,11 +312,11 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) msgLen, ver, total, size + SIZE_IN_MB(msgLen)); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { - if (streamQueueIsFull(pQueue)) { + if (streamQueueIsFull(pQueue, true)) { double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); qTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", - pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); + pTask->id.idStr, STREAM_TASK_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); destroyStreamDataBlock((SStreamDataBlock*)pItem); return -1; } @@ -356,6 +362,38 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) return 0; } +// the result should be put into the outputQ in any cases, otherwise, the result may be lost +int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock) { + STaosQueue* pQueue = pTask->outputInfo.queue->pQueue; + + while (streamQueueIsFull(pQueue, false)) { + if (streamTaskShouldStop(&pTask->status)) { + qInfo("s-task:%s discard result block due to task stop", pTask->id.idStr); + return TSDB_CODE_STREAM_EXEC_CANCELLED; + } + + int32_t total = streamQueueGetNumOfItems(pTask->outputInfo.queue); + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); + // let's wait for there are enough space to hold this result pBlock + qDebug("s-task:%s outputQ is full, wait for 500ms and retry, outputQ items:%d, size:%.2fMiB", pTask->id.idStr, + total, size); + taosMsleep(500); + } + + int32_t code = taosWriteQitem(pQueue, pBlock); + + int32_t total = streamQueueGetNumOfItems(pTask->outputInfo.queue); + double size = SIZE_IN_MB(taosQueueMemorySize(pQueue)); + if (code != 0) { + qError("s-task:%s failed to put res into outputQ, outputQ items:%d, size:%.2fMiB code:%s, result lost", + pTask->id.idStr, total + 1, size, tstrerror(code)); + } else { + qInfo("s-task:%s data put into outputQ, outputQ items:%d, size:%.2fMiB", pTask->id.idStr, total, size); + } + + return TSDB_CODE_SUCCESS; +} + int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t cap, int32_t rate) { if (cap < 100 || rate < 50 || pBucket == NULL) { qError("failed to init sink task bucket, cap:%d, rate:%d", cap, rate); From 8f5dd5a9b9502486080684fbfd738a21539eb405 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 09:56:26 +0800 Subject: [PATCH 88/96] fix case error --- source/libs/transport/src/transSvr.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index f4d0a0371c..35209f4977 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -43,6 +43,7 @@ typedef struct SSvrConn { ConnStatus status; + uint32_t serverIp; uint32_t clientIp; uint16_t port; @@ -324,6 +325,7 @@ bool uvWhiteListFilte(SWhiteList* pWhite, char* user, uint32_t ip, int64_t ver) } bool uvWhiteListCheckConn(SWhiteList* pWhite, SSvrConn* pConn) { if (pConn->inType == TDMT_MND_STATUS || pConn->inType == TDMT_MND_RETRIEVE_IP_WHITE || + pConn->serverIp == pConn->clientIp || pWhite->ver == pConn->whiteListVer /*|| strncmp(pConn->user, "_dnd", strlen("_dnd")) == 0*/) return true; @@ -358,7 +360,7 @@ static bool uvHandleReq(SSvrConn* pConn) { int8_t forbiddenIp = 0; if (pThrd->enableIpWhiteList) { - forbiddenIp = uvWhiteListCheckConn(pThrd->pWhiteList, pConn) == false ? 1 : 0; + forbiddenIp = !uvWhiteListCheckConn(pThrd->pWhiteList, pConn) ? 1 : 0; if (forbiddenIp == 0) { uvWhiteListSetConnVer(pThrd->pWhiteList, pConn); } @@ -959,7 +961,11 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { transSockInfo2Str(&sockname, pConn->src); struct sockaddr_in addr = *(struct sockaddr_in*)&peername; + struct sockaddr_in saddr = *(struct sockaddr_in*)&sockname; + pConn->clientIp = addr.sin_addr.s_addr; + pConn->serverIp = saddr.sin_addr.s_addr; + pConn->port = ntohs(addr.sin_port); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb); From 0152646eacf062a81b6f017e01141524ffd1fb09 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 10:11:29 +0800 Subject: [PATCH 89/96] fix case error --- source/libs/transport/src/transSvr.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 35209f4977..04f5077988 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -965,7 +965,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->clientIp = addr.sin_addr.s_addr; pConn->serverIp = saddr.sin_addr.s_addr; - pConn->port = ntohs(addr.sin_port); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb); From 808630f406b955b34dacaa24a42a3410c89aed73 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 14:15:15 +0800 Subject: [PATCH 90/96] update test case --- tests/system-test/0-others/information_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index baf0682fbb..245fc9ba21 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -217,7 +217,7 @@ class TDTestCase: tdSql.checkEqual(20470,len(tdSql.queryResult)) tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") - tdSql.checkEqual(193, len(tdSql.queryResult)) + tdSql.checkEqual(194, len(tdSql.queryResult)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(54, len(tdSql.queryResult)) From c0a8f811761b47708ecff9a65aa46f183503babf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 14:27:58 +0800 Subject: [PATCH 91/96] update test case --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/dnode/mnode/impl/src/mndUser.c | 25 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 93bc80b705..4bdf877933 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -79,7 +79,7 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) { char buf[36] = {0}; rpcUtilSIpRangeToStr(&range, buf); - dError("User %s host:%s not in ip white list", user, buf); + dError("User:%s host:%s not in ip white list", user, buf); return true; } else { return false; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index a14a9c5b7d..a8847e71ad 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1203,29 +1203,30 @@ _OVER: return code; } -//TODO: for community version use the commented version -int32_t mndSetUserWhiteListRsp(SMnode* pMnode, SUserObj* pUser, SGetUserWhiteListRsp* pWhiteListRsp) { +// TODO: for community version use the commented version +int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteListRsp *pWhiteListRsp) { memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN); -// pWhiteListRsp->numWhiteLists = 1; -// pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); -// if (pWhiteListRsp->pWhiteLists == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// memset(pUser->pIpWhiteList->pIpRange, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); + // pWhiteListRsp->numWhiteLists = 1; + // pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); + // if (pWhiteListRsp->pWhiteLists == NULL) { + // return TSDB_CODE_OUT_OF_MEMORY; + // } + // memset(pUser->pIpWhiteList->pIpRange, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); pWhiteListRsp->numWhiteLists = pUser->pIpWhiteList->num; pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); if (pWhiteListRsp->pWhiteLists == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(pUser->pIpWhiteList->pIpRange, pUser->pIpWhiteList->pIpRange, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); + memcpy(pUser->pIpWhiteList->pIpRange, pUser->pIpWhiteList->pIpRange, + pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); return 0; } int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SUserObj *pUser = NULL; + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SUserObj *pUser = NULL; SGetUserWhiteListReq wlReq = {0}; SGetUserWhiteListRsp wlRsp = {0}; From 6cfab63ee05123205c65550ea3a55fc216f6dfb5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 14:50:26 +0800 Subject: [PATCH 92/96] update test case --- source/libs/transport/src/transSvr.c | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 04f5077988..923e7d475f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -80,7 +80,7 @@ typedef struct { typedef struct { SHashObj* pList; int64_t ver; -} SWhiteList; +} SIpWhiteListTab; typedef struct SWorkThrd { TdThread thread; uv_connect_t connect_req; @@ -95,7 +95,7 @@ typedef struct SWorkThrd { void* pTransInst; bool quit; - SWhiteList* pWhiteList; + SIpWhiteListTab* pWhiteList; int64_t whiteListVer; int8_t enableIpWhiteList; } SWorkThrd; @@ -120,13 +120,13 @@ typedef struct SServerObj { bool inited; } SServerObj; -SWhiteList* uvWhiteListCreate(); -void uvWhiteListDestroy(SWhiteList* pWhite); -void uvWhiteListAdd(SWhiteList* pWhite, char* user, SIpWhiteList* pList, int64_t ver); -void uvWhiteListUpdate(SWhiteList* pWhite, SHashObj* pTable); -bool uvWhiteListCheckConn(SWhiteList* pWhite, SSvrConn* pConn); -bool uvWhiteListFilte(SWhiteList* pWhite, char* user, uint32_t ip, int64_t ver); -void uvWhiteListSetConnVer(SWhiteList* pWhite, SSvrConn* pConn); +SIpWhiteListTab* uvWhiteListCreate(); +void uvWhiteListDestroy(SIpWhiteListTab* pWhite); +void uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* pList, int64_t ver); +void uvWhiteListUpdate(SIpWhiteListTab* pWhite, SHashObj* pTable); +bool uvWhiteListCheckConn(SIpWhiteListTab* pWhite, SSvrConn* pConn); +bool uvWhiteListFilte(SIpWhiteListTab* pWhite, char* user, uint32_t ip, int64_t ver); +void uvWhiteListSetConnVer(SIpWhiteListTab* pWhite, SSvrConn* pConn); static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -219,14 +219,14 @@ static bool uvCheckIp(SIpV4Range* pRange, int32_t ip) { } return subnetCheckIp(&subnet, ip); } -SWhiteList* uvWhiteListCreate() { - SWhiteList* pWhiteList = taosMemoryCalloc(1, sizeof(SWhiteList)); +SIpWhiteListTab* uvWhiteListCreate() { + SIpWhiteListTab* pWhiteList = taosMemoryCalloc(1, sizeof(SIpWhiteListTab)); pWhiteList->pList = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 0, HASH_NO_LOCK); pWhiteList->ver = -1; return pWhiteList; } -void uvWhiteListDestroy(SWhiteList* pWhite) { +void uvWhiteListDestroy(SIpWhiteListTab* pWhite) { SHashObj* pWhiteList = pWhite->pList; void* pIter = taosHashIterate(pWhiteList, NULL); while (pIter) { @@ -250,7 +250,7 @@ void uvWhiteListToStr(SWhiteUserList* plist, char* user, char** ppBuf) { *ppBuf = pBuf; } -void uvWhiteListDebug(SWhiteList* pWrite) { +void uvWhiteListDebug(SIpWhiteListTab* pWrite) { SHashObj* pWhiteList = pWrite->pList; void* pIter = taosHashIterate(pWhiteList, NULL); while (pIter) { @@ -268,7 +268,7 @@ void uvWhiteListDebug(SWhiteList* pWrite) { pIter = taosHashIterate(pWhiteList, pIter); } } -void uvWhiteListAdd(SWhiteList* pWhite, char* user, SIpWhiteList* plist, int64_t ver) { +void uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* plist, int64_t ver) { SHashObj* pWhiteList = pWhite->pList; SWhiteUserList** ppUserList = taosHashGet(pWhiteList, user, strlen(user)); @@ -289,7 +289,7 @@ void uvWhiteListAdd(SWhiteList* pWhite, char* user, SIpWhiteList* plist, int64_t uvWhiteListDebug(pWhite); } -void uvWhiteListUpdate(SWhiteList* pWhite, SHashObj* pTable) { +void uvWhiteListUpdate(SIpWhiteListTab* pWhite, SHashObj* pTable) { pWhite->ver++; // impl later } @@ -299,7 +299,7 @@ static bool uvWhiteListIsDefaultAddr(uint32_t ip) { static SIpV4Range range = {.ip = 16777343, .mask = 32}; return range.ip == ip; } -bool uvWhiteListFilte(SWhiteList* pWhite, char* user, uint32_t ip, int64_t ver) { +bool uvWhiteListFilte(SIpWhiteListTab* pWhite, char* user, uint32_t ip, int64_t ver) { // impl check SHashObj* pWhiteList = pWhite->pList; bool valid = false; @@ -310,10 +310,10 @@ bool uvWhiteListFilte(SWhiteList* pWhite, char* user, uint32_t ip, int64_t ver) if (ppList == NULL || *ppList == NULL) { return false; } - SWhiteUserList* pList = *ppList; - if (pList->ver == ver) return true; + SWhiteUserList* pUserList = *ppList; + if (pUserList->ver == ver) return true; - SIpWhiteList* pIpWhiteList = pList->pList; + SIpWhiteList* pIpWhiteList = pUserList->pList; for (int i = 0; i < pIpWhiteList->num; i++) { SIpV4Range* range = &pIpWhiteList->pIpRange[i]; if (uvCheckIp(range, ip)) { @@ -323,7 +323,7 @@ bool uvWhiteListFilte(SWhiteList* pWhite, char* user, uint32_t ip, int64_t ver) } return valid; } -bool uvWhiteListCheckConn(SWhiteList* pWhite, SSvrConn* pConn) { +bool uvWhiteListCheckConn(SIpWhiteListTab* pWhite, SSvrConn* pConn) { if (pConn->inType == TDMT_MND_STATUS || pConn->inType == TDMT_MND_RETRIEVE_IP_WHITE || pConn->serverIp == pConn->clientIp || pWhite->ver == pConn->whiteListVer /*|| strncmp(pConn->user, "_dnd", strlen("_dnd")) == 0*/) @@ -331,7 +331,7 @@ bool uvWhiteListCheckConn(SWhiteList* pWhite, SSvrConn* pConn) { return uvWhiteListFilte(pWhite, pConn->user, pConn->clientIp, pConn->whiteListVer); } -void uvWhiteListSetConnVer(SWhiteList* pWhite, SSvrConn* pConn) { +void uvWhiteListSetConnVer(SIpWhiteListTab* pWhite, SSvrConn* pConn) { // if conn already check by current whiteLis pConn->whiteListVer = pWhite->ver; } From 4faa791ffdbdd7da6005679c15f677dc285451ae Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 15:38:44 +0800 Subject: [PATCH 93/96] refactor code --- source/libs/transport/src/transComm.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index b7dd438fc8..b9683ec487 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -681,13 +681,13 @@ void subnetIp2int(const char* const ip_addr, uint8_t* dst) { uint32_t subnetIpRang2Int(SIpV4Range* pRange) { SIpV4Range range = {.ip = pRange->ip, .mask = 32}; uint8_t el[4] = {0}; - char buf[32] = {0}; - transUtilSIpRangeToStr(&range, buf); + el[0] = range.ip & 0xFF; + el[1] = (range.ip >> 8) & 0xFF; + el[2] = (range.ip >> 16) & 0xFF; + el[3] = (range.ip >> 24) & 0xFF; - subnetIp2int(buf, el); - - return (el[0] << 24) | (el[1] << 16) | (el[2] << 8) | (el[0]); + return (el[0] << 24) | (el[1] << 16) | (el[2] << 8) | (el[3]); } int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { if (pRange->mask == 32) { @@ -695,7 +695,6 @@ int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { pUtils->address = pRange->ip; return 0; } - // pUtils->address = ntohl(pRange->ip); pUtils->address = subnetIpRang2Int(pRange); for (int i = 0; i < pRange->mask; i++) { @@ -721,7 +720,6 @@ int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip) { } else { SIpV4Range range = {.ip = ip, .mask = 32}; - // uint32_t ip = ntohl(ip); uint32_t ip = subnetIpRang2Int(&range); return ip >= pUtils->network && ip <= pUtils->broadcast; } From 772e9c6627efdb3876a9be5f529bf7a61c229055 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 16:27:58 +0800 Subject: [PATCH 94/96] update test case --- source/libs/transport/src/transComm.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index b9683ec487..c0fe41b8ca 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -679,15 +679,8 @@ void subnetIp2int(const char* const ip_addr, uint8_t* dst) { } uint32_t subnetIpRang2Int(SIpV4Range* pRange) { - SIpV4Range range = {.ip = pRange->ip, .mask = 32}; - uint8_t el[4] = {0}; - - el[0] = range.ip & 0xFF; - el[1] = (range.ip >> 8) & 0xFF; - el[2] = (range.ip >> 16) & 0xFF; - el[3] = (range.ip >> 24) & 0xFF; - - return (el[0] << 24) | (el[1] << 16) | (el[2] << 8) | (el[3]); + uint32_t ip = pRange->ip; + return ((ip & 0xFF) << 24) | ((ip & 0xFF00) << 8) | ((ip & 0xFF0000) >> 8) | ((ip >> 24) & 0xFF); } int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { if (pRange->mask == 32) { @@ -720,8 +713,8 @@ int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip) { } else { SIpV4Range range = {.ip = ip, .mask = 32}; - uint32_t ip = subnetIpRang2Int(&range); - return ip >= pUtils->network && ip <= pUtils->broadcast; + uint32_t t = subnetIpRang2Int(&range); + return t >= pUtils->network && t <= pUtils->broadcast; } } From f29b3b27ace4c8ff59991722e980499dcb189e5e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 16:29:19 +0800 Subject: [PATCH 95/96] update test case --- source/libs/transport/src/transComm.c | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index c0fe41b8ca..3dc59a93ee 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -655,28 +655,28 @@ void transDestoryExHandle(void* handle) { taosMemoryFree(handle); } -void subnetIp2int(const char* const ip_addr, uint8_t* dst) { - char ip_addr_cpy[20]; - char ip[5]; +// void subnetIp2int(const char* const ip_addr, uint8_t* dst) { +// char ip_addr_cpy[20]; +// char ip[5]; - tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy)); +// tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy)); - char *s_start, *s_end; - s_start = ip_addr_cpy; - s_end = ip_addr_cpy; +// char *s_start, *s_end; +// s_start = ip_addr_cpy; +// s_end = ip_addr_cpy; - int32_t k = 0; +// int32_t k = 0; - for (k = 0; *s_start != '\0'; s_start = s_end) { - for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { - } - if (*s_end == '.') { - *s_end = '\0'; - s_end++; - } - dst[k++] = (char)atoi(s_start); - } -} +// for (k = 0; *s_start != '\0'; s_start = s_end) { +// for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { +// } +// if (*s_end == '.') { +// *s_end = '\0'; +// s_end++; +// } +// dst[k++] = (char)atoi(s_start); +// } +// } uint32_t subnetIpRang2Int(SIpV4Range* pRange) { uint32_t ip = pRange->ip; From e7815229ca472810c8460855795c8cccf176954c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Sep 2023 19:54:45 +0800 Subject: [PATCH 96/96] update test case --- source/dnode/mnode/impl/inc/mndUser.h | 2 ++ source/dnode/mnode/impl/src/mndSync.c | 4 ++- source/dnode/mnode/impl/src/mndUser.c | 37 +++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index f4dfb58cb8..fab8ee4707 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -49,6 +49,8 @@ int64_t mndGetIpWhiteVer(SMnode *pMnode); void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock); +int32_t mndRefreshUserIpWhiteList(SMnode *pMnode); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index a601d3c121..5759737a6a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,6 +17,7 @@ #include "mndSync.h" #include "mndCluster.h" #include "mndTrans.h" +#include "mndUser.h" static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { if (pMsg == NULL || pMsg->pCont == NULL) { @@ -167,7 +168,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { SSdbRaw *pRaw = pMsg->pCont; STrans *pTrans = NULL; int32_t code = -1; - int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); + int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); if (transId <= 0) { mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); @@ -304,6 +305,7 @@ void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) { } else { mInfo("vgId:1, sync restore finished"); } + mndRefreshUserIpWhiteList(pMnode); ASSERT(commitIdx == mndSyncAppliedIndex(pFsm)); } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index a8847e71ad..9ca4b24681 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -57,6 +57,7 @@ static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter); SHashObj *mndFetchAllIpWhite(SMnode *pMnode); static int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq); +void ipWhiteMgtUpdateAll(SMnode *pMnode); typedef struct { SHashObj *pIpWhiteTab; int64_t ver; @@ -188,6 +189,7 @@ int64_t mndGetIpWhiteVer(SMnode *pMnode) { int64_t ver = 0; taosThreadRwlockWrlock(&ipWhiteMgt.rw); if (ipWhiteMgt.ver == 0) { + // user and dnode r ipWhiteMgtUpdateAll(pMnode); ipWhiteMgt.ver = taosGetTimestampMs(); } @@ -204,7 +206,7 @@ int64_t mndGetIpWhiteVer(SMnode *pMnode) { bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type) { bool update = false; SIpV4Range range = {.ip = taosGetIpv4FromFqdn(fqdn), .mask = 32}; - + mDebug("ip-white-list may update for user: %s, fqdn: %s", user, fqdn); SIpWhiteList **ppList = taosHashGet(pIpWhiteTab, user, strlen(user)); SIpWhiteList *pList = NULL; if (ppList != NULL && *ppList != NULL) { @@ -260,16 +262,29 @@ bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t } } } + if (update) { + mDebug("ip-white-list update for user: %s, fqdn: %s", user, fqdn); + } return update; } + +int32_t mndRefreshUserIpWhiteList(SMnode *pMnode) { + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + + ipWhiteMgtUpdateAll(pMnode); + ipWhiteMgt.ver = taosGetTimestampMs(); + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + + return 0; +} void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) { if (lock) { taosThreadRwlockWrlock(&ipWhiteMgt.rw); if (ipWhiteMgt.ver == 0) { ipWhiteMgtUpdateAll(pMnode); ipWhiteMgt.ver = taosGetTimestampMs(); - mInfo("ip-white-mnode ver, %" PRId64 "", ipWhiteMgt.ver); + mInfo("ip-white-list, user: %" PRId64 "", ipWhiteMgt.ver); } } @@ -422,14 +437,22 @@ static void ipRangeToStr(SIpV4Range *range, char *buf) { } return; } -static void ipRangeListToStr(SIpV4Range *range, int32_t num, char *buf) { +static bool isDefualtRange(SIpV4Range *pRange) { + static SIpV4Range val = {.ip = 16777343, .mask = 32}; + return pRange->ip == val.ip && pRange->mask == val.mask; +} +static int32_t ipRangeListToStr(SIpV4Range *range, int32_t num, char *buf) { int32_t len = 0; for (int i = 0; i < num; i++) { - char tbuf[36] = {0}; + char tbuf[36] = {0}; + SIpV4Range *pRange = &range[i]; + if (isDefualtRange(pRange)) continue; + ipRangeToStr(&range[i], tbuf); len += sprintf(buf + len, "%s,", tbuf); } if (len > 0) buf[len - 1] = 0; + return len; } static bool isIpRangeEqual(SIpV4Range *a, SIpV4Range *b) { @@ -459,7 +482,11 @@ int32_t convertIpWhiteListToStr(SIpWhiteList *pList, char **buf) { return 0; } *buf = taosMemoryCalloc(1, pList->num * 36); - ipRangeListToStr(pList->pIpRange, pList->num, *buf); + int32_t len = ipRangeListToStr(pList->pIpRange, pList->num, *buf); + if (len == 0) { + taosMemoryFree(*buf); + return 0; + } return strlen(*buf); } int32_t tSerializeIpWhiteList(void *buf, int32_t len, SIpWhiteList *pList) {