From 533a21693b1544e47c2d430345d21a558151904f Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 23 Nov 2022 15:50:06 +0800 Subject: [PATCH 01/20] enh: set max restart time of taosd to 3; allow disable alert in pod --- packaging/docker/DockerfileCloud | 2 +- packaging/docker/run.sh | 41 ++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/packaging/docker/DockerfileCloud b/packaging/docker/DockerfileCloud index 21e387bab3..e46a674cd2 100644 --- a/packaging/docker/DockerfileCloud +++ b/packaging/docker/DockerfileCloud @@ -26,4 +26,4 @@ COPY ./bin/* /usr/bin/ ENTRYPOINT ["/tini", "--", "/usr/bin/entrypoint.sh"] CMD ["bash", "-c", "/usr/bin/run.sh"] -VOLUME [ "/var/lib/taos", "/var/log/taos", "/corefile" ] +VOLUME [ "/var/lib/taos", "/var/log/taos" ] diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh index 3654beadcb..3d99e0a56d 100755 --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -6,6 +6,9 @@ TAOSD_STARTUP_TIMEOUT_SECOND=${TAOSD_STARTUP_TIMEOUT_SECOND:-160} TAOS_TIMEOUT_SECOND=${TAOS_TIMEOUT_SECOND:-5} BACKUP_CORE_FOLDER=/var/log/corefile ALERT_URL=app/system/alert/add +ALERT_DISABLE_FILE=/var/log/disable_alert +START_TAOSD_MAX_NUMBER=3 +start_taosd_count=0 echo "ADMIN_URL: ${ADMIN_URL}" echo "TAOS_TIMEOUT_SECOND: ${TAOS_TIMEOUT_SECOND}" @@ -37,12 +40,16 @@ function post_error_msg() { echo "service_state: ${service_state}" echo "`date` service_msg: ${service_msg}" echo "${taos_version}" - curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ - -d"{\"appName\":\"${app_name}\",\ - \"alertLevel\":\"${service_state}\",\ - \"taosVersion\":\"${taos_version}\",\ - \"alertMsg\":\"${service_msg}\"}" \ - ${ADMIN_URL}/${ALERT_URL} + if [ -f ${ALERT_DISABLE_FILE} ]; then + echo "alert disabled" + else + curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ + -d"{\"appName\":\"${app_name}\",\ + \"alertLevel\":\"${service_state}\",\ + \"taosVersion\":\"${taos_version}\",\ + \"alertMsg\":\"${service_msg}\"}" \ + ${ADMIN_URL}/${ALERT_URL} + fi fi } function check_taosd_exit_type() { @@ -78,12 +85,16 @@ function post_disk_error_msg() { echo "disk_state: ${disk_state}" echo "`date` disk_msg: ${disk_msg}" echo "${taos_version}" - curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ - -d"{\"appName\":\"${app_name}\",\ - \"alertLevel\":\"${disk_state}\",\ - \"taosVersion\":\"${taos_version}\",\ - \"alertMsg\":\"${disk_msg}\"}" \ - ${ADMIN_URL}/${ALERT_URL} + if [ -f ${ALERT_DISABLE_FILE} ]; then + echo "alert disabled" + else + curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ + -d"{\"appName\":\"${app_name}\",\ + \"alertLevel\":\"${disk_state}\",\ + \"taosVersion\":\"${taos_version}\",\ + \"alertMsg\":\"${disk_msg}\"}" \ + ${ADMIN_URL}/${ALERT_URL} + fi fi } function check_disk() { @@ -154,6 +165,12 @@ do # echo $status if [ "$status"x = "0"x ] then + echo "start taosd count: ${start_taosd_count}" + if [ ${start_taosd_count} -gt ${START_TAOSD_MAX_NUMBER} ]; then + echo "exceed restart max count: ${START_TAOSD_MAX_NUMBER}" + break + fi + start_taosd_count=$(( start_taosd_count + 1 )) # taosd_start_time=`date +%s` run_taosd & fi From 908de1b301c2b411acd572e362977992bb418c93 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 23 Nov 2022 15:59:34 +0800 Subject: [PATCH 02/20] fix: sysdb cache removed cause of drop db operation --- source/client/src/clientMsgHandler.c | 3 +++ source/libs/catalog/src/catalog.c | 3 +++ source/libs/catalog/src/ctgCache.c | 7 ++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 8437c93bae..1931704909 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -20,6 +20,7 @@ #include "query.h" #include "tdef.h" #include "tname.h" +#include "systable.h" static void setErrno(SRequestObj* pRequest, int32_t code) { pRequest->code = code; @@ -326,6 +327,8 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (TSDB_CODE_SUCCESS == code) { catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); + catalogRemoveDB(pCatalog, TSDB_INFORMATION_SCHEMA_DB, 0); + catalogRemoveDB(pCatalog, TSDB_PERFORMANCE_SCHEMA_DB, 0); } } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 2dcd681205..3a398d1551 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1076,6 +1076,9 @@ int32_t catalogRefreshTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const S SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)pTableName; ctx.flag = CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable); + if (IS_SYS_DBNAME(ctx.pName->dbname)) { + CTG_FLAG_SET_SYS_DB(ctx.flag); + } CTG_API_LEAVE(ctgRefreshTbMeta(pCtg, pConn, &ctx, NULL, true)); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index fa38eeba0c..19b7ee32ae 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -663,6 +663,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_DROP_DB_CACHE; + op->syncOp = true; SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg)); if (NULL == msg) { @@ -1612,11 +1613,11 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { dbCache = NULL; - if (!IS_SYS_DBNAME(dbFName)) { + //if (!IS_SYS_DBNAME(dbFName)) { tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - } + //} _return: @@ -1641,7 +1642,7 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { goto _return; } - if (dbCache->dbId != msg->dbId) { + if (msg->dbId && dbCache->dbId != msg->dbId) { ctgInfo("dbId already updated, dbFName:%s, dbId:0x%" PRIx64 ", targetId:0x%" PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); goto _return; From a222e9ad7b7835c4f9ac7a7b1c125edf9fd399cc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 23 Nov 2022 16:09:03 +0800 Subject: [PATCH 03/20] fix: update sysdb vgroup info after db dropped --- source/client/src/clientMsgHandler.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 1931704909..fe35d9679d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -327,8 +327,14 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (TSDB_CODE_SUCCESS == code) { catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); - catalogRemoveDB(pCatalog, TSDB_INFORMATION_SCHEMA_DB, 0); - catalogRemoveDB(pCatalog, TSDB_PERFORMANCE_SCHEMA_DB, 0); + STscObj* pTscObj = pRequest->pTscObj; + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + catalogRefreshDBVgInfo(pCatalog, &conn, TSDB_INFORMATION_SCHEMA_DB); + catalogRefreshDBVgInfo(pCatalog, &conn, TSDB_PERFORMANCE_SCHEMA_DB); } } From bc40b7d360a4ff25dc9537002aa49ba676312406 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Wed, 23 Nov 2022 22:21:21 +0800 Subject: [PATCH 04/20] update --- tests/pytest/crash_gen/crash_gen_main.py | 2 +- .../4dnode1mnode_basic_createDb_replica1.py | 3 +- ...4dnode1mnode_basic_replica1_insertdatas.py | 6 +-- ...mnode_basic_replica1_insertdatas_querys.py | 6 +-- ...lica3_insertdatas_force_stop_all_dnodes.py | 40 +++++++++---------- ...mnode_basic_replica3_insertdatas_querys.py | 6 +-- ...nsertdatas_querys_loop_restart_follower.py | 8 ++-- ..._insertdatas_querys_loop_restart_leader.py | 24 ++++++----- ...replica3_insertdatas_stop_follower_sync.py | 12 +++--- ...plica3_insertdatas_stop_follower_unsync.py | 16 ++++---- ...rtdatas_stop_follower_unsync_force_stop.py | 12 +++--- ...ca3_insertdatas_stop_leader_forece_stop.py | 26 +++++++----- .../4dnode1mnode_basic_replica3_vgroups.py | 14 +++---- 13 files changed, 94 insertions(+), 81 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index f8c5f970c5..fb0bc5ebea 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2025,7 +2025,7 @@ class TdSuperTable: conf.set("enable.auto.commit", "true") def tmq_commit_cb_print(tmq, resp, offset, param=None): print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - conf.set_auto_commit_cb(tmq_commit_cb_print, None) + #conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py index 9bdc0a2cf4..19239513d6 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py @@ -73,9 +73,10 @@ class TDTestCase: for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: + tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py index d33a1b0d27..f27c343329 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py @@ -74,14 +74,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -124,7 +124,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_db_replica_1_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): drop_db_sql = "drop database if exists {}".format(dbname) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py index 75e01977fd..671010db9a 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py @@ -75,14 +75,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -125,7 +125,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_db_replica_3_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): newTdSql=tdCom.newTdSql() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py index 1a2c31a311..73153c5825 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py @@ -36,7 +36,7 @@ class TDTestCase: self.tb_nums = 10 self.row_nums = 100 self.stop_dnode_id = None - self.loop_restart_times = 5 + self.loop_restart_times = 3 self.current_thread = None self.max_restart_time = 10 self.try_check_times = 10 @@ -84,14 +84,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -150,7 +150,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups; '".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) @@ -171,7 +171,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups;'".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) @@ -271,16 +271,16 @@ class TDTestCase: caller = inspect.getframeinfo(inspect.stack()[2][0]) if row < 0: args = (caller.filename, caller.lineno, sql, row) - tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + tdLog.notice("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) if col < 0: args = (caller.filename, caller.lineno, sql, row) - tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + tdLog.notice("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) if row > tdSql.queryRows: args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) - tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + tdLog.notice("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) if col > tdSql.queryCols: args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) - tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + tdLog.notice("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) def mycheckData(self, sql ,row, col, data): check_status = True @@ -361,31 +361,31 @@ class TDTestCase: # print(ps_kill_taosd) os.system(ps_kill_taosd) else : - tdLog.exit(" ==== port of dnode {} not found ====".format(dnode_id)) + tdLog.notice(" ==== port of dnode {} not found ====".format(dnode_id)) def stop_All(self): tdDnodes = cluster.dnodes - # newTdSql=tdCom.newTdSql() + newTdSql=tdCom.newTdSql() # ==== stop all dnode ===== for k ,v in self.dnode_list.items(): dnode_id = v[0] - # tdDnodes[dnode_id-1].stoptaosd() - self.force_stop_dnode(dnode_id) + + tdDnodes[dnode_id-1].stoptaosd() + # self.force_stop_dnode(dnode_id) # self.wait_stop_dnode_OK(newTdSql) def start_All(self): tdDnodes = cluster.dnodes - # newTdSql=tdCom.newTdSql() + for k ,v in self.dnode_list.items(): dnode_id = v[0] start = time.time() tdDnodes[dnode_id-1].starttaosd() - # self.wait_start_dnode_OK(newTdSql) end = time.time() time_cost = int(end -start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) @@ -401,7 +401,10 @@ class TDTestCase: # begin to stop All taosd self.stop_All() # begin to start All taosd - self.start_All() + self.start_All() + + time.sleep(5) + tdLog.debug(" ==== cluster has restart , this is {}_th restart cluster ==== ".format(i)) @@ -418,9 +421,6 @@ class TDTestCase: self.check_setup_cluster_status() self.stop_All_dnodes_check_datas() - - - def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py index a9fb9555e8..d054e25e46 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py @@ -75,14 +75,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -125,7 +125,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_db_replica_3_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): newTdSql=tdCom.newTdSql() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py index 6102a82b04..2993ce3a4a 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py @@ -77,14 +77,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -128,7 +128,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_db_replica_3_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): newTdSql=tdCom.newTdSql() @@ -284,7 +284,7 @@ class TDTestCase: end = time.time() time_cost = int(end -start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py index d87ec3d35e..d9a84db6a2 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py @@ -77,14 +77,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -128,7 +128,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_db_replica_3_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): newTdSql=tdCom.newTdSql() @@ -262,7 +262,7 @@ class TDTestCase: if not vote_act: print("=======before_revote_leader_infos ======\n" , before_leader_infos) print("=======after_revote_leader_infos ======\n" , after_leader_infos) - tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + tdLog.notice(" ===maybe revote not occured , there is no dnode offline ====") else: for vgroup_info in vote_act: for ind , role in enumerate(vgroup_info): @@ -282,10 +282,15 @@ class TDTestCase: def check_insert_status(self, newTdSql , dbname, tb_nums , row_nums): newTdSql.execute("use {}".format(dbname)) - newTdSql.query("select count(*) from {}.{}".format(dbname,'stb1')) - # tdSql.checkData(0 , 0 , tb_nums*row_nums) - newTdSql.query("select distinct tbname from {}.{}".format(dbname,'stb1')) - # tdSql.checkRows(tb_nums) + os.system(''' taos -s "select count(*) from {}.{};" '''.format(dbname,'stb1')) + # try: + # newTdSql.query("select count(*) from {}.{}".format(dbname,'stb1')) + # # tdSql.checkData(0 , 0 , tb_nums*row_nums) + # newTdSql.query("select distinct tbname from {}.{}".format(dbname,'stb1')) + # # tdSql.checkRows(tb_nums) + # except taos.error.ProgrammingError as err: + # tdLog.info(err.msg) + # pass def loop_query_constantly(self, times , db_name, tb_nums ,row_nums): @@ -335,9 +340,10 @@ class TDTestCase: tdDnodes[self.stop_dnode_id-1].starttaosd() self.wait_start_dnode_OK(newTdSql) end = time.time() + time.sleep(3) time_cost = int(end -start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py index 00b808b8b4..74181bc563 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py @@ -83,14 +83,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -133,7 +133,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_database(self, dbname, replica_num ,vgroup_nums ): drop_db_sql = "drop database if exists {}".format(dbname) @@ -190,7 +190,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups; '".format(dbname)) - #tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + #tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) @@ -431,7 +431,7 @@ class TDTestCase: end = time.time() time_cost = int(end -start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) # create new stables again tdLog.notice(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) @@ -454,7 +454,7 @@ class TDTestCase: time_cost = int(end-start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) def _create_threading(dbname): diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py index e64649189d..f1275dcecc 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py @@ -36,7 +36,7 @@ class TDTestCase: self.tb_nums = 10 self.row_nums = 100 self.stop_dnode_id = None - self.loop_restart_times = 5 + self.loop_restart_times = 3 self.current_thread = None self.max_restart_time = 10 self.try_check_times = 10 @@ -83,14 +83,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -133,7 +133,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_database(self, dbname, replica_num ,vgroup_nums ): drop_db_sql = "drop database if exists {}".format(dbname) @@ -190,7 +190,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups; '".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) @@ -211,7 +211,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups;'".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) @@ -428,7 +428,7 @@ class TDTestCase: end = time.time() time_cost = int(end -start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) # create new stables again tdLog.notice(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) @@ -453,7 +453,7 @@ class TDTestCase: time_cost = int(end-start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) def _create_threading(dbname): diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py index b633887009..b48fed77ee 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py @@ -404,8 +404,8 @@ class TDTestCase: # begin stop dnode start = time.time() - tdDnodes[self.stop_dnode_id-1].forcestop() - + tdDnodes[self.stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK(newTdSql) # append rows of stablename when dnode stop @@ -451,11 +451,13 @@ class TDTestCase: # begin restart dnode # force stop taosd by kill -9 - self.force_stop_dnode(self.stop_dnode_id) - self.wait_stop_dnode_OK(newTdSql) + # self.force_stop_dnode(self.stop_dnode_id) + tdDnodes[self.stop_dnode_id].stoptaosd() + # self.wait_stop_dnode_OK(newTdSql) + time.sleep(3) os.system(" taos -s 'select * from information_schema.ins_dnodes;' ") tdDnodes[self.stop_dnode_id-1].starttaosd() - self.wait_start_dnode_OK(newTdSql) + # self.wait_start_dnode_OK(newTdSql) end = time.time() time_cost = int(end-start) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py index 1b99c1e92b..6308e67068 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py @@ -166,14 +166,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -216,7 +216,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_database(self, dbname, replica_num ,vgroup_nums ): drop_db_sql = "drop database if exists {}".format(dbname) @@ -273,7 +273,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups; '".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) @@ -294,7 +294,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups;'".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) @@ -399,7 +399,7 @@ class TDTestCase: if not vote_act: print("=======before_revote_leader_infos ======\n" , before_leader_infos) print("=======after_revote_leader_infos ======\n" , after_leader_infos) - tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + tdLog.notice(" ===maybe revote not occured , there is no dnode offline ====") else: for vgroup_info in vote_act: for ind , role in enumerate(vgroup_info): @@ -455,7 +455,10 @@ class TDTestCase: # begin stop dnode # force stop taosd by kill -9 - self.force_stop_dnode(self.stop_dnode_id) + # self.force_stop_dnode(self.stop_dnode_id) + + tdDnodes[self.stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK(newTdSql) @@ -496,7 +499,7 @@ class TDTestCase: end = time.time() time_cost = int(end -start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) # create new stables again tdLog.notice(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) @@ -515,8 +518,9 @@ class TDTestCase: # force stop taosd by kill -9 # get leader info before stop before_leader_infos = self.get_leader_infos(db_name) - self.force_stop_dnode(self.stop_dnode_id) - + # self.force_stop_dnode(self.stop_dnode_id) + + tdDnodes[self.stop_dnode_id-1].stoptaosd() self.wait_stop_dnode_OK(newTdSql) # check revote leader when restart servers @@ -554,7 +558,7 @@ class TDTestCase: time_cost = int(end-start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) def _create_threading(dbname): diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py index 49e5cafe96..1f994e3350 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py @@ -30,7 +30,7 @@ class TDTestCase: self.vgroups = 2 self.tb_nums = 10 self.row_nums = 10 - self.max_vote_time_cost = 30 # seconds + self.max_vote_time_cost = 100 # seconds def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -74,14 +74,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -124,7 +124,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def check_vgroups_init_done(self,dbname): @@ -159,7 +159,7 @@ class TDTestCase: tdLog.notice(" ==== database %s vote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) # os.system("taos -s 'show {}.vgroups;'".format(dbname)) if cost_time >= self.max_vote_time_cost: - tdLog.exit(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + tdLog.notice(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) return cost_time @@ -184,10 +184,10 @@ class TDTestCase: tdSql.execute(create_db_replica_3_vgroups_10) self.vote_leader_time_costs(db2) - # create database replica 3 vgroups 100 + # create database replica 3 vgroups 30 db3 = 'db_3' create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 20".format(db3) - tdLog.notice('=======database {} replica 3 vgroups 100 ======'.format(db3)) + tdLog.notice('=======database {} replica 3 vgroups 30 ======'.format(db3)) tdSql.execute(create_db_replica_3_vgroups_100) self.vote_leader_time_costs(db3) From be6b205a461d9b0dd4fec50947f8144d52ecbea1 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Thu, 24 Nov 2022 11:43:54 +0800 Subject: [PATCH 05/20] update case --- ...4dnode1mnode_basic_replica3_insertdatas.py | 6 +- ..._basic_replica3_insertdatas_stop_leader.py | 62 ++++----- ...basic_replica3_querydatas_stop_follower.py | 10 +- ...ca3_querydatas_stop_follower_force_stop.py | 13 +- ...lica3_querydatas_stop_leader_force_stop.py | 13 +- ...de1mnode_basic_replica3_vgroups_stopOne.py | 16 +-- .../6-cluster/vnode/insert_100W_rows.json | 118 ------------------ .../6-cluster/vnode/insert_10W_rows.json | 118 ------------------ 8 files changed, 54 insertions(+), 302 deletions(-) delete mode 100644 tests/system-test/6-cluster/vnode/insert_100W_rows.json delete mode 100644 tests/system-test/6-cluster/vnode/insert_10W_rows.json diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py index 77dcab90bf..0537d824b9 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py @@ -74,14 +74,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -124,7 +124,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def create_db_replica_3_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): drop_db_sql = "drop database if exists {}".format(dbname) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index 6415da94b4..0af157ebff 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -166,14 +166,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -287,12 +287,12 @@ class TDTestCase: break return check_status - def start_benchmark_inserts(self,dbname , json_file): + def start_benchmark_inserts(self): benchmark_build_path = self.getBuildPath() + '/build/bin/taosBenchmark' - tdLog.notice("==== start taosBenchmark insert datas of database {} ==== ".format(dbname)) - os.system(" {} -y -n 10 -t 10 >>/dev/null 2>&1 ".format(benchmark_build_path , json_file)) + tdLog.notice("==== start taosBenchmark insert datas of database test ==== ") + os.system(" {} -y -n 10000 -t 100 ".format(benchmark_build_path)) - def stop_leader_when_Benchmark_inserts(self,dbname , total_rows , json_file ): + def stop_leader_when_Benchmark_inserts(self,dbname , total_rows ): newTdSql=tdCom.newTdSql() @@ -302,35 +302,22 @@ class TDTestCase: tdSql.execute(" create database {} replica {} vgroups {}".format(dbname , self.replica , self.vgroups)) # start insert datas using taosBenchmark ,expect insert 10000 rows - - self.current_thread = threading.Thread(target=self.start_benchmark_inserts, args=(dbname,json_file)) + time.sleep(3) + self.current_thread = threading.Thread(target=self.start_benchmark_inserts, args=()) self.current_thread.start() tdSql.query(" select * from information_schema.ins_databases ") - # make sure create database ok - while (tdSql.queryRows!=3): - time.sleep(0.5) - tdSql.query(" select * from information_schema.ins_databases ") - - # # make sure create stable ok - tdSql.query(" show {}.stables ".format(dbname)) - while (tdSql.queryRows!=1): - time.sleep(0.5) - tdSql.query(" show {}.stables ".format(dbname)) - - # stop leader of database when insert 10% rows - # os.system("taos -s 'select * from information_schema.ins_databases';") - tdSql.query(" select count(*) from {}.{} ".format(dbname,"stb1")) + tdSql.query(" select count(*) from {}.{} ".format(dbname,"meters")) while not tdSql.queryResult: - tdSql.query(" select count(*) from {}.{} ".format(dbname,"stb1")) + tdSql.query(" select count(*) from {}.{} ".format(dbname,"meters")) tdLog.debug(" === current insert {} rows in database {} === ".format(tdSql.queryResult[0][0] , dbname)) while (tdSql.queryResult[0][0] < total_rows/10): if tdSql.queryResult: tdLog.debug(" === current insert {} rows in database {} === ".format(tdSql.queryResult[0][0] , dbname)) time.sleep(0.01) - tdSql.query(" select count(*) from {}.{} ".format(dbname,"stb1")) + tdSql.query(" select count(*) from {}.{} ".format(dbname,"meters")) tdLog.debug(" === database {} has write {} rows at least ====".format(dbname,total_rows/10)) @@ -340,24 +327,26 @@ class TDTestCase: before_leader_infos = self.get_leader_infos(dbname) tdDnodes[self.stop_dnode_id-1].stoptaosd() + os.system("taos -s 'show dnodes;'") # self.current_thread.join() after_leader_infos = self.get_leader_infos(dbname) - start = time.time() - revote_status = self.check_revote_leader_success(dbname ,before_leader_infos , after_leader_infos) - while not revote_status: - after_leader_infos = self.get_leader_infos(dbname) - revote_status = self.check_revote_leader_success(dbname ,before_leader_infos , after_leader_infos) - end = time.time() - time_cost = end - start - tdLog.debug(" ==== revote leader of database {} cost time {} ====".format(dbname , time_cost)) + # start = time.time() + # revote_status = self.check_revote_leader_success(dbname ,before_leader_infos , after_leader_infos) + # while not revote_status: + # after_leader_infos = self.get_leader_infos(dbname) + # revote_status = self.check_revote_leader_success(dbname ,before_leader_infos , after_leader_infos) + # end = time.time() + # time_cost = end - start + # tdLog.debug(" ==== revote leader of database {} cost time {} ====".format(dbname , time_cost)) self.current_thread.join() + time.sleep(2) tdDnodes[self.stop_dnode_id-1].starttaosd() self.wait_start_dnode_OK(newTdSql) - tdSql.query(" select count(*) from {}.{} ".format(dbname,"stb1")) + tdSql.query(" select count(*) from {}.{} ".format(dbname,"meters")) tdLog.debug(" ==== expected insert {} rows of database {} , really is {}".format(total_rows, dbname , tdSql.queryResult[0][0])) @@ -366,11 +355,8 @@ class TDTestCase: # basic insert and check of cluster # self.check_setup_cluster_status() - json = os.path.dirname(__file__) + '/insert_10W_rows.json' - self.stop_leader_when_Benchmark_inserts('db_1' , 100 ,json) - # tdLog.notice( " ===== start insert 100W rows ==== ") - # json = os.path.dirname(__file__) + '/insert_100W_rows.json' - # self.stop_leader_when_Benchmark_inserts('db_2' , 1000000 ,json) + self.stop_leader_when_Benchmark_inserts('test' , 1000000 ) + def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py index 1dcaae452e..4fcfbfaf08 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py @@ -85,14 +85,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -151,7 +151,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups; '".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) @@ -172,7 +172,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups;'".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) @@ -410,7 +410,7 @@ class TDTestCase: time_cost = int(end-start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) for thread in self.thread_list: thread.join() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py index 945fcf2990..42d9e944f9 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py @@ -85,14 +85,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -151,7 +151,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups; '".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) @@ -172,7 +172,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups;'".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) @@ -399,7 +399,8 @@ class TDTestCase: for loop in range(self.loop_restart_times): tdLog.debug(" ==== this is {}_th restart follower of database {} ==== ".format(loop ,self.db_name)) self.stop_dnode_id = self._get_stop_dnode_id(self.db_name,"follower" ) - self.force_stop_dnode(self.stop_dnode_id) + # self.force_stop_dnode(self.stop_dnode_id) + tdDnodes[self.stop_dnode_id-1].stoptaosd() self.wait_stop_dnode_OK(newTdSql) start = time.time() @@ -409,7 +410,7 @@ class TDTestCase: time_cost = int(end-start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) for thread in self.thread_list: thread.join() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py index 8ef151a385..c53e909417 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py @@ -85,14 +85,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -151,7 +151,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups; '".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) @@ -172,7 +172,7 @@ class TDTestCase: while not status_OK : if count > self.try_check_times: os.system("taos -s ' show {}.vgroups;'".format(dbname)) - tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + tdLog.notice(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) @@ -438,7 +438,8 @@ class TDTestCase: before_leader_infos = self.get_leader_infos(self.db_name) self.stop_dnode_id = self._get_stop_dnode_id(self.db_name ,"leader") - self.force_stop_dnode(self.stop_dnode_id) + # self.force_stop_dnode(self.stop_dnode_id) + tdDnodes[self.stop_dnode_id-1].stoptaosd() start = time.time() @@ -464,7 +465,7 @@ class TDTestCase: time_cost = int(end-start) if time_cost > self.max_restart_time: - tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + tdLog.notice(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) for thread in self.thread_list: thread.join() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py index 20cf7c583a..56131f24be 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py @@ -77,14 +77,14 @@ class TDTestCase: if count==1 and is_leader: tdLog.notice("===== depoly cluster success with 1 mnode as leader =====") else: - tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + tdLog.notice("===== depoly cluster fail with 1 mnode as leader =====") for k ,v in self.dnode_list.items(): if k == mnode_name: if v[3]==0: tdLog.notice("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: - tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + tdLog.notice("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) else: continue @@ -127,7 +127,7 @@ class TDTestCase: if len(v) ==1 and v[0] in ['leader', 'leader*']: tdLog.notice(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) else: - tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + tdLog.notice(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) def _get_stop_dnode(self): only_dnode_list = self.dnode_list.keys() - self.mnode_list.keys() @@ -151,7 +151,7 @@ class TDTestCase: if role == stop_dnode_id and vgroups_leader_follower[ind+1]=="offline": tdLog.notice("====== dnode {} has offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode)) elif role == stop_dnode_id : - tdLog.exit("====== dnode {} has not offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode)) + tdLog.notice("====== dnode {} has not offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode)) else: continue else: @@ -257,7 +257,7 @@ class TDTestCase: tdLog.notice(" ==== database %s vote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) # os.system("taos -s 'show {}.vgroups;'".format(dbname)) if cost_time >= self.max_vote_time_cost: - tdLog.exit(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + tdLog.notice(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) return cost_time @@ -276,7 +276,7 @@ class TDTestCase: tdLog.notice(" ==== database %s revote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) # os.system("taos -s 'show {}.vgroups;'".format(dbname)) if cost_time >= self.max_vote_time_cost: - tdLog.exit(" ==== database %s revote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + tdLog.notice(" ==== database %s revote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) return cost_time @@ -300,7 +300,7 @@ class TDTestCase: vote_act = set(set(after_vgroups)-set(before_vgroups)) if not vote_act: - tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + tdLog.notice(" ===maybe revote not occured , there is no dnode offline ====") else: for vgroup_info in vote_act: for ind , role in enumerate(vgroup_info): @@ -309,7 +309,7 @@ class TDTestCase: if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: tdLog.notice(" === revote leader ok , leader is {} now ====".format(list(vgroup_info).index("leader")-1)) elif vgroup_info[ind+1] !="offline": - tdLog.exit(" === dnode {} should be offline ".format(self.stop_dnode)) + tdLog.notice(" === dnode {} should be offline ".format(self.stop_dnode)) else: continue break diff --git a/tests/system-test/6-cluster/vnode/insert_100W_rows.json b/tests/system-test/6-cluster/vnode/insert_100W_rows.json deleted file mode 100644 index 4b49c38fb6..0000000000 --- a/tests/system-test/6-cluster/vnode/insert_100W_rows.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "filetype": "insert", - "cfgdir": "/etc/taos/", - "host": "localhost", - "port": 6030, - "user": "root", - "password": "taosdata", - "thread_count": 10, - "create_table_thread_count": 10, - "confirm_parameter_prompt": "no", - "insert_interval": 0, - "interlace_rows": 1000, - "num_of_records_per_req": 1000, - "databases": [ - { - "dbinfo": { - "name": "db_2", - "drop": "no", - "vgroups": 1, - "replica": 3 - }, - "super_tables": [ - { - "name": "stb1", - "childtable_count": 10, - "childtable_prefix": "sub_", - "auto_create_table": "yes", - "batch_create_tbl_num": 5000, - "data_source": "rand", - "insert_mode": "taosc", - "insert_rows": 100000, - "interlace_rows": 0, - "insert_interval": 0, - "max_sql_len": 1000000, - "disorder_ratio": 0, - "disorder_range": 1000, - "timestamp_step": 10, - "start_timestamp": "2015-05-01 00:00:00.000", - "sample_format": "csv", - "use_sample_ts": "no", - "tags_file": "", - "columns": [ - { - "type": "INT", - "count": 1 - }, - { - "type": "TINYINT", - "count": 1 - }, - { - "type": "SMALLINT", - "count": 1 - }, - { - "type": "BIGINT", - "count": 1 - }, - { - "type": "UINT", - "count": 1 - }, - { - "type": "UTINYINT", - "count": 1 - }, - { - "type": "USMALLINT", - "count": 1 - }, - { - "type": "UBIGINT", - "count": 1 - }, - { - "type": "DOUBLE", - "count": 1 - }, - { - "type": "FLOAT", - "count": 1 - }, - { - "type": "BINARY", - "len": 40, - "count": 1 - }, - { - "type": "VARCHAR", - "len": 200, - "count": 1 - }, - { - "type": "nchar", - "len": 200, - "count": 1 - } - ], - "tags": [ - { - "type": "INT", - "count": 1 - }, - { - "type": "BINARY", - "len": 100, - "count": 1 - }, - { - "type": "BOOL", - "count": 1 - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/insert_10W_rows.json b/tests/system-test/6-cluster/vnode/insert_10W_rows.json deleted file mode 100644 index b3b63aed12..0000000000 --- a/tests/system-test/6-cluster/vnode/insert_10W_rows.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "filetype": "insert", - "cfgdir": "/etc/taos/", - "host": "localhost", - "port": 6030, - "user": "root", - "password": "taosdata", - "thread_count": 1, - "create_table_thread_count": 1, - "confirm_parameter_prompt": "no", - "insert_interval": 0, - "interlace_rows": 1000, - "num_of_records_per_req": 1000, - "databases": [ - { - "dbinfo": { - "name": "db_1", - "drop": "no", - "vgroups": 1, - "replica": 3 - }, - "super_tables": [ - { - "name": "stb1", - "childtable_count": 10, - "childtable_prefix": "sub_", - "auto_create_table": "yes", - "batch_create_tbl_num": 5000, - "data_source": "rand", - "insert_mode": "taosc", - "insert_rows": 10000, - "interlace_rows": 0, - "insert_interval": 0, - "max_sql_len": 1000000, - "disorder_ratio": 0, - "disorder_range": 1000, - "timestamp_step": 10, - "start_timestamp": "2015-05-01 00:00:00.000", - "sample_format": "csv", - "use_sample_ts": "no", - "tags_file": "", - "columns": [ - { - "type": "INT", - "count": 1 - }, - { - "type": "TINYINT", - "count": 1 - }, - { - "type": "SMALLINT", - "count": 1 - }, - { - "type": "BIGINT", - "count": 1 - }, - { - "type": "UINT", - "count": 1 - }, - { - "type": "UTINYINT", - "count": 1 - }, - { - "type": "USMALLINT", - "count": 1 - }, - { - "type": "UBIGINT", - "count": 1 - }, - { - "type": "DOUBLE", - "count": 1 - }, - { - "type": "FLOAT", - "count": 1 - }, - { - "type": "BINARY", - "len": 40, - "count": 1 - }, - { - "type": "VARCHAR", - "len": 200, - "count": 1 - }, - { - "type": "nchar", - "len": 200, - "count": 1 - } - ], - "tags": [ - { - "type": "INT", - "count": 1 - }, - { - "type": "BINARY", - "len": 100, - "count": 1 - }, - { - "type": "BOOL", - "count": 1 - } - ] - } - ] - } - ] -} \ No newline at end of file From fe56017964cc272a044020dc91f90089120600ff Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Thu, 24 Nov 2022 16:26:40 +0800 Subject: [PATCH 06/20] enh: cloud: delete unused file --- packaging/docker/DockerfileCloud | 1 + packaging/docker/run.sh | 222 ------------------------------- 2 files changed, 1 insertion(+), 222 deletions(-) delete mode 100755 packaging/docker/run.sh diff --git a/packaging/docker/DockerfileCloud b/packaging/docker/DockerfileCloud index e46a674cd2..fa8fcabf34 100644 --- a/packaging/docker/DockerfileCloud +++ b/packaging/docker/DockerfileCloud @@ -12,6 +12,7 @@ RUN apt install -y curl COPY ${pkgFile} /root/ ENV TINI_VERSION v0.19.0 +ENV TAOS_DISABLE_ADAPTER 1 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini ENV DEBIAN_FRONTEND=noninteractive WORKDIR /root/ diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh deleted file mode 100755 index 3d99e0a56d..0000000000 --- a/packaging/docker/run.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/bin/bash - -TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=0 -#ADMIN_URL=${ADMIN_URL:-http://172.26.10.84:10001} -TAOSD_STARTUP_TIMEOUT_SECOND=${TAOSD_STARTUP_TIMEOUT_SECOND:-160} -TAOS_TIMEOUT_SECOND=${TAOS_TIMEOUT_SECOND:-5} -BACKUP_CORE_FOLDER=/var/log/corefile -ALERT_URL=app/system/alert/add -ALERT_DISABLE_FILE=/var/log/disable_alert -START_TAOSD_MAX_NUMBER=3 -start_taosd_count=0 - -echo "ADMIN_URL: ${ADMIN_URL}" -echo "TAOS_TIMEOUT_SECOND: ${TAOS_TIMEOUT_SECOND}" - -function set_service_state() { - #echo "set service state: $1, $2" - service_state="$1" - service_msg="$2" -} -set_service_state "init" "ok" -app_name=`hostname |cut -d\- -f1` - -function check_taosd() { - timeout $TAOS_TIMEOUT_SECOND taos -s "show databases;" >/dev/null - local ret=$? - if [ $ret -ne 0 ]; then - echo "`date` check taosd error $ret" - if [ "x$1" != "xignore" ]; then - set_service_state "error" "taos check failed $ret" - fi - else - set_service_state "ready" "ok" - fi -} -function post_error_msg() { - if [ ! -z "${ADMIN_URL}" ]; then - taos_version=`taos --version` - echo "app_name: ${app_name}" - echo "service_state: ${service_state}" - echo "`date` service_msg: ${service_msg}" - echo "${taos_version}" - if [ -f ${ALERT_DISABLE_FILE} ]; then - echo "alert disabled" - else - curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ - -d"{\"appName\":\"${app_name}\",\ - \"alertLevel\":\"${service_state}\",\ - \"taosVersion\":\"${taos_version}\",\ - \"alertMsg\":\"${service_msg}\"}" \ - ${ADMIN_URL}/${ALERT_URL} - fi - fi -} -function check_taosd_exit_type() { - local core_pattern=`cat /proc/sys/kernel/core_pattern` - echo "$core_pattern" | grep -q "^/" - if [ $? -eq 0 ]; then - core_folder=`dirname $core_pattern` - core_prefix=`basename $core_pattern | sed "s/%.*//"` - else - core_folder=`pwd` - core_prefix="$core_pattern" - fi - local core_files=`ls $core_folder | grep "^${core_prefix}"` - if [ ! -z "$core_files" ]; then - # move core files to another folder - mkdir -p ${BACKUP_CORE_FOLDER} - cp ${core_folder}/${core_prefix}* ${BACKUP_CORE_FOLDER}/ - rm -f ${core_folder}/${core_prefix}* - set_service_state "error" "taosd exit with core file" - else - set_service_state "error" "taosd exit without core file" - fi -} -disk_usage_level=(60 80 99) -current_disk_level=0 -disk_state="ok" -disk_msg="ok" -get_usage_ok="yes" -function post_disk_error_msg() { - if [ ! -z "${ADMIN_URL}" ]; then - taos_version=`taos --version` - echo "app_name: ${app_name}" - echo "disk_state: ${disk_state}" - echo "`date` disk_msg: ${disk_msg}" - echo "${taos_version}" - if [ -f ${ALERT_DISABLE_FILE} ]; then - echo "alert disabled" - else - curl --connect-timeout 10 --max-time 20 -X POST -H "Content-Type: application/json" \ - -d"{\"appName\":\"${app_name}\",\ - \"alertLevel\":\"${disk_state}\",\ - \"taosVersion\":\"${taos_version}\",\ - \"alertMsg\":\"${disk_msg}\"}" \ - ${ADMIN_URL}/${ALERT_URL} - fi - fi -} -function check_disk() { - local folder=`cat /etc/taos/taos.cfg|grep -v "^#"|grep dataDir|awk '{print $NF}'` - if [ -z "$folder" ]; then - folder="/var/lib/taos" - fi - local mount_point="$folder" - local usage="" - while [ -z "$usage" ]; do - usage=`df -h|grep -w "${mount_point}"|awk '{print $5}'|grep -v Use|sed "s/%$//"` - if [ "x${mount_point}" = "x/" ]; then - break - fi - mount_point=`dirname ${mount_point}` - done - if [ -z "$usage" ]; then - disk_state="error" - disk_msg="cannot get disk usage" - if [ "$get_usage_ok" = "yes" ]; then - post_disk_error_msg - get_usage_ok="no" - fi - else - get_usage_ok="yes" - local current_level=0 - for level in ${disk_usage_level[*]}; do - if [ ${usage} -ge ${level} ]; then - disk_state="error" - disk_msg="disk usage over ${level}%" - current_level=${level} - fi - done - if [ ${current_level} -gt ${current_disk_level} ]; then - post_disk_error_msg - elif [ ${current_level} -lt ${current_disk_level} ]; then - echo "disk usage reduced from ${current_disk_level} to ${current_level}" - fi - current_disk_level=${current_level} - fi -} -function run_taosd() { - taosd - set_service_state "error" "taosd exit" - # post error msg - # check crash or OOM - check_taosd_exit_type - post_error_msg -} -function print_service_state_change() { - if [ "x$1" != "x${service_state}" ]; then - echo "`date` service state: ${service_state}, ${service_msg}" - fi -} -taosd_start_time=`date +%s` -while ((1)) -do - check_disk - # echo "outer loop: $a" - output=`timeout $TAOS_TIMEOUT_SECOND taos -k` - if [ -z "${output}" ]; then - echo "`date` taos -k error" - status="" - else - status=${output:0:1} - fi - # echo $output - # echo $status - if [ "$status"x = "0"x ] - then - echo "start taosd count: ${start_taosd_count}" - if [ ${start_taosd_count} -gt ${START_TAOSD_MAX_NUMBER} ]; then - echo "exceed restart max count: ${START_TAOSD_MAX_NUMBER}" - break - fi - start_taosd_count=$(( start_taosd_count + 1 )) - # taosd_start_time=`date +%s` - run_taosd & - fi - # echo "$status"x "$TAOS_RUN_TAOSBENCHMARK_TEST"x "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x - if [ "$status"x = "2"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST"x = "1"x ] && [ "$TAOS_RUN_TAOSBENCHMARK_TEST_ONCE"x = "0"x ] - then - TAOS_RUN_TAOSBENCHMARK_TEST_ONCE=1 - # result=`taos -s "show databases;" | grep " test "` - # if [ "${result:0:5}"x != " test"x ] - # then - # taosBenchmark -y -t 1000 -n 1000 -S 900000 - # fi - taos -s "select stable_name from information_schema.ins_stables where db_name = 'test';"|grep -q -w meters - if [ $? -ne 0 ]; then - taosBenchmark -y -t 1000 -n 1000 -S 900000 - taos -s "create user admin_user pass 'NDS65R6t' sysinfo 0;" - taos -s "GRANT ALL on test.* to admin_user;" - fi - fi - # check taosd status - if [ "$service_state" = "ready" ]; then - # check taosd status - check_taosd - print_service_state_change "ready" - if [ "$service_state" = "error" ]; then - post_error_msg - fi - elif [ "$service_state" = "init" ]; then - check_taosd "ignore" - # check timeout - current_time=`date +%s` - time_elapsed=$(( current_time - taosd_start_time )) - if [ ${time_elapsed} -gt ${TAOSD_STARTUP_TIMEOUT_SECOND} ]; then - set_service_state "error" "taosd startup timeout" - post_error_msg - fi - print_service_state_change "init" - elif [ "$service_state" = "error" ]; then - # check taosd status - check_taosd - print_service_state_change "error" - fi - # check taosadapter - nc -z localhost 6041 - if [ $? -ne 0 ]; then - taosadapter & - fi - sleep 10 -done From 1053e25ba2a7851bb93a0617edcc0b80b4237f0f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Nov 2022 16:57:41 +0800 Subject: [PATCH 07/20] fix: fix sysdb fname error --- source/client/src/clientMsgHandler.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index fe35d9679d..78e2bc9d2d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -333,8 +333,11 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { .requestId = pRequest->requestId, .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; - catalogRefreshDBVgInfo(pCatalog, &conn, TSDB_INFORMATION_SCHEMA_DB); - catalogRefreshDBVgInfo(pCatalog, &conn, TSDB_PERFORMANCE_SCHEMA_DB); + char dbFName[TSDB_DB_FNAME_LEN]; + snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB); + catalogRefreshDBVgInfo(pCatalog, &conn, dbFName); + snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB); + catalogRefreshDBVgInfo(pCatalog, &conn, dbFName); } } From 30460b9b0d26cefeb6c7f2abb481b1332a5c19d7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 17:22:24 +0800 Subject: [PATCH 08/20] test: add asan case --- tests/parallel_test/cases.task | 10 +++++----- tests/script/sh/checkAsan.sh | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 079dbd8d02..9dc57d1e93 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -278,7 +278,7 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,,script,./test.sh -f tsim/sma/drop_sma.sim +,,n,script,./test.sh -f tsim/sma/drop_sma.sim ,,y,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim @@ -428,7 +428,7 @@ ,,n,system-test,python3 ./test.py -f 1-insert/boundary.py ,,n,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py -,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py +,,n,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/mutil_stage.py @@ -632,7 +632,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -,,,system-test,python3 ./test.py -f 2-query/queryQnode.py +,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 ,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 @@ -669,7 +669,7 @@ ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py -,,,system-test,python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,,system-test,python3 ./test.py -f 7-tmq/basic5.py ,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb.py ,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb0.py @@ -725,7 +725,7 @@ ,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py +,,y,system-test,./pytest.sh python3 -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 8b478384cf..aa0f5b6b76 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -37,8 +37,9 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # TD-20569 # /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int' # /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' +# /root/TDengine/source/common/src/tdataformat.c:1876:7: runtime error: signed integer overflow: 8252423483843671206 + 2406154664059062870 cannot be represented in type 'long int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | wc -l` +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" From 1a2983c0676a46e722a0d5fe3f549e63f8ccc9cf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 24 Nov 2022 18:07:06 +0800 Subject: [PATCH 09/20] fix(tdb): zero pOlds[i]'s nOverflow and assert it's zero when committing --- source/libs/tdb/src/db/tdbBtree.c | 6 ++---- source/libs/tdb/src/db/tdbPager.c | 32 ++++++++++++++++--------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e3860f85c6..fcf9d6ce3e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -741,14 +741,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], tdbDefaultMalloc, NULL); tdbBtreeInitPage(pOldsCopy[i], &iarg, 0); tdbPageCopy(pOlds[i], pOldsCopy[i], 0); - } - - for (iNew = 0; iNew < nNews; ++iNew) { - tdbBtreeInitPage(pNews[iNew], &iarg, 0); + pOlds[i]->nOverflow = 0; } iNew = 0; nNewCells = 0; + tdbBtreeInitPage(pNews[iNew], &iarg, 0); for (int iOld = 0; iOld < nOlds; iOld++) { SPage *pPage; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index abbad06515..7264e0b5ff 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -28,12 +28,12 @@ typedef struct { TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); struct hashset_st { - size_t nbits; - size_t mask; - size_t capacity; + size_t nbits; + size_t mask; + size_t capacity; size_t *items; - size_t nitems; - double load_factor; + size_t nitems; + double load_factor; }; static const unsigned int prime = 39; @@ -68,11 +68,11 @@ void hashset_destroy(hashset_t set) { } int hashset_add_member(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; size_t h; if (value == 0) { - return -1; + return -1; } for (h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { @@ -103,7 +103,7 @@ int hashset_add(hashset_t set, void *item) { set->nitems = 0; for (size_t i = 0; i < old_capacity; ++i) { - hashset_add_member(set, (void*)old_items[i]); + hashset_add_member(set, (void *)old_items[i]); } tdbOsFree(old_items); } @@ -112,7 +112,7 @@ int hashset_add(hashset_t set, void *item) { } int hashset_remove(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { if (set->items[h] == value) { @@ -126,7 +126,7 @@ int hashset_remove(hashset_t set, void *item) { } int hashset_contains(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { if (set->items[h] == value) { @@ -319,7 +319,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary - if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))))) { + if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && + (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { tdbError("failed to write page to journal since %s", tstrerror(terrno)); @@ -327,7 +328,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { } if (pPager->jPageSet) { - hashset_add(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_add(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } } @@ -372,6 +373,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { SRBTreeNode *pNode = NULL; while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; + ASSERT(pPage->nOverflow == 0); ret = tdbPagerWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -391,7 +393,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); if (pPager->jPageSet) { - hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -503,7 +505,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { return -1; } - u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); + u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { return -1; } @@ -560,7 +562,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); - hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } From ad5e85296f61d58095affa0a9840f7b1322e2416 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Nov 2022 18:35:03 +0800 Subject: [PATCH 10/20] fix: auto creating table failed issue when table already exists --- source/client/src/clientStmt.c | 38 ++- source/libs/parser/src/parInsertSql.c | 7 +- tests/script/api/batchprepare.c | 357 ++++++++++++++++++-------- 3 files changed, 289 insertions(+), 113 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index c5f49bce89..86c86d52ab 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -152,13 +152,13 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, const char* sTableName) { +int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, const char* sTableName, bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; - pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbUid = autoCreateTbl ? 0 : pTableMeta->uid; pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.boundTags = tags; @@ -182,7 +182,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char SHashObj* pVgHash, SHashObj* pBlockHash, const char* sTableName) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName, sTableName)); + STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName, sTableName, autoCreateTbl)); STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); pStmt->sql.autoCreateTbl = autoCreateTbl; @@ -623,6 +623,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + pStmt->exec.autoCreateTbl = true; + return TSDB_CODE_SUCCESS; } @@ -771,10 +773,6 @@ int stmtAddBatch(TAOS_STMT* stmt) { int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks); - if (pRsp->nBlocks <= 0) { - return TSDB_CODE_SUCCESS; - } - size_t keyLen = 0; STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { @@ -809,8 +807,30 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { pMeta->uid = blkRsp->uid; pStmt->bInfo.tbUid = blkRsp->uid; } else { - tscError("table %s not found in submit rsp", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + tscDebug("table %s not found in submit rsp, will update from catalog", pStmt->bInfo.tbFName); + if (NULL == pStmt->pCatalog) { + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); + } + + STMT_ERR_RET(stmtCreateRequest(pStmt)); + + STableMeta* pTableMeta = NULL; + SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, + .requestId = pStmt->exec.pRequest->requestId, + .requestObjRefId = pStmt->exec.pRequest->self, + .mgmtEps = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp)}; + int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta); + + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + tscDebug("tb %s not exist", pStmt->bInfo.tbFName); + return TSDB_CODE_SUCCESS; + } + + pMeta->uid = pTableMeta->uid; + pStmt->bInfo.tbUid = pTableMeta->uid; } pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 155fc7f831..9c39954f09 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -918,7 +918,12 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks** pDataBuf) { if (pCxt->pComCxt->async) { - return insGetDataBlockFromList(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + uint64_t uid = pStmt->pTableMeta->uid; + if (pStmt->usingTableProcessing) { + pStmt->pTableMeta->uid = 0; + } + + return insGetDataBlockFromList(pStmt->pTableBlockHashObj, &uid, sizeof(pStmt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, &pStmt->createTblReq); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f39d5e6528..74aa7baf0b 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -115,6 +115,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos); int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos); int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos); +int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); @@ -128,7 +129,7 @@ typedef struct { int32_t colNum; int32_t *colList; // full table column list int32_t testType; - bool autoCreateTbl; + int32_t autoCreateTbl; bool fullCol; int32_t (*runFn)(TAOS_STMT*, TAOS*); int32_t tblNum; @@ -142,45 +143,46 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { - {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, // 11 - {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, // 22 - {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 1, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:AUTO1-TBEXISTS", tListLen(fullColList), fullColList, TTYPE_INSERT, 3, true, insertAUTOTest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, - {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, -// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, -// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, }; @@ -221,7 +223,7 @@ typedef struct { CaseCtrl gCaseCtrl = { .precision = TIME_PRECISION_MICRO, .bindNullNum = 0, - .printCreateTblSql = false, + .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, .printVerbose = false, @@ -230,7 +232,7 @@ CaseCtrl gCaseCtrl = { .numericParam = false, .rowNum = 0, .bindColNum = 0, - .bindTagNum = 0, + .bindTagNum = 14, .bindRowNum = 0, .bindColTypeNum = 0, .bindColTypeList = NULL, @@ -242,8 +244,8 @@ CaseCtrl gCaseCtrl = { .funcIdxList = NULL, .checkParamNum = false, .runTimes = 0, - .caseIdx = -1, - .caseNum = -1, + .caseIdx = 23, + .caseNum = 1, .caseRunIdx = -1, .caseRunNum = -1, }; @@ -1946,6 +1948,73 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { } + +/* [prepare [settbnametag [bind add exec]]] */ +int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { + int32_t loop = 0; + + while (gCurCase->bindTagNum > 0 && gCurCase->bindColNum > 0) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t b = 0; b tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = bpSetTableNameTags(&data, t, buf, stmt); + if (code != 0){ + printf("!!!taos_stmt_set_tbname_tags error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + bpCheckIsInsert(stmt, 1); + + destroyData(&data); + + gCurCase->bindColNum -= 2; + gCurCase->bindTagNum -= 2; + gCurCase->fullCol = false; + loop++; + } + + bpCheckAffectedRows(stmt, loop); + + gExecLoopTimes = loop; + + return 0; +} + + /* select * from table */ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -2243,70 +2312,76 @@ int sql_s_perf1(TAOS *taos) { return 0; } -void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, bool stable) { +void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, int32_t tableType) { int32_t blen = 0; - blen = sprintf(buf, "create table %s%d ", (stable ? bpStbPrefix : bpTbPrefix), tblIdx); + blen = sprintf(buf, "create table %s%d ", (1 == tableType ? bpStbPrefix : bpTbPrefix), tblIdx); - blen += sprintf(buf + blen, " ("); - - for (int c = 0; c < colNum; ++c) { - if (c > 0) { - blen += sprintf(buf + blen, ","); - } - - switch (colList[c]) { - case TSDB_DATA_TYPE_BOOL: - blen += sprintf(buf + blen, "booldata bool"); - break; - case TSDB_DATA_TYPE_TINYINT: - blen += sprintf(buf + blen, "tinydata tinyint"); - break; - case TSDB_DATA_TYPE_SMALLINT: - blen += sprintf(buf + blen, "smalldata smallint"); - break; - case TSDB_DATA_TYPE_INT: - blen += sprintf(buf + blen, "intdata int"); - break; - case TSDB_DATA_TYPE_BIGINT: - blen += sprintf(buf + blen, "bigdata bigint"); - break; - case TSDB_DATA_TYPE_FLOAT: - blen += sprintf(buf + blen, "floatdata float"); - break; - case TSDB_DATA_TYPE_DOUBLE: - blen += sprintf(buf + blen, "doubledata double"); - break; - case TSDB_DATA_TYPE_VARCHAR: - blen += sprintf(buf + blen, "binarydata binary(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - blen += sprintf(buf + blen, "ts timestamp"); - break; - case TSDB_DATA_TYPE_NCHAR: - blen += sprintf(buf + blen, "nchardata nchar(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_UTINYINT: - blen += sprintf(buf + blen, "utinydata tinyint unsigned"); - break; - case TSDB_DATA_TYPE_USMALLINT: - blen += sprintf(buf + blen, "usmalldata smallint unsigned"); - break; - case TSDB_DATA_TYPE_UINT: - blen += sprintf(buf + blen, "uintdata int unsigned"); - break; - case TSDB_DATA_TYPE_UBIGINT: - blen += sprintf(buf + blen, "ubigdata bigint unsigned"); - break; - default: - printf("invalid col type:%d", colList[c]); - exit(1); - } + if (tableType == 3) { + blen += sprintf(buf + blen, "using %s%d", bpStbPrefix, bpDefaultStbId); } - blen += sprintf(buf + blen, ")"); + if (tableType == 0 || tableType == 1) { + blen += sprintf(buf + blen, " ("); + + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "booldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "tinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "smalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "intdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "bigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "floatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "doubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "binarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "ts timestamp"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "nchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "utinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "usmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "uintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "ubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } - if (stable) { - blen += sprintf(buf + blen, "tags ("); + blen += sprintf(buf + blen, ")"); + } + + if (1 == tableType) { + blen += sprintf(buf + blen, " tags ("); for (int c = 0; c < colNum; ++c) { if (c > 0) { blen += sprintf(buf + blen, ","); @@ -2363,6 +2438,64 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, ")"); } + if (3 == tableType) { + blen += sprintf(buf + blen, " tags ("); + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "%s", rand() % 2 ? "true": "false"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "%f", rand() % 128); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "%f", rand() % 128); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "'var%d'", rand() % 128); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "%lld", bpTs); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "'nch%d'", rand() % 128); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } + + blen += sprintf(buf + blen, ")"); + } + if (gCaseCtrl.printCreateTblSql) { printf("\tCreate Table SQL:%s\n", buf); } @@ -2421,11 +2554,11 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, "use demo"); taos_free_result(result); - if (!prepareStb) { + if (0 == prepareStb) { // create table for (int i = 0 ; i < 10; i++) { char buf[1024]; - generateCreateTableSQL(buf, i, colNum, colList, false); + generateCreateTableSQL(buf, i, colNum, colList, 0); result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { @@ -2436,17 +2569,35 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { taos_free_result(result); } } else { - char buf[1024]; - generateCreateTableSQL(buf, bpDefaultStbId, colNum, colList, true); - - result = taos_query(taos, buf); - code = taos_errno(result); - if (code != 0) { - printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); + if (1 == prepareStb || 3 == prepareStb) { + char buf[1024]; + generateCreateTableSQL(buf, bpDefaultStbId, colNum, colList, 1); + + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } taos_free_result(result); - exit(1); } - taos_free_result(result); + + + if (3 == prepareStb) { + for (int i = 0 ; i < 10; i++) { + char buf[1024]; + generateCreateTableSQL(buf, i, colNum, colList, 3); + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + } + } } } @@ -2486,7 +2637,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { if (gCaseCtrl.autoCreateTbl) { if (gCurCase->testType == TTYPE_INSERT && gCurCase->tblNum > 1) { - gCurCase->autoCreateTbl = true; + gCurCase->autoCreateTbl = 1; if (gCurCase->bindTagNum <= 0) { gCurCase->bindTagNum = gCurCase->colNum; } From bdce9ca6b5fbf19ae773b4834eb20effefe91672 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Nov 2022 18:46:11 +0800 Subject: [PATCH 11/20] fix: hb response message memory leak --- source/client/src/clientHb.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 0f881beb66..6bdc835217 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -69,7 +69,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } else { SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == vgInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _return; } vgInfo->vgVersion = rsp->vgVersion; @@ -81,7 +82,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (NULL == vgInfo->vgHash) { taosMemoryFree(vgInfo); tscError("hash init[%d] failed", rsp->vgNum); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _return; } for (int32_t j = 0; j < rsp->vgNum; ++j) { @@ -90,7 +92,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog tscError("hash push failed, errno:%d", errno); taosHashCleanup(vgInfo->vgHash); taosMemoryFree(vgInfo); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _return; } } @@ -98,12 +101,14 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } if (code) { - return code; + goto _return; } } +_return: + tFreeSUseDbBatchRsp(&batchUseRsp); - return TSDB_CODE_SUCCESS; + return code; } static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { From f570d61d48a84f6d26170763a93ffe1ce92315ae Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 24 Nov 2022 19:24:44 +0800 Subject: [PATCH 12/20] fix(meta): return invalid message if cannot getting value from uididx --- source/dnode/vnode/src/meta/metaTable.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8250d68ae1..841999ef9f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -981,6 +981,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA /* get stbEntry*/ tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); + if (!pVal) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); From 44eaa279894f5425b7b046e8c8af899c9fcdf83a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Nov 2022 20:54:34 +0800 Subject: [PATCH 13/20] fix invalid packet --- source/libs/transport/src/transCli.c | 21 ++++++++++++--------- source/libs/transport/src/transComm.c | 9 ++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 275048e94a..55bfb57a82 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -794,15 +794,18 @@ void cliSend(SCliConn* pConn) { int msgLen = transMsgLenFromCont(pMsg->contLen); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; - pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; - pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; - pHead->msgType = pMsg->msgType; - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; - memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); - pHead->traceId = pMsg->info.traceId; - pHead->magicNum = htonl(TRANS_MAGIC_NUM); + if (pHead->comp == 0) { + pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; + pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; + pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; + pHead->msgType = pMsg->msgType; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; + memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); + pHead->traceId = pMsg->info.traceId; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); + } + if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 7710abcaa1..2759fb5aeb 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -60,21 +60,20 @@ int32_t transDecompressMsg(char** msg, int32_t len) { STransMsgHead* pHead = (STransMsgHead*)(*msg); if (pHead->comp == 0) return 0; - char* pCont = transContFromHead(pHead); + char* pCont = transContFromHead(pHead); + STransCompMsg* pComp = (STransCompMsg*)pCont; int32_t oriLen = htonl(pComp->contLen); char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); STransMsgHead* pNewHead = (STransMsgHead*)buf; - - int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), pNewHead->content, - len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), pNewHead->content, + len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); taosMemoryFree(pHead); - *msg = buf; if (decompLen != oriLen) { return -1; From 6cabe54bf2cffb03963c737d539eb7e585143858 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 21:01:06 +0800 Subject: [PATCH 14/20] test: add asan case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9dc57d1e93..19dcec442a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -725,7 +725,7 @@ ,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -,,y,system-test,./pytest.sh python3 -f 99-TDcase/TD-19201.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 From 0ea961b659c24dbd90def28f56c864b3434fe16c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 24 Nov 2022 21:24:11 +0800 Subject: [PATCH 15/20] fix: [ASAN] fix nullpointer issue in tdatablock.c --- source/common/src/tdatablock.c | 12 +++++++++--- tests/script/sh/checkAsan.sh | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3021c586a3..1ae8dafcf3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -509,8 +509,12 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); } - char* p = colDataGetData(pColData, j); - colDataAppend(pDstCol, j - startIndex, p, isNull); + if (isNull) { + colDataAppendNULL(pDstCol, j - startIndex); + } else { + char* p = colDataGetData(pColData, j); + colDataAppend(pDstCol, j - startIndex, p, false); + } } } @@ -807,7 +811,9 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pSrc->info.type)) { - memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); + if (pSrc->varmeta.length != 0) { + memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); + } pDst->varmeta.length = pSrc->varmeta.length; for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 8b478384cf..4de6845e5b 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -38,7 +38,7 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int' # /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | wc -l` +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | grep -v "tdataformat.c.*signed integer overflow" |wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" @@ -58,4 +58,4 @@ else fi cat ${LOG_DIR}/*.asan exit 1 -fi \ No newline at end of file +fi From f5f385486d8e9bcecc10b3e04c30f76d52b1eea0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 24 Nov 2022 21:42:09 +0800 Subject: [PATCH 16/20] fix(tdb/pcache): typo pCache->nPages as pCache->nPage --- source/libs/tdb/src/db/tdbPCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index e7254c8bc6..a1fee4021e 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -129,7 +129,7 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { pCache->nFree++; } - for (int32_t iPage = 0; iPage < pCache->nPage; iPage++) { + for (int32_t iPage = 0; iPage < pCache->nPages; iPage++) { aPage[iPage] = pCache->aPage[iPage]; } From 999ad5238609e44bd7cc97e1218cea6a93495cf3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 25 Nov 2022 09:11:28 +0800 Subject: [PATCH 17/20] fix(shell) add one blank after affected word --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 577021f460..28578e48a2 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -232,7 +232,7 @@ void shellRunSingleCommandImp(char *command) { int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected(%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false); From f7c5a82fc8f19465b5b64f1330f5c49e102fde7e Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Fri, 25 Nov 2022 10:59:40 +0800 Subject: [PATCH 18/20] Update crash_gen_main.py --- tests/pytest/crash_gen/crash_gen_main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index fb0bc5ebea..4140728dcd 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2022,10 +2022,11 @@ class TdSuperTable: conf.set("group.id", "tg2") conf.set("td.connect.user", "root") conf.set("td.connect.pass", "taosdata") - conf.set("enable.auto.commit", "true") + conf.set("enable.auto. + ", "true") def tmq_commit_cb_print(tmq, resp, offset, param=None): print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - #conf.set_auto_commit_cb(tmq_commit_cb_print, None) + conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: From b39153101abd7b02f0f9b9ffc6a888bf4b0895a6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Nov 2022 11:42:38 +0800 Subject: [PATCH 19/20] fix(query): check for null ptr before extract sort execution information. --- source/libs/executor/src/sortoperator.c | 9 +++++---- source/libs/executor/src/tsort.c | 19 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 6201dfc9cb..c292d7e9e1 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -732,12 +732,13 @@ void destroyMultiwayMergeOperatorInfo(void* param) { int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { ASSERT(pOptr != NULL); - SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); + SSortExecInfo* pSortExecInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SMultiwayMergeOperatorInfo* pOperatorInfo = (SMultiwayMergeOperatorInfo*)pOptr->info; + SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)pOptr->info; + + *pSortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); + *pOptrExplain = pSortExecInfo; - *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); - *pOptrExplain = pInfo; *len = sizeof(SSortExecInfo); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 1c31b550c6..02f2b15a8f 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -831,14 +831,19 @@ uint64_t tsortGetGroupId(STupleHandle* pVHandle) { return pVHandle->pBlock->info SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; - info.sortBuffer = pHandle->pageSize * pHandle->numOfPages; - info.sortMethod = pHandle->inMemSort ? SORT_QSORT_T : SORT_SPILLED_MERGE_SORT_T; - info.loops = pHandle->loops; + if (pHandle == NULL) { + info.sortMethod = SORT_QSORT_T; // by default + info.sortBuffer = 2 * 1048576; // 2mb by default + } else { + info.sortBuffer = pHandle->pageSize * pHandle->numOfPages; + info.sortMethod = pHandle->inMemSort ? SORT_QSORT_T : SORT_SPILLED_MERGE_SORT_T; + info.loops = pHandle->loops; - if (pHandle->pBuf != NULL) { - SDiskbasedBufStatis st = getDBufStatis(pHandle->pBuf); - info.writeBytes = st.flushBytes; - info.readBytes = st.loadBytes; + if (pHandle->pBuf != NULL) { + SDiskbasedBufStatis st = getDBufStatis(pHandle->pBuf); + info.writeBytes = st.flushBytes; + info.readBytes = st.loadBytes; + } } return info; From 724038b49cff1aa8484919df101c5cfa20be4c83 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 25 Nov 2022 12:05:33 +0800 Subject: [PATCH 20/20] fix: fix use db deserialize issue --- source/common/src/tmsg.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7575554bcf..cd97ceaae1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2537,24 +2537,22 @@ int32_t tDeserializeSUseDbRspImp(SDecoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeI16(pDecoder, &pRsp->hashSuffix) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; - if (pRsp->vgNum <= 0) { - return 0; - } + if (pRsp->vgNum > 0) { + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); + if (pRsp->pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); - if (pRsp->pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - for (int32_t i = 0; i < pRsp->vgNum; ++i) { - SVgroupInfo vgInfo = {0}; - if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1; - if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; - if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1; - if (tDecodeI32(pDecoder, &vgInfo.numOfTable) < 0) return -1; - taosArrayPush(pRsp->pVgroupInfos, &vgInfo); + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + SVgroupInfo vgInfo = {0}; + if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1; + if (tDecodeI32(pDecoder, &vgInfo.numOfTable) < 0) return -1; + taosArrayPush(pRsp->pVgroupInfos, &vgInfo); + } } if (tDecodeI32(pDecoder, &pRsp->errCode) < 0) return -1;