From 53ce344b362b10d0e7afd5f49d06893ee9b10f15 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 28 Dec 2023 17:09:52 +0800 Subject: [PATCH 01/18] fix:set error msg if build consumer error --- source/client/src/clientTmq.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index d05cdc0156..15c8903978 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1028,11 +1028,16 @@ static void tmqMgmtInit(void) { } } +#define SET_ERROR_MSG(MSG) if(errstr!=NULL)snprintf(errstr,errstrLen,MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - if(conf == NULL) return NULL; + if(conf == NULL) { + SET_ERROR_MSG("configure is null") + return NULL; + } taosThreadOnce(&tmqInit, tmqMgmtInit); if (tmqInitRes != 0) { terrno = tmqInitRes; + SET_ERROR_MSG("tmq timer init error") return NULL; } @@ -1040,6 +1045,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr()); + SET_ERROR_MSG("malloc tmq failed") return NULL; } @@ -1055,6 +1061,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG("malloc tmq element failed or group is empty") goto _failed; } @@ -1086,6 +1093,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (tsem_init(&pTmq->rspSem, 0, 0) != 0) { tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG("init t_sem failed") goto _failed; } @@ -1094,11 +1102,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq->pTscObj == NULL) { tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); tsem_destroy(&pTmq->rspSem); + SET_ERROR_MSG("init tscObj failed") goto _failed; } pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq); if (pTmq->refId < 0) { + SET_ERROR_MSG("add tscObj ref failed") goto _failed; } From 68cb86602eff35725e39592c054ab22954ee9cb6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 29 Dec 2023 11:37:34 +0800 Subject: [PATCH 02/18] fix: reuse cid with fs eid updated in tsdbFSetRAWWriteFileDataBegin --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbFS2.h | 1 + source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 6 ++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index b2ccc3e859..1d2fd60df2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -805,6 +805,12 @@ int64_t tsdbFSAllocEid(STFileSystem *fs) { return cid; } +void tsdbFSUpdateEid(STFileSystem *fs, int64_t cid) { + taosThreadMutexLock(&fs->tsdb->mutex); + fs->neid = TMAX(fs->neid, cid); + taosThreadMutexUnlock(&fs->tsdb->mutex); +} + int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { int32_t code = 0; int32_t lino; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 714bf5bf16..7099ce8b26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -52,6 +52,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); // txn int64_t tsdbFSAllocEid(STFileSystem *fs); +void tsdbFSUpdateEid(STFileSystem *fs, int64_t cid); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); int32_t tsdbFSEditCommit(STFileSystem *fs); int32_t tsdbFSEditAbort(STFileSystem *fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index 03c12502d5..ddb8e7e27e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -14,6 +14,7 @@ */ #include "tsdbFSetRAW.h" +#include "tsdbFS2.h" // SFSetRAWWriter ================================================== typedef struct SFSetRAWWriter { @@ -76,7 +77,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .szPage = writer->config->szPage, .fid = bHdr->file.fid, .did = writer->config->did, - .cid = writer->config->cid, + .cid = bHdr->file.cid, .level = writer->config->level, .file = @@ -84,7 +85,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .type = bHdr->file.type, .fid = bHdr->file.fid, .did = writer->config->did, - .cid = writer->config->cid, + .cid = bHdr->file.cid, .size = bHdr->file.size, .minVer = bHdr->file.minVer, .maxVer = bHdr->file.maxVer, @@ -94,6 +95,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA }, }; + tsdbFSUpdateEid(config.tsdb->pFS, config.cid); writer->ctx->offset = 0; writer->ctx->file = config.file; From 7abeb5a27a3cd6077d0e4e0c1c659c00e41d3b26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 29 Dec 2023 17:44:10 +0800 Subject: [PATCH 03/18] fix(stream): update the stream error code. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- tests/system-test/8-stream/max_delay_session.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 25f51b85df..57f3ac174c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -792,7 +792,7 @@ static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { / if (pStream->targetStbUid == pStreamObj->targetStbUid) { mError("Cannot write the same stable as other stream:%s", pStream->name); sdbCancelFetch(pMnode->pSdb, pIter); - terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; + terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE; return terrno; } } diff --git a/tests/system-test/8-stream/max_delay_session.py b/tests/system-test/8-stream/max_delay_session.py index 46c4c5801d..7e9d365fc8 100644 --- a/tests/system-test/8-stream/max_delay_session.py +++ b/tests/system-test/8-stream/max_delay_session.py @@ -27,6 +27,8 @@ class TDTestCase: self.tb_stream_des_table = f'{self.tb_name}{self.tdCom.des_table_suffix}' self.tdCom.date_time = self.tdCom.dataDict["start_ts"] + time.sleep(1) + if watermark is not None: watermark_value = f'{self.tdCom.dataDict["watermark"]}s' else: From 1e8244f3c692cf181fa1a29d1c4d4b3a7493fa52 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 10:31:15 +0800 Subject: [PATCH 04/18] feat: add s3 minio ci case --- source/common/src/tglobal.c | 2 +- tests/army/enterprise/s3/s3_basic.json | 58 ++++++++++++++ tests/army/enterprise/s3/s3_basic.py | 105 +++++++++++++++++++++++++ tests/army/frame/caseBase.py | 4 + tests/army/frame/srvCtl.py | 5 ++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 tests/army/enterprise/s3/s3_basic.json create mode 100644 tests/army/enterprise/s3/s3_basic.py diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fdac3882c3..d6d1e1810d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -789,7 +789,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json new file mode 100644 index 0000000000..1c2b9274d2 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"1d", + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 4, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py new file mode 100644 index 0000000000..12c0720112 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.py @@ -0,0 +1,105 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * + +# +# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +# + +''' +s3EndPoint http://192.168.1.52:9000 +s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +s3BucketName ci-bucket +s3UploadDelaySec 60 +''' + + +class TDTestCase(TBase): + updatecfgDict = { + 's3EndPoint': 'http://192.168.1.52:9000', + 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', + 's3BucketName': 'ci-bucket', + 's3UploadDelaySec':'60' + } + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = etool.curFile(__file__, "mlevel_basic.json") + etool.runBenchmark(json=json) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 4 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + def doAction(self): + tdLog.info(f"do action.") + self.flushDb() + self.compactDb() + + # sleep 70s + time.sleep(70) + self.trimDb() + + loop = 0 + while len(sc.dnodeDataFiles()) > 0 and loop < 10: + time.sleep(10) + self.trimDb() + loop += 1 + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + # drop database and free s3 file + self.dropDb() + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 441196f050..6009111d43 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -72,6 +72,10 @@ class TBase: def flushDb(self): tdSql.execute(f"flush database {self.db}") + def dropDb(self): + tdSql.execute(f"drop database {self.db}") + + # # check db correct # diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index e7ef08cde9..99f503d4b6 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -24,4 +24,9 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # return dnode data files list + def dnodeDataFiles(self, idx): + files = [] + return files + sc = srvCtl() \ No newline at end of file From 64164f5684289af8a36ec52a2d41845359536551 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:39:39 +0800 Subject: [PATCH 05/18] fix: run s3_basic.py ok --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 16 ++++++++++++---- tests/army/frame/caseBase.py | 16 ++++++++-------- tests/army/frame/eos.py | 21 +++++++++++++++++++++ tests/army/frame/epath.py | 3 +++ tests/army/frame/srvCtl.py | 12 ++++++++++++ 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 1c2b9274d2..169da290bf 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-12d", + "start_timestamp":"now-15d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 12c0720112..84c91209b3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -17,6 +17,7 @@ import time import taos import frame import frame.etool +import frame.eos from frame.log import * from frame.cases import * @@ -24,6 +25,7 @@ from frame.sql import * from frame.caseBase import * from frame.srvCtl import * from frame import * +from frame.eos import * # # 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa @@ -42,13 +44,16 @@ class TDTestCase(TBase): 's3EndPoint': 'http://192.168.1.52:9000', 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', 's3BucketName': 'ci-bucket', + 's3BlockSize': '10240', + 's3BlockCacheSize': '320', + 's3PageCacheSize': '10240', 's3UploadDelaySec':'60' } def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run - json = etool.curFile(__file__, "mlevel_basic.json") + json = etool.curFile(__file__, "s3_basic.json") etool.runBenchmark(json=json) tdSql.execute(f"use {self.db}") @@ -63,12 +68,15 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s + tdLog.info(f"wait 70s ...") time.sleep(70) self.trimDb() - + + rootPath = sc.clusterRootPath() + cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" loop = 0 - while len(sc.dnodeDataFiles()) > 0 and loop < 10: - time.sleep(10) + while len(eos.runRetList(cmd)) > 0 and loop < 40: + time.sleep(5) self.trimDb() loop += 1 diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 6009111d43..ad6e744fe6 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -63,17 +63,17 @@ class TBase: # db action # - def trimDb(self): - tdSql.execute(f"trim database {self.db}") + def trimDb(self, show = False): + tdSql.execute(f"trim database {self.db}", show = show) - def compactDb(self): - tdSql.execute(f"compact database {self.db}") + def compactDb(self, show = False): + tdSql.execute(f"compact database {self.db}", show = show) - def flushDb(self): - tdSql.execute(f"flush database {self.db}") + def flushDb(self, show = False): + tdSql.execute(f"flush database {self.db}", show = show) - def dropDb(self): - tdSql.execute(f"drop database {self.db}") + def dropDb(self, show = False): + tdSql.execute(f"drop database {self.db}", show = show) # diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index dcb63ff3aa..b24be74393 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -20,11 +20,16 @@ import os import time import datetime import platform +import subprocess # if windows platform return True def isWin(): return platform.system().lower() == 'windows' +# +# execute programe +# + # wait util execute file finished def exe(file): return os.system(file) @@ -34,3 +39,19 @@ def exeNoWait(file): print("exe no wait") +# run return output and error +def run(command): + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait(3) + + output = process.stdout.read().decode(encoding="gbk") + error = process.stderr.read().decode(encoding="gbk") + + return output, error + + +# return list after run +def runRetList(command): + lines = [] + output,error = run(command) + return output.splitlines() diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py index edff9c78a4..c7796befbc 100644 --- a/tests/army/frame/epath.py +++ b/tests/army/frame/epath.py @@ -51,3 +51,6 @@ def binPath(): def binFile(filename): return binPath() + filename + + + diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 99f503d4b6..75f0d79513 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -16,6 +16,8 @@ import os import time import datetime +from frame.server.dnodes import * + class srvCtl: def __init__(self): # record server information @@ -24,9 +26,19 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # + # about path + # + + # get cluster root path like /root/TDinternal/sim/ + def clusterRootPath(self): + return tdDnodes.path + # return dnode data files list def dnodeDataFiles(self, idx): files = [] return files + + sc = srvCtl() \ No newline at end of file From 26023864f14ab5a6b6926bca68a6a43c2cb12a8a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:48:36 +0800 Subject: [PATCH 06/18] fix: add show path information --- tests/army/enterprise/s3/s3_basic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 84c91209b3..85beb195c3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -70,14 +70,15 @@ class TDTestCase(TBase): # sleep 70s tdLog.info(f"wait 70s ...") time.sleep(70) - self.trimDb() + self.trimDb(True) rootPath = sc.clusterRootPath() cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: time.sleep(5) - self.trimDb() + self.trimDb(True) loop += 1 # run From 1600f0654d6c3f46aae1242199aa61f7af223a2a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 15:16:29 +0800 Subject: [PATCH 07/18] fix: s3 for minio finished --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 6 +++--- tests/army/frame/srvCtl.py | 2 +- tests/parallel_test/cases.task | 11 +++++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 169da290bf..e56a18e757 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-15d", + "start_timestamp":"now-13d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 85beb195c3..389b8a52c7 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -68,12 +68,12 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s - tdLog.info(f"wait 70s ...") - time.sleep(70) + tdLog.info(f"wait 65s ...") + time.sleep(65) self.trimDb(True) rootPath = sc.clusterRootPath() - cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 75f0d79513..155c83094d 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -32,7 +32,7 @@ class srvCtl: # get cluster root path like /root/TDinternal/sim/ def clusterRootPath(self): - return tdDnodes.path + return tdDnodes.getDnodesRootDir() # return dnode data files list def dnodeDataFiles(self, idx): diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b5cde02d25..e7395a26f1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -5,10 +5,17 @@ #unit-test ,,y,unit-test,bash test.sh -#army-test +# +# army-test +# ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 -#system test + + +# +# system test +# ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py From 4f56af59c588377d9370fb9e3c46fe10901df6e2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 16:44:52 +0800 Subject: [PATCH 08/18] add -B set bi mode option --- tools/shell/inc/shellInt.h | 1 + tools/shell/src/shellArguments.c | 6 ++++++ tools/shell/src/shellEngine.c | 9 +++++++++ 3 files changed, 16 insertions(+) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 1c885b151c..ba3dadc646 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -66,6 +66,7 @@ typedef struct { char file[PATH_MAX]; char password[TSDB_USET_PASSWORD_LEN]; bool is_gen_auth; + bool is_bi_mode; bool is_raw_time; bool is_version; bool is_dump_config; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 4817b23029..1930a16f46 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -44,6 +44,7 @@ #define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server." #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." +#define SHELL_BI_MODE "Set BI mode" #define SHELL_VERSION "Print program version." #ifdef WEBSOCKET @@ -59,6 +60,7 @@ void shellPrintHelp() { printf("Usage: taos [OPTION...] \r\n\r\n"); printf("%s%s%s%s\r\n", indent, "-a,", indent, SHELL_AUTH); printf("%s%s%s%s\r\n", indent, "-A,", indent, SHELL_GEN_AUTH); + printf("%s%s%s%s\r\n", indent, "-B,", indent, SHELL_BI_MODE); printf("%s%s%s%s\r\n", indent, "-c,", indent, SHELL_CFG_DIR); printf("%s%s%s%s\r\n", indent, "-C,", indent, SHELL_DMP_CFG); printf("%s%s%s%s\r\n", indent, "-d,", indent, SHELL_DB); @@ -127,6 +129,7 @@ static struct argp_option shellOptions[] = { {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, + {"bimode", 'B', "BIMODE", 0, SHELL_BI_MODE}, {0}, }; @@ -173,6 +176,9 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { case 'A': pArgs->is_gen_auth = true; break; + case 'B': + pArgs->is_bi_mode = true; + break; case 'c': #ifdef WEBSOCKET pArgs->cloud = false; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index e8a5b04178..40b0236d07 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1291,6 +1291,15 @@ int32_t shellExecute() { shellSetConn(shell.conn, runOnce); shellReadHistory(); + if(shell.args.is_bi_mode) { + // need set bi mode +#ifdef WEBSOCKET + //ws_taos_set_conn_mode(shell.ws_conn, TAOS_CONN_MODE_BI, 1); +#else + taos_set_conn_mode(shell.conn, TAOS_CONN_MODE_BI, 1); +#endif + } + if (runOnce) { if (pArgs->commands != NULL) { printf("%s%s\r\n", shell.info.promptHeader, pArgs->commands); From 861cb8572adcbd56c72834b3660ece57bdb52ad0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 17:15:08 +0800 Subject: [PATCH 09/18] fix: taos-tools can not build passed --- source/libs/parser/src/parTranslater.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..3030ec5734 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1104,11 +1104,9 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p return DEAL_RES_CONTINUE; } -#ifndef TD_ENTERPRISE bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { return false; } -#endif static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { if (NULL == pCxt->pCurrStmt || @@ -3178,9 +3176,7 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { return TSDB_CODE_SUCCESS; } -#ifndef TD_ENTERPRISE int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } -#endif static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { SNode* pNode = NULL; @@ -5743,11 +5739,9 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions return code; } -#ifndef TD_ENTERPRISE int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { return TSDB_CODE_SUCCESS; } -#endif static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { if (NULL != strchr(pStmt->tableName, '.')) { From c6549b5ff65c7afccd9294f87693d175559a563c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 21:24:58 +0800 Subject: [PATCH 10/18] fix: format tweak --- tools/shell/src/shellArguments.c | 2 +- tools/shell/src/shellEngine.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 1930a16f46..71fa777d63 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -129,7 +129,7 @@ static struct argp_option shellOptions[] = { {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, - {"bimode", 'B', "BIMODE", 0, SHELL_BI_MODE}, + {"bimode", 'B', 0, 0, SHELL_BI_MODE}, {0}, }; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 40b0236d07..60b1dfab41 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1293,6 +1293,7 @@ int32_t shellExecute() { if(shell.args.is_bi_mode) { // need set bi mode + printf("Set BI mode is true.\n"); #ifdef WEBSOCKET //ws_taos_set_conn_mode(shell.ws_conn, TAOS_CONN_MODE_BI, 1); #else From 62b2a5320e6e35d909048e7745aa88490cd4c3c3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 21:29:49 +0800 Subject: [PATCH 11/18] fix: prune assert --- tools/shell/src/shellCommand.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 8c91ff53e2..c6459c4590 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -62,7 +62,6 @@ int32_t shellCountPrefixOnes(uint8_t c) { } void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - ASSERT(pos > 0); if (pos <= 0) return; TdWchar wc; @@ -82,7 +81,6 @@ void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - ASSERT(pos >= 0); if(pos < 0) return; TdWchar wc; @@ -91,7 +89,6 @@ void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; TdWchar wc; @@ -138,7 +135,6 @@ void shellInsertStr(SShellCmd *cmd, char *str, int32_t size) { } void shellBackspaceChar(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { @@ -159,7 +155,6 @@ void shellBackspaceChar(SShellCmd *cmd) { } void shellClearLineBefore(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -174,7 +169,6 @@ void shellClearLineBefore(SShellCmd *cmd) { } void shellClearLineAfter(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -184,7 +178,6 @@ void shellClearLineAfter(SShellCmd *cmd) { } void shellDeleteChar(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { @@ -203,7 +196,6 @@ void shellDeleteChar(SShellCmd *cmd) { } void shellMoveCursorLeft(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { @@ -218,7 +210,6 @@ void shellMoveCursorLeft(SShellCmd *cmd) { } void shellMoveCursorRight(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { @@ -233,7 +224,6 @@ void shellMoveCursorRight(SShellCmd *cmd) { } void shellPositionCursorHome(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { @@ -254,7 +244,6 @@ void positionCursorMiddle(SShellCmd *cmd) { } void shellPositionCursorEnd(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { @@ -290,7 +279,6 @@ void shellPositionCursor(int32_t step, int32_t direction) { } void shellUpdateBuffer(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); @@ -306,7 +294,6 @@ void shellUpdateBuffer(SShellCmd *cmd) { } bool shellIsReadyGo(SShellCmd *cmd) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return false; char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); @@ -334,7 +321,6 @@ void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { } void shellResetCommand(SShellCmd *cmd, const char s[]) { - ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); From 3302e22c791f560035653650bba1cd7a14083322 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 22:39:53 +0800 Subject: [PATCH 12/18] case: create db with wa set small --- tests/system-test/0-others/splitVGroupRep1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index 0b75a3e6e1..f799c9dbd8 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 1 wal_retention_size 1" self.exeDouble(sql) # create super talbe st From fcc8185e452d37bf38e85ca3cf66cb4e5e2aa3d2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 11:35:39 +0800 Subject: [PATCH 13/18] coverage: add snapshot.json case --- tests/army/community/cluster/snapshot.json | 60 +++++++++++++++++ tests/army/community/cluster/snapshot.py | 76 ++++++++++++++++++++++ tests/army/frame/caseBase.py | 5 -- tests/parallel_test/cases.task | 2 +- 4 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 tests/army/community/cluster/snapshot.json create mode 100644 tests/army/community/cluster/snapshot.py diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json new file mode 100644 index 0000000000..56a3630bca --- /dev/null +++ b/tests/army/community/cluster/snapshot.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 3, + "duration":"1d", + "wal_retention_period": 1, + "wal_retention_size": 1, + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 10000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 32} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py new file mode 100644 index 0000000000..984ea0bece --- /dev/null +++ b/tests/army/community/cluster/snapshot.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + jfile = etool.curFile(__file__, "snapshot.json") + etool.runBenchmark(json=jfile) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 10 + self.insert_rows = 100000 + self.timestamp_step = 10000 + + def doAction(self): + tdLog.info(f"do action.") + self.flushDb() + self.trimDb() + self.compactDb() + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index ad6e744fe6..ece3d2ccc3 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -42,11 +42,6 @@ class TBase: self.db = "db" self.stb = "stb" - # variant in taosBenchmark json - self.childtable_count = 2 - self.insert_rows = 1000000 - self.timestamp_step = 1000 - # sql self.sqlSum = f"select sum(ic) from {self.stb}" self.sqlMax = f"select max(ic) from {self.stb}" diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e7395a26f1..093fec78ab 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,7 +10,7 @@ # ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 - +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 # From aed127ab0b92e4b99773ff6315717b573e569a20 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 16:53:57 +0800 Subject: [PATCH 14/18] coverage: snapshot add balance operator --- tests/army/community/cluster/snapshot.json | 2 +- tests/army/community/cluster/snapshot.py | 16 ++++ tests/army/frame/caseBase.py | 91 +++++++++++++++++++++- tools/shell/src/shellAuto.c | 8 +- 4 files changed, 109 insertions(+), 8 deletions(-) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json index 56a3630bca..64bb2aaf3c 100644 --- a/tests/army/community/cluster/snapshot.json +++ b/tests/army/community/cluster/snapshot.json @@ -36,7 +36,7 @@ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, { "type": "double", "name": "dc"}, - { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, { "type": "int", "name": "ic" }, { "type": "bigint", "name": "bi" }, diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 984ea0bece..40783ad4ce 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -13,11 +13,13 @@ import sys import time +import random import taos import frame import frame.etool + from frame.log import * from frame.cases import * from frame.sql import * @@ -43,7 +45,21 @@ class TDTestCase(TBase): def doAction(self): tdLog.info(f"do action.") self.flushDb() + + # split vgroups + self.splitVGroups() self.trimDb() + + # balance vgroups + self.balanceVGroupLeader() + + # replica to 1 + self.alterReplica(1) + + vgids = self.getVGroup() + selid = random.choice(vgids) + self.balanceVGroupLeaderOn(selid) + self.compactDb() # run diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index ece3d2ccc3..05ccf975a7 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +import random from frame.log import * from frame.sql import * @@ -47,7 +48,8 @@ class TBase: self.sqlMax = f"select max(ic) from {self.stb}" self.sqlMin = f"select min(ic) from {self.stb}" self.sqlAvg = f"select avg(ic) from {self.stb}" - + self.sqlFirst = f"select first(ts) from {self.stb}" + self.sqlLast = f"select last(ts) from {self.stb}" # stop def stop(self): @@ -70,6 +72,50 @@ class TBase: def dropDb(self, show = False): tdSql.execute(f"drop database {self.db}", show = show) + def splitVGroups(self): + vgids = self.getVGroup(self.db) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + + def alterReplica(self, replica): + sql = f"alter database {self.db} replica {replica}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + def balanceVGroup(self): + sql = f"balance vgroup" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + def balanceVGroupLeader(self): + sql = f"balance vgroup leader" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + + def balanceVGroupLeaderOn(self, vgId): + sql = f"balance vgroup leader on {vgId}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + # # check db correct @@ -90,12 +136,13 @@ class TBase: tdSql.checkAgg(sql, 0) # save agg result - def snapshotAgg(self): - + def snapshotAgg(self): self.sum = tdSql.getFirstValue(self.sqlSum) self.avg = tdSql.getFirstValue(self.sqlAvg) self.min = tdSql.getFirstValue(self.sqlMin) self.max = tdSql.getFirstValue(self.sqlMax) + self.first = tdSql.getFirstValue(self.sqlFirst) + self.last = tdSql.getFirstValue(self.sqlLast) # check agg def checkAggCorrect(self): @@ -103,3 +150,41 @@ class TBase: tdSql.checkAgg(self.sqlAvg, self.avg) tdSql.checkAgg(self.sqlMin, self.min) tdSql.checkAgg(self.sqlMax, self.max) + tdSql.checkAgg(self.sqlFirst, self.first) + tdSql.checkAgg(self.sqlLast, self.last) + + +# +# get db information +# + + # get vgroups + def getVGroup(self, db_name): + vgidList = [] + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + res = tdSql.getResult(sql) + rows = len(res) + for i in range(rows): + vgidList.append(res[i][0]) + + return vgidList + + + +# +# util +# + + # wait transactions count to zero , return False is translation not finished + def waitTransactionZero(self, seconds = 300, interval = 1): + # wait end + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("split vgroup finished.") + return True + #tdLog.info(f"i={i} wait ...") + time.sleep(interval) + + return False diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index bd5329d810..e9b9b9e944 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -84,8 +84,8 @@ SWords shellCommands[] = { {"alter topic", 0, 0, NULL}, {"alter user ;", 0, 0, NULL}, #ifdef TD_ENTERPRISE - {"balance vgroup;", 0, 0, NULL}, - {"balance vgroup leader ", 0, 0, NULL}, + {"balance vgroup ;", 0, 0, NULL}, + {"balance vgroup leader on ", 0, 0, NULL}, #endif // 20 @@ -531,8 +531,8 @@ void showHelp() { printf( "\n\n\ ----- special commands on enterpise version ----- \n\ - balance vgroup; \n\ - balance vgroup leader \n\ + balance vgroup ;\n\ + balance vgroup leader on \n\ compact database ; \n\ redistribute vgroup dnode ;\n\ split vgroup ;"); From d318c8f9831db04533bd2e4821e49ffe60d29aa5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:04:18 +0800 Subject: [PATCH 15/18] fix: add data correct check point --- tests/army/community/cluster/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 40783ad4ce..66f9d9f598 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -49,14 +49,16 @@ class TDTestCase(TBase): # split vgroups self.splitVGroups() self.trimDb() + self.checkAggCorrect() # balance vgroups self.balanceVGroupLeader() # replica to 1 self.alterReplica(1) + self.checkAggCorrect() - vgids = self.getVGroup() + vgids = self.getVGroup(self.db) selid = random.choice(vgids) self.balanceVGroupLeaderOn(selid) From 6dba2a7aefe52a1dfd9529b17002670f7e1b4570 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:11:47 +0800 Subject: [PATCH 16/18] fix: add replica to 3 --- tests/army/community/cluster/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 66f9d9f598..b177ba3b29 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -57,12 +57,14 @@ class TDTestCase(TBase): # replica to 1 self.alterReplica(1) self.checkAggCorrect() + self.compactDb() + self.alterReplica(3) vgids = self.getVGroup(self.db) selid = random.choice(vgids) self.balanceVGroupLeaderOn(selid) - self.compactDb() + # run def run(self): From 902529352dd361b2d9625ec52e3057d1ea10c36a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:16:54 +0800 Subject: [PATCH 17/18] fix: transaction tips --- tests/army/frame/caseBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 05ccf975a7..8d2c1e6d18 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -182,7 +182,7 @@ class TBase: sql ="show transactions;" rows = tdSql.query(sql) if rows == 0: - tdLog.info("split vgroup finished.") + tdLog.info("transaction count became zero.") return True #tdLog.info(f"i={i} wait ...") time.sleep(interval) From 28238fa7dc6792f426567836597d2edfbab6fc53 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 18:29:37 +0800 Subject: [PATCH 18/18] coverage: no call function with BUILD_NO_CALL MACRO define --- source/libs/stream/src/streamBackendRocksdb.c | 28 +++++++++++++++++-- source/libs/stream/src/streamMeta.c | 2 ++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5485c13616..a5a4bd4dc1 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -665,6 +665,7 @@ void streamBackendHandleCleanup(void* arg) { return; } +#ifdef BUILD_NO_CALL int32_t getLatestCheckpoint(void* arg, int64_t* checkpoint) { SStreamMeta* pMeta = arg; taosWLockLatch(&pMeta->chkpDirLock); @@ -738,6 +739,7 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) { taosArrayDestroy(chkpDel); return 0; } +#endif /* * checkpointSave |--cp1--|--cp2--|--cp3--|--cp4--|--cp5--| * chkpInUse: |--cp2--|--cp4--| @@ -855,6 +857,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { return 0; } +#ifdef BUILD_NO_CALL int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { return 0; // SArray* pHandle = taosArrayInit(16, POINTER_BYTES); @@ -891,6 +894,7 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* // *ppHandle = ppCf; // return nCf; } +#endif int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { SArray* pHandle = taosArrayInit(8, POINTER_BYTES); @@ -1006,6 +1010,7 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { return code; } +#ifdef BUILD_NO_CALL int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { // if (arg == NULL) return 0; @@ -1029,6 +1034,7 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId) { // } // taosWUnLockLatch(&pMeta->chkpDirLock); } +#endif /* 0 @@ -1099,7 +1105,9 @@ void streamBackendDelCompare(void* backend, void* arg) { taosMemoryFree(node); } } +#ifdef BUILD_NO_CALL void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } +#endif void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); if (inst->pHandle) { @@ -2170,6 +2178,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t taosMemoryFree(cfOpts); return 0; } +#ifdef BUILD_NO_CALL int streamStateOpenBackend(void* backend, SStreamState* pState) { taosAcquireRef(streamBackendId, pState->streamBackendRid); SBackendWrapper* handle = backend; @@ -2279,6 +2288,7 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { wrapper->remove |= remove; // update by other pState taosReleaseRef(streamBackendCfWrapperId, pState->pTdbState->backendCfWrapperId); } +#endif void streamStateDestroyCompar(void* arg) { SCfComparator* comp = (SCfComparator*)arg; for (int i = 0; i < comp->numOfComp; i++) { @@ -2669,7 +2679,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { STREAM_STATE_DEL_ROCKSDB(pState, "state", &maxStateKey); return pCur; } - +#ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -2719,6 +2729,7 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) { STREAM_STATE_DEL_ROCKSDB(pState, "func", key); return 0; } +#endif // session cf int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { @@ -3128,6 +3139,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const streamStateFreeCur(pCur); return NULL; } +#ifdef BUILD_NO_CALL int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { stDebug("streamStateSessionGetKeyByRange_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -3185,6 +3197,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes streamStateFreeCur(pCur); return -1; } +#endif int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen) { @@ -3317,6 +3330,7 @@ _end: return res; } +#ifdef BUILD_NO_CALL // partag cf int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen) { int code = 0; @@ -3329,6 +3343,7 @@ int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void STREAM_STATE_GET_ROCKSDB(pState, "partag", &groupId, tagVal, tagLen); return code; } +#endif // parname cfg int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { int code = 0; @@ -3342,11 +3357,13 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi return code; } +#ifdef BUILD_NO_CALL int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } +#endif int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); @@ -3400,6 +3417,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co rocksdb_iter_destroy(pIter); return code; } +#ifdef BUILD_NO_CALL void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -3443,6 +3461,7 @@ char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) { return ret; } +#endif // batch func void* streamStateCreateBatch() { rocksdb_writebatch_t* pBatch = rocksdb_writebatch_create(); @@ -3796,11 +3815,12 @@ void dbChkpDestroy(SDbChkp* pChkp) { taosMemoryFree(pChkp->pManifest); taosMemoryFree(pChkp); } - +#ifdef BUILD_NO_CALL int32_t dbChkpInit(SDbChkp* p) { if (p == NULL) return 0; return 0; } +#endif int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { taosThreadRwlockRdlock(&p->rwLock); int32_t code = -1; @@ -3945,6 +3965,7 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, return code; } +#ifdef BUILD_NO_CALL int32_t bkdMgtAddChkp(SBkdMgt* bm, char* task, char* path) { int32_t code = -1; @@ -3974,4 +3995,5 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) { taosThreadRwlockUnlock(&bm->rwLock); return code; -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d09d370a36..a658c46c28 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -419,6 +419,7 @@ _err: } // todo refactor: the lock shoud be restricted in one function +#ifdef BUILD_NO_CALL void streamMetaInitBackend(SStreamMeta* pMeta) { pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); if (pMeta->streamBackend == NULL) { @@ -440,6 +441,7 @@ void streamMetaInitBackend(SStreamMeta* pMeta) { pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); streamBackendLoadCheckpointInfo(pMeta); } +#endif void streamMetaClear(SStreamMeta* pMeta) { // remove all existed tasks in this vnode