From 8ff091de25affc3ef52d488cb5f3cc9cd2a485df Mon Sep 17 00:00:00 2001 From: Tao Liu Date: Thu, 14 May 2020 10:19:29 +0000 Subject: [PATCH 1/5] [TD-324] add affected rows in the insert cmd result --- src/inc/tsdb.h | 2 +- src/tsdb/src/tsdbMain.c | 12 +++++++----- src/vnode/src/vnodeWrite.c | 11 +++++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 32e3541692..ee851ca064 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -140,7 +140,7 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid); * * @return the number of points inserted, -1 for failure and the error number is set */ -int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg); +int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ; // -- FOR QUERY TIME SERIES DATA diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c85f5cc838..f1bddadabe 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -29,7 +29,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo); static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo); // static int tsdbOpenMetaFile(char *tsdbDir); -static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now); +static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); @@ -406,22 +406,23 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) { } // TODO: need to return the number of data inserted -int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg) { +int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) { SSubmitMsgIter msgIter; STsdbRepo *pRepo = (STsdbRepo *)repo; tsdbInitSubmitMsgIter(pMsg, &msgIter); SSubmitBlk *pBlock = NULL; int32_t code = TSDB_CODE_SUCCESS; + int32_t affectedrows = 0; TSKEY now = taosGetTimestamp(pRepo->config.precision); while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) { - if ((code = tsdbInsertDataToTable(repo, pBlock, now)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbInsertDataToTable(repo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) { return code; } } - + pRsp->affectedRows = htonl(affectedrows); return code; } @@ -846,7 +847,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable return 0; } -static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now) { +static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) { STsdbRepo *pRepo = (STsdbRepo *)repo; STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; @@ -875,6 +876,7 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY if (tdInsertRowToTable(pRepo, row, pTable) < 0) { return -1; } + (*affectedrows)++; } return TSDB_CODE_SUCCESS; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index ec0a3b2f0b..635c466978 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -91,17 +91,16 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR // save insert result into item vTrace("vgId:%d, submit msg is processed", pVnode->vgId); - code = tsdbInsertData(pVnode->tsdb, pCont); - + pRet->len = sizeof(SShellSubmitRspMsg); pRet->rsp = rpcMallocCont(pRet->len); SShellSubmitRspMsg *pRsp = pRet->rsp; - + code = tsdbInsertData(pVnode->tsdb, pCont, pRsp); + pRsp->numOfFailedBlocks = 0; //TODO + //pRet->len += pRsp->numOfFailedBlocks * sizeof(SShellSubmitRspBlock); //TODO pRsp->code = 0; pRsp->numOfRows = htonl(1); - pRsp->affectedRows = htonl(1); - pRsp->numOfFailedBlocks = 0; - + return code; } From f45bf7705f9b0c83c683f7a911d039012309dbe8 Mon Sep 17 00:00:00 2001 From: Shuaiqiang Chang Date: Thu, 14 May 2020 18:47:57 +0800 Subject: [PATCH 2/5] test: add stable/insert.py;query/query.py --- tests/pytest/query/query.py | 53 +++++++++++++++++++++++++++++++ tests/pytest/stable/__init__.py | 0 tests/pytest/stable/insert.py | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 tests/pytest/query/query.py create mode 100644 tests/pytest/stable/__init__.py create mode 100644 tests/pytest/stable/insert.py diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py new file mode 100644 index 0000000000..a906943247 --- /dev/null +++ b/tests/pytest/query/query.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev_02")') + + print("==============step2") + + tdSql.execute( + """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_002 VALUES('2020-05-13 10:00:00.001', 1)""") + + tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stable/__init__.py b/tests/pytest/stable/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/stable/insert.py b/tests/pytest/stable/insert.py new file mode 100644 index 0000000000..4f41d4aa93 --- /dev/null +++ b/tests/pytest/stable/insert.py @@ -0,0 +1,56 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev_02")') + + print("==============step2") + + tdSql.execute( + """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_002 VALUES('2020-05-13 10:00:00.001', 1)""") + + tdSql.query("select * from db.st where dev='dev_01'") + tdSql.checkRows(2) + + tdSql.query("select * from db.st where dev='dev_02'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 24058f1d357185051912c6a7e154bbffeae55261 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 14 May 2020 19:05:20 +0800 Subject: [PATCH 3/5] scripts --- tests/script/general/db/delete.sim | 6 ++++++ tests/script/jenkins/basic.txt | 2 +- tests/script/unique/db/delete.sim | 17 +++++++++++------ tests/script/unique/db/delete_part.sim | 14 +++++++++----- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/script/general/db/delete.sim b/tests/script/general/db/delete.sim index f86774c2cf..f95676088b 100644 --- a/tests/script/general/db/delete.sim +++ b/tests/script/general/db/delete.sim @@ -20,6 +20,12 @@ while $i < 2000 $i = $i + 1 endw +sql show db.vgroups +if $rows != 2 then + return -1 +endi + +return print ======== step2 sleep 1000 sql drop database db diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 94c9c71edc..b86113eb51 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -49,7 +49,7 @@ cd ../../../debug; make ./test.sh -f general/db/basic3.sim ./test.sh -f general/db/basic4.sim ./test.sh -f general/db/basic5.sim -./test.sh -f unique/db/delete.sim +./test.sh -f general/db/delete.sim ./test.sh -f general/db/delete_reuse1.sim ./test.sh -f general/db/delete_reuse2.sim ./test.sh -f general/db/delete_reusevnode.sim diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index dcdd1ae9e9..0ae6a3c13c 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -34,6 +34,11 @@ while $i < 2000 $i = $i + 1 endw +sql show db.vgroups +if $rows != 2 then + return -1 +endi + print ======== step2 sleep 1000 sql drop database db @@ -59,14 +64,14 @@ endi print ======== step3 -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec_up.sh -n dnode1 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sleep 1000 -system sh/exec.sh -n dnode1 -s start -t -system sh/exec.sh -n dnode2 -s start -t -system sh/exec.sh -n dnode3 -s start -t +system sh/exec_up.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode3 -s start $x = 0 step3: diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index 3d1cc5dc63..7cf326e191 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -31,17 +31,17 @@ system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 print ========= start dnodes system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect sql create dnode $hostname2 system sh/exec_up.sh -n dnode2 -s start -sleep 3000 $loop = 0 begin: $db = db . $loop - print ======== step1 + print ======== step1 $loop sql create database $db sql use $db @@ -53,15 +53,16 @@ begin: $x = $x + 1 endw - print ======== step2 + print ======== step2 $loop system sh/exec_up.sh -n dnode2 -s stop + sleep 1000 sql drop database $db - print ======== step3 + print ======== step3 $loop sleep 3000 system sh/exec_up.sh -n dnode2 -s start - sleep 20000 + sleep 5000 print ===> test times : $loop if $loop > 5 then @@ -69,6 +70,9 @@ begin: endi $loop = $loop + 1 + + sql reset query cache + sleep 1000 goto begin From b814a46320eae4579264570e0114d2392a015135 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 14 May 2020 19:43:14 +0800 Subject: [PATCH 4/5] make python test framework detect TDinternal and TDengine automatically. [TD-328] --- tests/pytest/util/dnodes.py | 53 ++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 23adab2c47..b641927c9f 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -38,9 +38,9 @@ class TDSimClient: tdLog.exit(cmd) def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path,) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + self.logDir = "%s/pysim/psim/log" % (self.path,) + self.cfgDir = "%s/pysim/psim/cfg" % (self.path) + self.cfgPath = "%s/pysim/psim/cfg/taos.cfg" % (self.path) cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -100,10 +100,10 @@ class TDDnode: self.valgrind = value def deploy(self): - self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) - self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) - self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) - self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (self.path, self.index) + self.logDir = "%s/pysim/dnode%d/log" % (self.path, self.index) + self.dataDir = "%s/pysim/dnode%d/data" % (self.path, self.index) + self.cfgDir = "%s/pysim/dnode%d/cfg" % (self.path, self.index) + self.cfgPath = "%s/pysim/dnode%d/cfg/taos.cfg" % (self.path, self.index) cmd = "rm -rf " + self.dataDir if os.system(cmd) != 0: @@ -177,21 +177,42 @@ class TDDnode: (self.index, self.cfgPath)) def start(self): - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - binPath = os.path.realpath(binPath) - binPath += "/build/bin/" + selfPath = os.path.dirname(os.path.realpath(__file__)) + binPath = "" + + if ("TDinternal" in selfPath): + projPath = selfPath + "/../../../../" + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("community" not in rootRealPath): + binPath = os.path.join(root, "taosd") + break; + else: + projPath = selfPath + "/../../../" + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + binPath = os.path.join(root, "taosd") + break; + + if (binPath == ""): + tdLog.exit("taosd not found!s") + else: + tdLog.notice("taosd found in %s" % rootRealPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) if self.valgrind == 0: - cmd = "nohup %staosd -c %s > /dev/null 2>&1 & " % ( + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - cmd = "nohup %s %staosd -c %s 2>&1 & " % ( + cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) print(cmd) @@ -268,11 +289,11 @@ class TDDnode: tdLog.exit(cmd) def getDnodeRootDir(self, index): - dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + dnodeRootDir = "%s/pysim/psim/dnode%d" % (self.path, index) return dnodeRootDir def getDnodesRootDir(self): - dnodesRootDir = "%s/sim/psim" % (self.path) + dnodesRootDir = "%s/pysim/psim" % (self.path) return dnodesRootDir @@ -417,7 +438,7 @@ class TDDnodes: # tdLog.exit(cmd) def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) + dnodesRootDir = "%s/pysim" % (self.path) return dnodesRootDir def getSimCfgPath(self): From f8c6980cfca83bb3a8ad35979d2698212a8b1663 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 15 May 2020 03:28:46 +0000 Subject: [PATCH 5/5] change tsdb log format --- src/tsdb/inc/tsdbMain.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 6b7a77c798..e9714285f0 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -29,20 +29,20 @@ extern "C" { extern int tsdbDebugFlag; -#define tsdbError(...) \ - if (tsdbDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("ERROR TSDB ", tsdbDebugFlag, __VA_ARGS__); \ +#define tsdbError(...) \ + if (tsdbDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("ERROR TDB ", tsdbDebugFlag, __VA_ARGS__); \ } -#define tsdbWarn(...) \ - if (tsdbDebugFlag & DEBUG_WARN) { \ - taosPrintLog("WARN TSDB ", tsdbDebugFlag, __VA_ARGS__); \ +#define tsdbWarn(...) \ + if (tsdbDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WARN TDB ", tsdbDebugFlag, __VA_ARGS__); \ } -#define tsdbTrace(...) \ - if (tsdbDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("TSDB ", tsdbDebugFlag, __VA_ARGS__); \ +#define tsdbTrace(...) \ + if (tsdbDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); \ } #define tsdbPrint(...) \ - { taosPrintLog("TSDB ", 255, __VA_ARGS__); } + { taosPrintLog("TDB ", 255, __VA_ARGS__); } // ------------------------------ TSDB META FILE INTERFACES ------------------------------ #define TSDB_META_FILE_NAME "META"