Merge branch 'feature/2.0tsdb' of github.com:taosdata/TDengine into feature/2.0tsdb

src/inc/tsdb.h

@@ -140,7 +140,7 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid);
  *
  * @return the number of points inserted, -1 for failure and the error number is set
  */
-int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg);
+int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ;
 
 // -- FOR QUERY TIME SERIES DATA
 

src/tsdb/inc/tsdbMain.h

@@ -29,20 +29,20 @@ extern "C" {
 
 extern int tsdbDebugFlag;
 
-#define tsdbError(...)                                       \
+#define tsdbError(...)                                      \
-  if (tsdbDebugFlag & DEBUG_ERROR) {                         \
+  if (tsdbDebugFlag & DEBUG_ERROR) {                        \
-    taosPrintLog("ERROR TSDB ", tsdbDebugFlag, __VA_ARGS__); \
+    taosPrintLog("ERROR TDB ", tsdbDebugFlag, __VA_ARGS__); \
   }
-#define tsdbWarn(...)                                       \
+#define tsdbWarn(...)                                      \
-  if (tsdbDebugFlag & DEBUG_WARN) {                         \
+  if (tsdbDebugFlag & DEBUG_WARN) {                        \
-    taosPrintLog("WARN TSDB ", tsdbDebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN TDB ", tsdbDebugFlag, __VA_ARGS__); \
   }
-#define tsdbTrace(...)                                 \
+#define tsdbTrace(...)                                \
-  if (tsdbDebugFlag & DEBUG_TRACE) {                   \
+  if (tsdbDebugFlag & DEBUG_TRACE) {                  \
-    taosPrintLog("TSDB ", tsdbDebugFlag, __VA_ARGS__); \
+    taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); \
   }
 #define tsdbPrint(...) \
-  { taosPrintLog("TSDB ", 255, __VA_ARGS__); }
+  { taosPrintLog("TDB ", 255, __VA_ARGS__); }
 
 // ------------------------------ TSDB META FILE INTERFACES ------------------------------
 #define TSDB_META_FILE_NAME "meta"

src/tsdb/src/tsdbMain.c

@@ -29,7 +29,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
 static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
 static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
 // static int     tsdbOpenMetaFile(char *tsdbDir);
-static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now);
+static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows);
 static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg);
 static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname);
 static void *  tsdbCommitData(void *arg);
@@ -406,22 +406,23 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) {
 }
 
 // TODO: need to return the number of data inserted
-int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg) {
+int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) {
   SSubmitMsgIter msgIter;
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   tsdbInitSubmitMsgIter(pMsg, &msgIter);
   SSubmitBlk *pBlock = NULL;
   int32_t code = TSDB_CODE_SUCCESS;
+  int32_t affectedrows = 0;
 
   TSKEY now = taosGetTimestamp(pRepo->config.precision);
 
   while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) {
-    if ((code = tsdbInsertDataToTable(repo, pBlock, now)) != TSDB_CODE_SUCCESS) {
+    if ((code = tsdbInsertDataToTable(repo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) {
       return code;
     }
   }
-
+  pRsp->affectedRows = htonl(affectedrows);
   return code;
 }
 
@@ -846,7 +847,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   return 0;
 }
 
-static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now) {
+static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
@@ -875,6 +876,7 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
     if (tdInsertRowToTable(pRepo, row, pTable) < 0) {
       return -1;
     }
+     (*affectedrows)++;
   }
 
   return TSDB_CODE_SUCCESS;

src/vnode/src/vnodeWrite.c

@@ -91,16 +91,15 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR
   // save insert result into item
 
   vTrace("vgId:%d, submit msg is processed", pVnode->vgId);
-  code = tsdbInsertData(pVnode->tsdb, pCont);
   
   pRet->len = sizeof(SShellSubmitRspMsg);
   pRet->rsp = rpcMallocCont(pRet->len);
   SShellSubmitRspMsg *pRsp = pRet->rsp;
-
+  code = tsdbInsertData(pVnode->tsdb, pCont, pRsp);
+  pRsp->numOfFailedBlocks = 0; //TODO
+  //pRet->len += pRsp->numOfFailedBlocks * sizeof(SShellSubmitRspBlock); //TODO
   pRsp->code              = 0;
   pRsp->numOfRows         = htonl(1);
-  pRsp->affectedRows      = htonl(1);
-  pRsp->numOfFailedBlocks = 0;
   
   return code;
 }

tests/pytest/query/query.py

@@ -0,0 +1,53 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))")
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags("dev_01")')
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags("dev_02")')
+
+        print("==============step2")
+
+        tdSql.execute(
+            """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), 
+            ('2020-05-13 10:00:00.001', 1)
+             dev_002 VALUES('2020-05-13 10:00:00.001', 1)""")
+
+        tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'")
+        tdSql.checkRows(1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/stable/insert.py

@@ -0,0 +1,56 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))")
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags("dev_01")')
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags("dev_02")')
+
+        print("==============step2")
+
+        tdSql.execute(
+            """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), 
+            ('2020-05-13 10:00:00.001', 1)
+             dev_002 VALUES('2020-05-13 10:00:00.001', 1)""")
+
+        tdSql.query("select * from db.st where dev='dev_01'")
+        tdSql.checkRows(2)
+
+        tdSql.query("select * from db.st where dev='dev_02'")
+        tdSql.checkRows(1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/util/dnodes.py

@@ -38,9 +38,9 @@ class TDSimClient:
             tdLog.exit(cmd)
 
     def deploy(self):
-        self.logDir = "%s/sim/psim/log" % (self.path,)
+        self.logDir = "%s/pysim/psim/log" % (self.path,)
-        self.cfgDir = "%s/sim/psim/cfg" % (self.path)
+        self.cfgDir = "%s/pysim/psim/cfg" % (self.path)
-        self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path)
+        self.cfgPath = "%s/pysim/psim/cfg/taos.cfg" % (self.path)
 
         cmd = "rm -rf " + self.logDir
         if os.system(cmd) != 0:
@@ -100,10 +100,10 @@ class TDDnode:
         self.valgrind = value
 
     def deploy(self):
-        self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index)
+        self.logDir = "%s/pysim/dnode%d/log" % (self.path, self.index)
-        self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index)
+        self.dataDir = "%s/pysim/dnode%d/data" % (self.path, self.index)
-        self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index)
+        self.cfgDir = "%s/pysim/dnode%d/cfg" % (self.path, self.index)
-        self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (self.path, self.index)
+        self.cfgPath = "%s/pysim/dnode%d/cfg/taos.cfg" % (self.path, self.index)
 
         cmd = "rm -rf " + self.dataDir
         if os.system(cmd) != 0:
@@ -177,21 +177,42 @@ class TDDnode:
             (self.index, self.cfgPath))
 
     def start(self):
-        binPath = os.path.dirname(os.path.realpath(__file__))
+        selfPath = os.path.dirname(os.path.realpath(__file__))
-        binPath = binPath + "/../../../debug/"
+        binPath = ""
-        binPath = os.path.realpath(binPath)
+
-        binPath += "/build/bin/"
+        if ("TDinternal" in selfPath):
+            projPath = selfPath + "/../../../../"
+
+            for root, dirs, files in os.walk(projPath):
+                if ("taosd" in files):
+                    rootRealPath = os.path.dirname(os.path.realpath(root))
+                    if ("community" not in rootRealPath):
+                        binPath = os.path.join(root, "taosd")
+                        break;
+        else:
+            projPath = selfPath + "/../../../"
+            for root, dirs, files in os.walk(projPath):
+                if ("taosd" in files):
+                    rootRealPath = os.path.dirname(os.path.realpath(root))
+                    if ("packaging" not in rootRealPath):
+                        binPath = os.path.join(root, "taosd")
+                        break;
+
+        if (binPath == ""):
+            tdLog.exit("taosd not found!s")
+        else:
+            tdLog.notice("taosd found in %s" % rootRealPath)
 
         if self.deployed == 0:
             tdLog.exit("dnode:%d is not deployed" % (self.index))
 
         if self.valgrind == 0:
-            cmd = "nohup %staosd -c %s > /dev/null 2>&1 & " % (
+            cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
                 binPath, self.cfgDir)
         else:
             valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
 
-            cmd = "nohup %s %staosd -c %s 2>&1 & " % (
+            cmd = "nohup %s %s -c %s 2>&1 & " % (
                 valgrindCmdline, binPath, self.cfgDir)
 
             print(cmd)
@@ -268,11 +289,11 @@ class TDDnode:
             tdLog.exit(cmd)
 
     def getDnodeRootDir(self, index):
-        dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index)
+        dnodeRootDir = "%s/pysim/psim/dnode%d" % (self.path, index)
         return dnodeRootDir
 
     def getDnodesRootDir(self):
-        dnodesRootDir = "%s/sim/psim" % (self.path)
+        dnodesRootDir = "%s/pysim/psim" % (self.path)
         return dnodesRootDir
 
 
@@ -417,7 +438,7 @@ class TDDnodes:
         # tdLog.exit(cmd)
 
     def getDnodesRootDir(self):
-        dnodesRootDir = "%s/sim" % (self.path)
+        dnodesRootDir = "%s/pysim" % (self.path)
         return dnodesRootDir
 
     def getSimCfgPath(self):

tests/script/general/db/delete.sim

@@ -20,6 +20,12 @@ while $i < 2000
   $i = $i + 1
 endw 
 
+sql show db.vgroups
+if $rows != 2 then 
+  return -1
+endi 
+
+return
 print ======== step2
 sleep 1000
 sql drop database db

tests/script/jenkins/basic.txt

@@ -49,7 +49,7 @@ cd ../../../debug; make
 ./test.sh -f general/db/basic3.sim
 ./test.sh -f general/db/basic4.sim
 ./test.sh -f general/db/basic5.sim
-./test.sh -f unique/db/delete.sim
+./test.sh -f general/db/delete.sim
 ./test.sh -f general/db/delete_reuse1.sim
 ./test.sh -f general/db/delete_reuse2.sim
 ./test.sh -f general/db/delete_reusevnode.sim

tests/script/unique/db/delete.sim

@@ -34,6 +34,11 @@ while $i < 2000
   $i = $i + 1
 endw 
 
+sql show db.vgroups
+if $rows != 2 then 
+  return -1
+endi 
+
 print ======== step2
 sleep 1000
 sql drop database db
@@ -59,14 +64,14 @@ endi
 
 print ======== step3
 
-system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode1 -s stop  -x SIGINT
-system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode2 -s stop  -x SIGINT
-system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode3 -s stop  -x SIGINT
 
 sleep 1000
-system sh/exec.sh -n dnode1 -s start -t
+system sh/exec_up.sh -n dnode1 -s start
-system sh/exec.sh -n dnode2 -s start -t
+system sh/exec_up.sh -n dnode2 -s start
-system sh/exec.sh -n dnode3 -s start -t
+system sh/exec_up.sh -n dnode3 -s start
 
 $x = 0
 step3: 

tests/script/unique/db/delete_part.sim

@@ -31,17 +31,17 @@ system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
 
 print ========= start dnodes
 system sh/exec_up.sh -n dnode1 -s start
+sleep 3000
 sql connect
 sql create dnode $hostname2
 system sh/exec_up.sh -n dnode2 -s start
-sleep 3000
 
 $loop = 0
 begin:
 
     $db = db . $loop
 	
-	print ======== step1
+	print ======== step1 $loop
 	sql create database $db
 	sql use $db
 	
@@ -53,15 +53,16 @@ begin:
 	  $x = $x + 1
 	endw
 
-	print ======== step2
+	print ======== step2 $loop
 	system sh/exec_up.sh -n dnode2 -s stop
+	sleep 1000
 	sql drop database $db 
 
 	
-	print ======== step3
+	print ======== step3 $loop
 	sleep 3000
 	system sh/exec_up.sh -n dnode2 -s start
-	sleep 20000
+	sleep 5000
 	
 	print ===> test times : $loop
     if $loop > 5 then	  
@@ -70,6 +71,9 @@ begin:
 
 	$loop = $loop + 1
 
+	sql reset query cache
+	sleep 1000
+	
 goto begin
   
 system sh/exec_up.sh -n dnode1 -s stop  -x SIGINT