Merge branch '3.0' of https://github.com/taosdata/TDengine into fix/tsdb_merge_bug

cmake/cmake.define

@@ -97,10 +97,10 @@ ELSE ()
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
     ENDIF ()
 
-    IF (${SANITIZER} MATCHES "true")
+    IF (${BUILD_SANITIZER})
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
-	MESSAGE(STATUS "Will compile with Address Sanitizer!")
+	      MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")

cmake/cmake.options

@@ -77,6 +77,12 @@ ELSE ()
     ENDIF ()
 ENDIF ()
 
+option(
+    BUILD_SANITIZER
+    "If build addr2line"
+    OFF
+    )
+
 option(
     BUILD_ADDR2LINE
     "If build addr2line"

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -1940,6 +1940,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo*
   }
 
   ASSERT(0);
+  return -1;
 }
 
 static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo,

source/libs/transport/src/tmsgcb.c

@@ -16,20 +16,33 @@
 #define _DEFAULT_SOURCE
 #include "tmsgcb.h"
 #include "taoserror.h"
+#include "trpc.h"
 
 static SMsgCb defaultMsgCb;
 
 void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; }
 
 int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) {
-  return (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg);
+  int32_t code = (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg);
+  if (code != 0) {
+    rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
+  }
+  return code;
 }
 
 int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) {
   return (*msgcb->qsizeFp)(msgcb->mgmt, vgId, qtype);
 }
 
-int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { return (*defaultMsgCb.sendReqFp)(epSet, pMsg); }
+int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) {
+  int32_t code = (*defaultMsgCb.sendReqFp)(epSet, pMsg);
+  if (code != 0) {
+    rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
+  }
+  return code;
+}
 
 void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); }
 

tests/pytest/crash_gen.sh

@@ -48,7 +48,12 @@ fi
 PYTHON_EXEC=python3.8
 
 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work.
-export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd)
+# export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd)
+# NOTE: we are now pointing outside the current github, per Wade on 7/7/2022, we'll be keeping connectors outside
+# and there does not seem to be a module to reference that.
+PROJECT_PARENT=$(pwd)/../../..
+TAOS_PYTHON_PROJECT_DIR=$PROJECT_PARENT/taos-connector-python 
+export PYTHONPATH=$TAOS_PYTHON_PROJECT_DIR:$(pwd)
 
 # Then let us set up the library path so that our compiled SO file can be loaded by Python
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR

tests/pytest/crash_gen/crash_gen_main.py

@@ -466,6 +466,7 @@ class ThreadCoordinator:
         self._te = None  # No more executor, time to end
         Logging.debug("Main thread tapping all threads one last time...")
         self.tapAllThreads()  # Let the threads run one last time
+        #TODO: looks like we are not capturing the failures for the last step yet (i.e. calling registerFailure if neccessary)
 
         Logging.debug("\r\n\n--> Main thread ready to finish up...")
         Logging.debug("Main thread joining all threads")
@@ -1290,6 +1291,7 @@ class Task():
 
     def _isErrAcceptable(self, errno, msg):
         if errno in [
+                # TDengine 2.x Error Codes:
                 0x05,  # TSDB_CODE_RPC_NOT_READY
                 0x0B,  # Unable to establish connection, more details in TD-1648
                 # 0x200, # invalid SQL， TODO: re-examine with TD-934
@@ -1310,6 +1312,18 @@ class Task():
                 0x14,   # db not ready, errno changed
                 0x600,  # Invalid table ID, why?
                 0x218,  # Table does not exist
+
+                # TDengine 3.0 Error Codes:
+                0x0333, # Object is creating # TODO: this really is NOT an acceptable error
+                0x03A0, # STable already exists
+                0x03A1, # STable [does] not exist
+                0x03AA, # Tag already exists
+                0x0603, # Table already exists
+                0x2602, # Table does not exist
+                0x260d, # Tags number not matched
+
+
+
                 1000  # REST catch-all error
             ]: 
             return True # These are the ALWAYS-ACCEPTABLE ones
@@ -1749,6 +1763,8 @@ class TdSuperTable:
             tagType = tags[tagName]
             if tagType == 'BINARY':
                 tagStrs.append("'Beijing-Shanghai-LosAngeles'")
+            elif tagType== 'VARCHAR':
+                tagStrs.append("'London-Paris-Berlin'")
             elif tagType == 'FLOAT':
                 tagStrs.append('9.9')
             elif tagType == 'INT':

tests/script/jenkins/basic.txt

@@ -163,8 +163,8 @@
 # --- sma
 ./test.sh -f tsim/sma/drop_sma.sim
 ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
-./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
-./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
+#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
+#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
 
 # --- valgrind
 ./test.sh -f tsim/valgrind/checkError1.sim

tests/script/tsim/valgrind/basic2.sim

@@ -4,29 +4,107 @@ system sh/cfg.sh -n dnode1 -c debugflag -v 131
 system sh/exec.sh -n dnode1 -s start -v
 sql connect
 
-print =============== step1: create drop show dnodes
+print =============== step1: show dnodes
 $x = 0
 step1:
 	$x = $x + 1
 	sleep 1000
 	if $x == 10 then
-	  print ----> dnode not ready!
+	  print ---> dnode not ready!
 		return -1
 	endi
 sql show dnodes
-print ----> $data00 $data01 $data02 $data03 $data04 $data05
+print ---> $data00 $data01 $data02 $data03 $data04 $data05
 if $rows != 1 then
   return -1
 endi
+if $data(1)[4] != ready then
+  goto step1
+endi
 
-print =============== step2
-sql create database db vgroups 1 buffer 3
-sql use db
-sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
-sql create table ct1 using stb tags(1000)
-sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
-sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
+print =============== step2: create alter drop show user
+sql create user u1 pass 'taosdata'
+sql show users
+sql alter user u1 sysinfo 1
+sql alter user u1 enable 1
+sql alter user u1 pass 'taosdata'
+sql drop user u1
+sql_error alter user u2 sysinfo 0
 
+print =============== step3: create drop dnode
+sql create dnode $hostname port 7200
+sql drop dnode 2
+sql alter dnode 1 'debugflag 131'
 
-_OVER:
+print =============== step4:
+
+print =============== run show xxxx
+sql show dnodes
+if $rows != 1 then 
+  return -1
+endi
+
+sql show mnodes
+if $rows != 1 then 
+  return -1
+endi
+
+sql show databases
+if $rows != 2 then
+  return -1
+endi
+
+sql show users
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== run select * from information_schema.xxxx
+sql select * from information_schema.`dnodes`
+if $rows != 1 then 
+  return -1
+endi
+
+sql select * from information_schema.`mnodes`
+if $rows != 1 then 
+  return -1
+endi
+
+sql select * from information_schema.user_users
+if $rows != 1 then 
+  return -1
+endi
+
+sql show variables;
+if $rows != 4 then
+  return -1
+endi
+
+sql show dnode 1 variables;
+if $rows <= 0 then
+  return -1
+endi
+
+sql show local variables;
+if $rows <= 0 then
+  return -1
+endi
+
+print =============== stop
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
+
+print =============== check
+print ----> start to check if there are ERRORS in vagrind log file for each dnode
+system_content sh/checkValgrind.sh -n dnode1 
+
+print cmd return result ----> [ $system_content ]
+if $system_content <= 0 then
+  return 0
+endi 
+
+$null=
+if $system_content == $null then
+  return 0
+endi 
+
+return -1

tests/system-test/2-query/and_or_for_byte.py

@@ -230,7 +230,7 @@ class TDTestCase:
                 work_sql += f"cast({arg} as bigint){opera}"
 
         if not agg:
-            work_sql+= f" from {tbname} order by ts"
+            work_sql+= f" from {tbname} order by tbname ,ts"
         else:
             work_sql+= f" from {tbname} "
         tdSql.query(work_sql)
@@ -243,7 +243,7 @@ class TDTestCase:
             else:
                 origin_sql += f"cast({arg} as bigint),"
         if not agg:
-            origin_sql+= f" from {tbname} order by ts"
+            origin_sql+= f" from {tbname} order by tbname ,ts"
         else:
             origin_sql+= f" from {tbname} "
         tdSql.query(origin_sql)

tests/system-test/fulltest.sh

@@ -117,7 +117,7 @@ python3 ./test.py -f 2-query/distribute_agg_avg.py
 python3 ./test.py -f 2-query/distribute_agg_stddev.py
 python3 ./test.py -f 2-query/twa.py
 python3 ./test.py -f 2-query/irate.py
-#python3 ./test.py -f 2-query/and_or_for_byte.py
+python3 ./test.py -f 2-query/and_or_for_byte.py
 
 python3 ./test.py -f 2-query/function_null.py
 python3 ./test.py -f 2-query/queryQnode.py