Merge branch 'develop' into hotfix/sangshuduo/TD-2893-fix-http-error-typo

CMakeLists.txt

@@ -15,6 +15,7 @@ SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
 SET(TD_MQTT FALSE)
 SET(TD_TSDB_PLUGINS FALSE)
+SET(TD_STORAGE FALSE)
 
 SET(TD_COVER FALSE)
 SET(TD_MEM_CHECK FALSE)

Jenkinsfile

@@ -5,7 +5,7 @@ node {
     git url: 'https://github.com/taosdata/TDengine.git'
 }
 
-
+def kipstage=0
 def abortPreviousBuilds() {
   def currentJobName = env.JOB_NAME
   def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@@ -45,6 +45,7 @@ def pre_test(){
     git pull
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
+    git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0
     cd ${WK}
     git reset --hard HEAD~10
     git checkout develop
@@ -62,6 +63,7 @@ def pre_test(){
     '''
     return 1
 }
+
 pipeline {
   agent none
   
@@ -71,21 +73,43 @@ pipeline {
   }
   
   stages {
-      
+      stage('pre_build'){
-    
+          agent{label 'master'}
-      stage('Parallel test stage') {
-        //only build pr
           when {
               changeRequest()
           }
+          steps {
+          sh'''
+          cd ${WORKSPACE}
+          git checkout develop
+          git pull
+          git fetch origin +refs/pull/${CHANGE_ID}/merge
+          git checkout -qf FETCH_HEAD
+          '''
+          script{
+            skipstage=sh(script:"git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 1 ",returnStdout:true) 
+          }
+          }
+      }
+    
+      stage('Parallel test stage') {
+        
+        //only build pr
+        when {
+              changeRequest()
+               expression {
+                    skipstage != 1
+              }
+          }
       parallel {
-        stage('python_1') {
+        stage('python_1_s1') {
           agent{label 'p1'}
           steps {
             
             pre_test()
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
+              date
               cd ${WKC}/tests
               find pytest -name '*'sql|xargs rm -rf
               ./test-all.sh p1
@@ -94,26 +118,38 @@ pipeline {
             
           }
         }
-        stage('python_2') {
+        stage('python_2_s5') {
           agent{label 'p2'}
           steps {
             
             pre_test()
+            timeout(time: 45, unit: 'MINUTES'){
             sh '''
+            date
             cd ${WKC}/tests
             find pytest -name '*'sql|xargs rm -rf
             ./test-all.sh p2
             date'''
+            }
+          }
+        }
+        stage('python_3_s6') {
+          agent{label 'p3'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
               sh '''
+              date
               cd ${WKC}/tests
-            ./test-all.sh b4fq
+              ./test-all.sh p3
-            '''
+              date'''
             }
           }
-        stage('test_b1') {
+        }
+        stage('test_b1_s2') {
           agent{label 'b1'}
           steps {     
-            timeout(time: 90, unit: 'MINUTES'){       
+            timeout(time: 45, unit: 'MINUTES'){       
               pre_test()
               sh '''
               cd ${WKC}/tests
@@ -123,7 +159,7 @@ pipeline {
           }
         }
 
-        stage('test_crash_gen') {
+        stage('test_crash_gen_s3') {
           agent{label "b2"}
           steps {
             pre_test()
@@ -139,7 +175,7 @@ pipeline {
                 ./handle_crash_gen_val_log.sh
                 '''
             }
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
               date
               cd ${WKC}/tests
@@ -150,7 +186,7 @@ pipeline {
           }
         }
 
-        stage('test_valgrind') {
+        stage('test_valgrind_s4') {
           agent{label "b3"}
 
           steps {
@@ -162,7 +198,7 @@ pipeline {
                 ./handle_val_log.sh
                 '''
             }     
-            timeout(time: 90, unit: 'MINUTES'){      
+            timeout(time: 45, unit: 'MINUTES'){      
               sh '''
               date
               cd ${WKC}/tests
@@ -171,17 +207,66 @@ pipeline {
             }
           }
         }
-   
+        stage('test_b4_s7') {
-        
+          agent{label 'b4'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b4fq
+              date'''
+            }
+          }
+        }
+        stage('test_b5_s8') {
+          agent{label 'b5'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b5fq
+              date'''
+            }
+          }
+        }
+        stage('test_b6_s9') {
+          agent{label 'b6'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b6fq
+              date'''
+            }
+          }
+        }
+        stage('test_b7_s10') {
+          agent{label 'b7'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b7fq
+              date'''
+            }
+          }
+        }        
     }
   }
   }
   post {  
-      
         success {
             emailext (
-                subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+                subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS",
-                body: '''<!DOCTYPE html>
+                body: """<!DOCTYPE html>
                 <html>
                 <head>
                 <meta charset="UTF-8">
@@ -197,29 +282,29 @@ pipeline {
                             <td>
                                 <ul>
                                 <div style="font-size:18px">
-                                    <li>构建名称>>分支：${PROJECT_NAME}</li>
+                                    <li>构建名称>>分支：${env.BRANCH_NAME}</li>
                                     <li>构建结果：<span style="color:green"> Successful </span></li>
                                     <li>构建编号：${BUILD_NUMBER}</li>
-                                    <li>触发用户：${CAUSE}</li>
+                                    <li>触发用户：${env.CHANGE_AUTHOR}</li>
-                                    <li>提交信息：${CHANGE_TITLE}</li>
+                                    <li>提交信息：${env.CHANGE_TITLE}</li>
                                     <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
                                     <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
-                                    <li>变更集：${JELLY_SCRIPT}</li>
+                                    
                                 </div>
                                 </ul>
                             </td>
                         </tr>
                     </table></font>
                 </body>
-                </html>''',
+                </html>""",
                 to: "${env.CHANGE_AUTHOR_EMAIL}",
                 from: "support@taosdata.com"
             )
         }
         failure {
             emailext (
-                subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+                subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL",
-                body: '''<!DOCTYPE html>
+                body: """<!DOCTYPE html>
                 <html>
                 <head>
                 <meta charset="UTF-8">
@@ -235,21 +320,21 @@ pipeline {
                             <td>
                                 <ul>
                                 <div style="font-size:18px">
-                                    <li>构建名称>>分支：${PROJECT_NAME}</li>
+                                    <li>构建名称>>分支：${env.BRANCH_NAME}</li>
-                                    <li>构建结果：<span style="color:green"> Successful </span></li>
+                                    <li>构建结果：<span style="color:red"> Failure </span></li>
                                     <li>构建编号：${BUILD_NUMBER}</li>
-                                    <li>触发用户：${CAUSE}</li>
+                                    <li>触发用户：${env.CHANGE_AUTHOR}</li>
-                                    <li>提交信息：${CHANGE_TITLE}</li>
+                                    <li>提交信息：${env.CHANGE_TITLE}</li>
                                     <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
                                     <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
-                                    <li>变更集：${JELLY_SCRIPT}</li>
+                                    
                                 </div>
                                 </ul>
                             </td>
                         </tr>
                     </table></font>
                 </body>
-                </html>''',
+                </html>""",
                 to: "${env.CHANGE_AUTHOR_EMAIL}",
                 from: "support@taosdata.com"
             )

cmake/define.inc

@@ -21,6 +21,10 @@ IF (TD_TSDB_PLUGINS)
   ADD_DEFINITIONS(-D_TSDB_PLUGINS)
 ENDIF ()
 
+IF (TD_STORAGE)
+  ADD_DEFINITIONS(-D_STORAGE)
+ENDIF ()
+
 IF (TD_GODLL)
   ADD_DEFINITIONS(-D_TD_GO_DLL_)
 ENDIF ()

cmake/version.inc

@@ -13,28 +13,40 @@ ELSE ()
   SET(TD_VER_COMPATIBLE "2.0.0.0")
 ENDIF ()
 
+find_program(HAVE_GIT NAMES git)
+
 IF (DEFINED GITINFO)
   SET(TD_VER_GIT ${GITINFO})
-ELSE ()
+ELSEIF (HAVE_GIT)
-  execute_process(
+  execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID)
-	  COMMAND git log -1 --format=%H
+  message(STATUS "git log result:${GIT_COMMITID}")
-	  WORKING_DIRECTORY ${TD_COMMUNITY_DIR} 
+  IF (GIT_COMMITID)
-	  OUTPUT_VARIABLE GIT_COMMITID
-  )
     string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
     SET(TD_VER_GIT ${GIT_COMMITID})
+  ELSE ()
+    message(STATUS "not a git repository")
+    SET(TD_VER_GIT "no git commit id")
+  ENDIF ()
+ELSE ()
+  message(STATUS "no git cmd")
+  SET(TD_VER_GIT "no git commit id")
 ENDIF ()
 
 IF (DEFINED GITINFOI)
   SET(TD_VER_GIT_INTERNAL ${GITINFOI})
-ELSE ()
+ELSEIF (HAVE_GIT)
-  execute_process(
+  execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
-	  COMMAND git log -1 --format=%H
+  message(STATUS "git log result:${GIT_COMMITID}")
-	  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} 
+  IF (GIT_COMMITID)
-	  OUTPUT_VARIABLE GIT_COMMITID
-  )
     string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
     SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID})
+  ELSE ()
+    message(STATUS "not a git repository")
+    SET(TD_VER_GIT "no git commit id")
+  ENDIF ()
+ELSE ()
+  message(STATUS "no git cmd")
+  SET(TD_VER_GIT_INTERNAL "no git commit id")
 ENDIF ()
 
 IF (DEFINED VERDATE)

deps/CMakeLists.txt

@@ -9,6 +9,7 @@ ADD_SUBDIRECTORY(lz4)
 ADD_SUBDIRECTORY(cJson)
 ADD_SUBDIRECTORY(wepoll)
 ADD_SUBDIRECTORY(MsvcLibX)
+ADD_SUBDIRECTORY(rmonotonic)
 
 IF (TD_LINUX AND TD_MQTT)
   ADD_SUBDIRECTORY(MQTT-C)

deps/MsvcLibX/include/msvcTime.h

@@ -38,6 +38,7 @@
 typedef int clockid_t;
 /* Supported values for clockid_t */
 #define CLOCK_REALTIME 0
+#define CLOCK_MONOTONIC 1
 
 int clock_gettime(clockid_t clock_id, struct timespec *tp);
 

deps/MsvcLibX/include/msvcUnistd.h

@@ -89,11 +89,12 @@ pid_t getppid(void);	/* Get parent PID */
 
 /* Path management */
 #if defined(_WIN32)
-#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
 #define realpath realpathU
+#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
+// #define realpath realpathU
 #define CompactPath CompactPathU
 #else /* _ANSI_SOURCE */
-#define realpath realpathA
+// #define realpath realpathA
 #define CompactPath CompactPathA
 #endif
 #endif /* defined(_WIN32) */

deps/MsvcLibX/src/clock_gettime.c

@@ -34,15 +34,56 @@
 #include "msvcTime.h"
 #include "sys/msvcStat.h" /* For MsvcLibX's Filetime2Timespec */
 
-int clock_gettime(clockid_t clock_id, struct timespec *pTS) {
+#define MS_PER_SEC      1000ULL     // MS = milliseconds
-  FILETIME ft;
+#define US_PER_MS       1000ULL     // US = microseconds
-  if (clock_id != CLOCK_REALTIME) {
+#define HNS_PER_US      10ULL       // HNS = hundred-nanoseconds (e.g., 1 hns = 100 ns)
-    errno = EINVAL;
+#define NS_PER_US       1000ULL
+
+#define HNS_PER_SEC     (MS_PER_SEC * US_PER_MS * HNS_PER_US)
+#define NS_PER_HNS      (100ULL)    // NS = nanoseconds
+#define NS_PER_SEC      (MS_PER_SEC * US_PER_MS * NS_PER_US)
+
+int clock_gettime_monotonic(struct timespec *tv) {
+  static LARGE_INTEGER ticksPerSec;
+  LARGE_INTEGER ticks;
+  double seconds;
+
+  if (!ticksPerSec.QuadPart) {
+    QueryPerformanceFrequency(&ticksPerSec);
+    if (!ticksPerSec.QuadPart) {
+      errno = ENOTSUP;
       return -1;
     }
-  GetSystemTimeAsFileTime(&ft);
+  }
-  Filetime2Timespec(&ft, pTS);
+
+  QueryPerformanceCounter(&ticks);
+
+  seconds = (double) ticks.QuadPart / (double) ticksPerSec.QuadPart;
+  tv->tv_sec = (time_t)seconds;
+  tv->tv_nsec = (long)((ULONGLONG)(seconds * NS_PER_SEC) % NS_PER_SEC);
+
   return 0;
 }
 
+int clock_gettime_realtime(struct timespec *pTS) {
+  FILETIME ft;
+
+  GetSystemTimeAsFileTime(&ft);
+  Filetime2Timespec(&ft, pTS);
+
+  return 0;
+}
+
+int clock_gettime(clockid_t clock_id, struct timespec *pTS) {
+  if (clock_id == CLOCK_MONOTONIC) {
+    return clock_gettime_monotonic(pTS);
+  } else if (clock_id == CLOCK_REALTIME) {
+    return clock_gettime_realtime(pTS);
+  }
+
+  errno = ENOTSUP;
+
+  return -1;
+}
+
 #endif /* defined(_WIN32) */

deps/rmonotonic/CMakeLists.txt

@@ -0,0 +1,11 @@
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
+
+add_definitions(-DUSE_PROCESSOR_CLOCK)
+
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../MsvcLibX/include)
+
+ADD_LIBRARY(rmonotonic ${SOURCE_LIST})
+TARGET_INCLUDE_DIRECTORIES(rmonotonic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)
+IF (TD_WINDOWS)
+  TARGET_LINK_LIBRARIES(rmonotonic MsvcLibXw)
+ENDIF ()

deps/rmonotonic/inc/monotonic.h

@@ -0,0 +1,56 @@
+#ifndef __MONOTONIC_H
+#define __MONOTONIC_H
+/* The monotonic clock is an always increasing clock source.  It is unrelated to
+ * the actual time of day and should only be used for relative timings.  The
+ * monotonic clock is also not guaranteed to be chronologically precise; there
+ * may be slight skew/shift from a precise clock.
+ *
+ * Depending on system architecture, the monotonic time may be able to be
+ * retrieved much faster than a normal clock source by using an instruction
+ * counter on the CPU.  On x86 architectures (for example), the RDTSC
+ * instruction is a very fast clock source for this purpose.
+ */
+
+//#include "fmacros.h"
+#include <stdint.h>
+//#include <unistd.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+  #define inline
+#endif
+
+/* A counter in micro-seconds.  The 'monotime' type is provided for variables
+ * holding a monotonic time.  This will help distinguish & document that the
+ * variable is associated with the monotonic clock and should not be confused
+ * with other types of time.*/
+typedef uint64_t monotime;
+
+/* Retrieve counter of micro-seconds relative to an arbitrary point in time.  */
+extern monotime (*getMonotonicUs)(void);
+
+
+/* Call once at startup to initialize the monotonic clock.  Though this only
+ * needs to be called once, it may be called additional times without impact.
+ * Returns a printable string indicating the type of clock initialized.
+ * (The returned string is static and doesn't need to be freed.)  */
+const char * monotonicInit();
+
+
+/* Functions to measure elapsed time.  Example:
+ *     monotime myTimer;
+ *     elapsedStart(&myTimer);
+ *     while (elapsedMs(myTimer) < 10) {} // loops for 10ms
+ */
+static inline void elapsedStart(monotime *start_time) {
+    *start_time = getMonotonicUs();
+}
+
+static inline uint64_t elapsedUs(monotime start_time) {
+    return getMonotonicUs() - start_time;
+}
+
+static inline uint64_t elapsedMs(monotime start_time) {
+    return elapsedUs(start_time) / 1000;
+}
+
+#endif

deps/rmonotonic/src/monotonic.c

@@ -0,0 +1,173 @@
+#include "monotonic.h"
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+
+#undef NDEBUG
+#include <assert.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+#include "msvcTime.h"
+#include "msvcStdio.h"
+#endif
+
+/* The function pointer for clock retrieval.  */
+monotime (*getMonotonicUs)(void) = NULL;
+
+static char monotonic_info_string[32];
+
+
+/* Using the processor clock (aka TSC on x86) can provide improved performance
+ * throughout Redis wherever the monotonic clock is used.  The processor clock
+ * is significantly faster than calling 'clock_getting' (POSIX).  While this is
+ * generally safe on modern systems, this link provides additional information
+ * about use of the x86 TSC: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage
+ *
+ * To use the processor clock, either uncomment this line, or build with
+ *   CFLAGS="-DUSE_PROCESSOR_CLOCK"
+#define USE_PROCESSOR_CLOCK
+ */
+
+
+#if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__)
+#include <regex.h>
+#include <x86intrin.h>
+
+static long mono_ticksPerMicrosecond = 0;
+
+static monotime getMonotonicUs_x86() {
+    return __rdtsc() / mono_ticksPerMicrosecond;
+}
+
+static void monotonicInit_x86linux() {
+    const int bufflen = 256;
+    char buf[bufflen];
+    regex_t cpuGhzRegex, constTscRegex;
+    const size_t nmatch = 2;
+    regmatch_t pmatch[nmatch];
+    int constantTsc = 0;
+    int rc;
+
+    /* Determine the number of TSC ticks in a micro-second.  This is
+     * a constant value matching the standard speed of the processor.
+     * On modern processors, this speed remains constant even though
+     * the actual clock speed varies dynamically for each core.  */
+    rc = regcomp(&cpuGhzRegex, "^model name\\s+:.*@ ([0-9.]+)GHz", REG_EXTENDED);
+    assert(rc == 0);
+
+    /* Also check that the constant_tsc flag is present.  (It should be
+     * unless this is a really old CPU.  */
+    rc = regcomp(&constTscRegex, "^flags\\s+:.* constant_tsc", REG_EXTENDED);
+    assert(rc == 0);
+
+    FILE *cpuinfo = fopen("/proc/cpuinfo", "r");
+    if (cpuinfo != NULL) {
+        while (fgets(buf, bufflen, cpuinfo) != NULL) {
+            if (regexec(&cpuGhzRegex, buf, nmatch, pmatch, 0) == 0) {
+                buf[pmatch[1].rm_eo] = '\0';
+                double ghz = atof(&buf[pmatch[1].rm_so]);
+                mono_ticksPerMicrosecond = (long)(ghz * 1000);
+                break;
+            }
+        }
+        while (fgets(buf, bufflen, cpuinfo) != NULL) {
+            if (regexec(&constTscRegex, buf, nmatch, pmatch, 0) == 0) {
+                constantTsc = 1;
+                break;
+            }
+        }
+
+        fclose(cpuinfo);
+    }
+    regfree(&cpuGhzRegex);
+    regfree(&constTscRegex);
+
+    if (mono_ticksPerMicrosecond == 0) {
+        //fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate");
+        return;
+    }
+    if (!constantTsc) {
+        //fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present");
+        return;
+    }
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "X86 TSC @ %ld ticks/us", mono_ticksPerMicrosecond);
+    getMonotonicUs = getMonotonicUs_x86;
+}
+#endif
+
+
+#if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__)
+static long mono_ticksPerMicrosecond = 0;
+
+/* Read the clock value.  */
+static inline uint64_t __cntvct() {
+    uint64_t virtual_timer_value;
+    __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value));
+    return virtual_timer_value;
+}
+
+/* Read the Count-timer Frequency.  */
+static inline uint32_t cntfrq_hz() {
+    uint64_t virtual_freq_value;
+    __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(virtual_freq_value));
+    return (uint32_t)virtual_freq_value;    /* top 32 bits are reserved */
+}
+
+static monotime getMonotonicUs_aarch64() {
+    return __cntvct() / mono_ticksPerMicrosecond;
+}
+
+static void monotonicInit_aarch64() {
+    mono_ticksPerMicrosecond = (long)cntfrq_hz() / 1000L / 1000L;
+    if (mono_ticksPerMicrosecond == 0) {
+        fprintf(stderr, "monotonic: aarch64, unable to determine clock rate");
+        return;
+    }
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "ARM CNTVCT @ %ld ticks/us", mono_ticksPerMicrosecond);
+    getMonotonicUs = getMonotonicUs_aarch64;
+}
+#endif
+
+static monotime getMonotonicUs_posix(void) {
+    /* clock_gettime() is specified in POSIX.1b (1993).  Even so, some systems
+     * did not support this until much later.  CLOCK_MONOTONIC is technically
+     * optional and may not be supported - but it appears to be universal.
+     * If this is not supported, provide a system-specific alternate version.  */
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000;
+}
+
+static void monotonicInit_posix() {
+    /* Ensure that CLOCK_MONOTONIC is supported.  This should be supported
+     * on any reasonably current OS.  If the assertion below fails, provide
+     * an appropriate alternate implementation.  */
+    struct timespec ts;
+    int rc = clock_gettime(CLOCK_MONOTONIC, &ts);
+    assert(rc == 0);
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "POSIX clock_gettime");
+    getMonotonicUs = getMonotonicUs_posix;
+}
+
+
+
+const char * monotonicInit() {
+    #if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__)
+    if (getMonotonicUs == NULL) monotonicInit_x86linux();
+    #endif
+
+    #if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__)
+    if (getMonotonicUs == NULL) monotonicInit_aarch64();
+    #endif
+
+    if (getMonotonicUs == NULL) monotonicInit_posix();
+
+    return monotonic_info_string;
+}

documentation20/webdocs/markdowndocs/connector-ch.md

@@ -852,7 +852,7 @@ npm install td2.0-connector
 ### Linux
 
 - `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
-- `node`  必须采用v10.x版本，其他版本存在包兼容性的问题。
+- `node`  2.0.6支持v12.x和v10.x，2.0.5及更早版本支持v10.x版本，其他版本可能存在包兼容性的问题。
 - `make`
 - c语言编译器比如<a href="https://gcc.gnu.org">GCC</a>
 

src/CMakeLists.txt

@@ -5,6 +5,7 @@ PROJECT(TDengine)
 ADD_SUBDIRECTORY(os)
 ADD_SUBDIRECTORY(common)
 ADD_SUBDIRECTORY(util)
+ADD_SUBDIRECTORY(tfs)
 ADD_SUBDIRECTORY(rpc)
 ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)

src/client/inc/tscSubquery.h

@@ -33,6 +33,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index);
 void tscHandleMasterJoinQuery(SSqlObj* pSql);
 
 int32_t tscHandleMasterSTableQuery(SSqlObj *pSql);
+int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql);
 
 int32_t tscHandleMultivnodeInsert(SSqlObj *pSql);
 

src/client/inc/tscUtil.h

@@ -132,8 +132,9 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo);
 
 bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
 bool tscQueryTags(SQueryInfo* pQueryInfo);
+bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
 
-SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
+SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
                                        SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType);
 
 int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql);
@@ -174,6 +175,7 @@ SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIn
 SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
                            int16_t size);
 size_t   tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo);
+void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex);
 
 SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index);
 int32_t   tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);

src/client/inc/tsclient.h

@@ -224,7 +224,9 @@ typedef struct SQueryInfo {
   int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
   int16_t          resColumnId;   // result column id
   bool             distinctTag;   // distinct tag or not
-  
+  int32_t          round;         // 0/1/....
+  int32_t          bufLen;
+  char*            buf;
 } SQueryInfo;
 
 typedef struct {
@@ -412,10 +414,9 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code);
 
 int tscProcessLocalCmd(SSqlObj *pSql);
 int tscCfgDynamicOptions(char *msg);
-int taos_retrieve(TAOS_RES *res);
 
-int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
+int32_t tscTansformFuncForSTableQuery(SQueryInfo *pQueryInfo);
-void    tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
+void    tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo);
 
 int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
 void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo);

src/client/src/tscLocal.c

@@ -892,7 +892,12 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
 
   if (pCmd->command == TSDB_SQL_CFG_LOCAL) {
-    pRes->code = (uint8_t)taosCfgDynamicOptions(pCmd->payload);
+    if (taosCfgDynamicOptions(pCmd->payload)) {
+       pRes->code = TSDB_CODE_SUCCESS;
+    } else {
+       pRes->code = TSDB_CODE_COM_INVALID_CFG_MSG; 
+    } 
+    pRes->numOfRows = 0;
   } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) {
     pRes->code = (uint8_t)tscProcessDescribeTable(pSql);
   } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {

src/client/src/tscLocalMerge.c

@@ -68,7 +68,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescr
     SQLFunctionCtx *pCtx = &pReducer->pCtx[i];
     SSqlExpr *      pExpr = tscSqlExprGet(pQueryInfo, i);
 
-    pCtx->aOutputBuf = pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity;
+    pCtx->pOutput = pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity;
     pCtx->order = pQueryInfo->order.order;
     pCtx->functionId = pExpr->functionId;
 
@@ -76,7 +76,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescr
     int16_t  offset = getColumnModelOffset(pDesc->pColumnModel, i);
     SSchema *pSchema = getColumnModelSchema(pDesc->pColumnModel, i);
 
-    pCtx->aInputElemBuf = pReducer->pTempBuffer->data + offset;
+    pCtx->pInput = pReducer->pTempBuffer->data + offset;
 
     // input data format comes from pModel
     pCtx->inputType = pSchema->type;
@@ -94,7 +94,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescr
     // for top/bottom function, the output of timestamp is the first column
     int32_t functionId = pExpr->functionId;
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
-      pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf;
+      pCtx->ptsOutputBuf = pReducer->pCtx[0].pOutput;
       pCtx->param[2].i64 = pQueryInfo->order.order;
       pCtx->param[2].nType  = TSDB_DATA_TYPE_BIGINT;
       pCtx->param[1].i64 = pQueryInfo->order.orderColId;
@@ -118,7 +118,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescr
     if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) {
       tagLen += pExpr->resBytes;
       pTagCtx[n++] = &pReducer->pCtx[i];
-    } else if ((aAggs[pExpr->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
+    } else if ((aAggs[pExpr->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
       pCtx = &pReducer->pCtx[i];
     }
   }
@@ -311,7 +311,7 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde
   pReducer->pCtx = (SQLFunctionCtx *)calloc(tscSqlExprNumOfExprs(pQueryInfo), sizeof(SQLFunctionCtx));
   pReducer->rowSize = pMemBuffer[0]->nElemSize;
 
-  tscRestoreSQLFuncForSTableQuery(pQueryInfo);
+  tscRestoreFuncForSTableQuery(pQueryInfo);
   tscFieldInfoUpdateOffset(pQueryInfo);
 
   if (pReducer->rowSize > pMemBuffer[0]->pageSize) {
@@ -383,7 +383,7 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde
   
   if (pQueryInfo->fillType != TSDB_FILL_NONE) {
     SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
-    pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
+    pReducer->pFillInfo = taosCreateFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
                                            4096, (int32_t)pQueryInfo->fieldsInfo.numOfOutput, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
                                            tinfo.precision, pQueryInfo->fillType, pFillCol, pSql);
   }
@@ -720,7 +720,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
     SSchema p1 = {0};
     if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) {
-      p1 = tGetTableNameColumnSchema();
+      p1 = *tGetTbnameColumnSchema();
     } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) {
       p1.bytes = pExpr->resBytes;
       p1.type  = (uint8_t) pExpr->resType;
@@ -744,6 +744,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
         functionId = TSDB_FUNC_FIRST;
       } else if (functionId == TSDB_FUNC_LAST_DST) {
         functionId = TSDB_FUNC_LAST;
+      } else if (functionId == TSDB_FUNC_STDDEV_DST) {
+        functionId = TSDB_FUNC_STDDEV;
       }
 
       int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false);
@@ -1041,7 +1043,7 @@ static void savePreviousRow(SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) {
   pLocalMerge->hasPrevRow = true;
 }
 
-static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, bool needInit) {
+static void doExecuteFinalMerge(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, bool needInit) {
   // the tag columns need to be set before all functions execution
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
@@ -1053,7 +1055,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, bo
     int32_t functionId = pCtx->functionId;
     if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS_DUMMY) {
       tVariantDestroy(&pCtx->tag);
-      char* input = pCtx->aInputElemBuf;
+      char* input = pCtx->pInput;
       
       if (pCtx->inputType == TSDB_DATA_TYPE_BINARY || pCtx->inputType == TSDB_DATA_TYPE_NCHAR) {
         assert(varDataLen(input) <= pCtx->inputBytes);
@@ -1061,6 +1063,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, bo
       } else {
         tVariantCreateFromBinary(&pCtx->tag, input, pCtx->inputBytes, pCtx->inputType);
       }
+
     } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
       SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, j);
       pCtx->param[0].i64 = pExpr->param[0].i64;
@@ -1086,7 +1089,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, bo
 static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) {
   if (pLocalMerge->hasUnprocessedRow) {
     pLocalMerge->hasUnprocessedRow = false;
-    doExecuteSecondaryMerge(pCmd, pLocalMerge, true);
+    doExecuteFinalMerge(pCmd, pLocalMerge, true);
     savePreviousRow(pLocalMerge, tmpBuffer);
   }
 }
@@ -1142,11 +1145,11 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo
 
     int32_t inc = numOfRes - 1;  // tsdb_func_tag function only produce one row of result
     memset(buf, 0, (size_t)maxBufSize);
-    memcpy(buf, pCtx->aOutputBuf, (size_t)pCtx->outputBytes);
+    memcpy(buf, pCtx->pOutput, (size_t)pCtx->outputBytes);
 
     for (int32_t i = 0; i < inc; ++i) {
-      pCtx->aOutputBuf += pCtx->outputBytes;
+      pCtx->pOutput += pCtx->outputBytes;
-      memcpy(pCtx->aOutputBuf, buf, (size_t)pCtx->outputBytes);
+      memcpy(pCtx->pOutput, buf, (size_t)pCtx->outputBytes);
     }
   }
 
@@ -1289,10 +1292,10 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge) {// reset
   size_t t = tscSqlExprNumOfExprs(pQueryInfo);
   for (int32_t i = 0; i < t; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-    pLocalMerge->pCtx[i].aOutputBuf = pLocalMerge->pResultBuf->data + pExpr->offset * pLocalMerge->resColModel->capacity;
+    pLocalMerge->pCtx[i].pOutput = pLocalMerge->pResultBuf->data + pExpr->offset * pLocalMerge->resColModel->capacity;
 
     if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM || pExpr->functionId == TSDB_FUNC_DIFF) {
-      pLocalMerge->pCtx[i].ptsOutputBuf = pLocalMerge->pCtx[0].aOutputBuf;
+      pLocalMerge->pCtx[i].ptsOutputBuf = pLocalMerge->pCtx[0].pOutput;
     }
   }
 
@@ -1404,7 +1407,7 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) {
 
   for (int32_t k = 0; k < size; ++k) {
     SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[k];
-    pCtx->aOutputBuf += pCtx->outputBytes * numOfRes;
+    pCtx->pOutput += pCtx->outputBytes * numOfRes;
 
     // set the correct output timestamp column position
     if (pCtx->functionId == TSDB_FUNC_TOP || pCtx->functionId == TSDB_FUNC_BOTTOM) {
@@ -1412,7 +1415,7 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) {
     }
   }
 
-  doExecuteSecondaryMerge(pCmd, pLocalMerge, true);
+  doExecuteFinalMerge(pCmd, pLocalMerge, true);
 }
 
 int32_t tscDoLocalMerge(SSqlObj *pSql) {
@@ -1504,7 +1507,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
     if (pLocalMerge->hasPrevRow) {
       if (needToMerge(pQueryInfo, pLocalMerge, tmpBuffer)) {
         // belong to the group of the previous row, continue process it
-        doExecuteSecondaryMerge(pCmd, pLocalMerge, false);
+        doExecuteFinalMerge(pCmd, pLocalMerge, false);
 
         // copy to buffer
         savePreviousRow(pLocalMerge, tmpBuffer);
@@ -1576,7 +1579,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
         }
       }
     } else {
-      doExecuteSecondaryMerge(pCmd, pLocalMerge, true);
+      doExecuteFinalMerge(pCmd, pLocalMerge, true);
       savePreviousRow(pLocalMerge, tmpBuffer);  // copy the processed row to buffer
     }
 

src/client/src/tscSQLParser.c

@@ -129,6 +129,7 @@ static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo);
 static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo);
 static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index);
 static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid);
+static bool    validateDebugFlag(int32_t flag);
 
 int16_t getNewResColId(SQueryInfo* pQueryInfo) {
   return pQueryInfo->resColumnId--;
@@ -173,6 +174,16 @@ static uint8_t convertOptr(SStrToken *pToken) {
   }
 }
 
+static bool validateDebugFlag(int32_t v) {
+  const static int validFlag[] = {131, 135, 143};
+  
+  for (int i = 0; i < tListLen(validFlag); i++) {
+    if (v == validFlag[i]) {
+        return true;
+    } 
+  }
+  return false;
+}
 /*
  * Used during parsing query sql. Since the query sql usually small in length, error position
  * is not needed in the final error message.
@@ -565,16 +576,16 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       }
 
       int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a);
-      SStrToken* t = taosArrayGet(pMiscInfo->a, 0);
+      assert(numOfToken >= 1 && numOfToken <= 2);
-      SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1);
 
+      SStrToken* t = taosArrayGet(pMiscInfo->a, 0);
       strncpy(pCmd->payload, t->z, t->n);
       if (numOfToken == 2) {
+        SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1);
         pCmd->payload[t->n] = ' ';  // add sep
         strncpy(&pCmd->payload[t->n + 1], t1->z, t1->n);
       } 
-
+      return TSDB_CODE_SUCCESS; 
-      break;
     }
 
     case TSDB_SQL_CREATE_TABLE: {
@@ -787,10 +798,10 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ
   }
 
   SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX};
-  tstrncpy(s.name, aAggs[TSDB_FUNC_TS].aName, sizeof(s.name));
+  tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name));
 
   SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
-  tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL);
+  tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL);
 
   if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_TSC_INVALID_SQL;
@@ -1319,7 +1330,7 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr
   return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL;
 }
 
-static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
+void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
   SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
   tscColumnListInsert(pQueryInfo->colList, &tsCol);
 }
@@ -1401,7 +1412,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
     insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr);
 
     // add ts column
-    tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
+    tscInsertPrimaryTsSourceColumn(pQueryInfo, &index);
 
     tbufCloseWriter(&bw);
     taosArrayDestroy(colList);
@@ -1506,7 +1517,7 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) {
   // add the timestamp column into the output columns
   SColumnIndex index = {0};  // primary timestamp column info
   int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
-  tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL);
+  tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL);
 
   SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols);
   pSupInfo->visible = false;
@@ -1602,7 +1613,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
    * in dealing with super table queries such as: count/first/last
    */
   if (isSTable) {
-    tscTansformSQLFuncForSTableQuery(pQueryInfo);
+    tscTansformFuncForSTableQuery(pQueryInfo);
 
     if (hasUnsupportFunctionsForSTableQuery(pCmd, pQueryInfo)) {
       return TSDB_CODE_TSC_INVALID_SQL;
@@ -1656,7 +1667,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tabl
                           (functionId == TSDB_FUNC_TAGPRJ));
 }
 
-SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
+SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
                                   SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) {
   int16_t colId = getNewResColId(pQueryInfo);
 
@@ -1738,7 +1749,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
     }
 
     // add the primary timestamp column even though it is not required by user
-    tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
+    tscInsertPrimaryTsSourceColumn(pQueryInfo, &index);
   } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) {  // simple column projection query
     SColumnIndex index = COLUMN_INDEX_INITIALIZER;
 
@@ -1748,7 +1759,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
 
     SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName);
     SSqlExpr* pExpr =
-        tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC);
+        tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC);
 
     // NOTE: the first parameter is reserved for the tag column id during join query process.
     pExpr->numOfParams = 2;
@@ -1761,11 +1772,11 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
     }
 
     if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
-      SSchema colSchema = tGetTableNameColumnSchema();
+      SSchema* colSchema = tGetTbnameColumnSchema();
-      tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG);
+      tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, colSchema, TSDB_COL_TAG);
     } else if (index.columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) {
       SSchema colSchema = tGetBlockDistColumnSchema();
-      tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_TAG);
+      tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_TAG);
     } else {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
       STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
@@ -1779,7 +1790,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
     }
 
     // add the primary timestamp column even though it is not required by user
-    tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
+    tscInsertPrimaryTsSourceColumn(pQueryInfo, &index);
   } else {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
@@ -1849,9 +1860,9 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT
     if (tsKeepOriginalColumnName) { // keep the original column name
       tstrncpy(name, uname, TSDB_COL_NAME_LEN);
     } else {
-      int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1;
+      int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1;
-      char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0};
+      char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1] = {0};
-      snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname);
+      snprintf(tmp, size, "%s(%s)", aAggs[functionId].name, uname);
 
       tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
     }
@@ -1966,7 +1977,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
       // the time stamp may be always needed
       if (index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) {
-        tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
+        tscInsertPrimaryTsSourceColumn(pQueryInfo, &index);
       }
 
       return TSDB_CODE_SUCCESS;
@@ -2036,7 +2047,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
                                            getNewResColId(pQueryInfo), TSDB_KEYSIZE, false);
 
         SColumnList ids = getColumnList(1, 0, 0);
-        insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName, pExpr);
+        insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr);
       }
 
       // functions can not be applied to tags
@@ -2079,7 +2090,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
         }
       }
 
-      tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
+      tscInsertPrimaryTsSourceColumn(pQueryInfo, &index);
       return TSDB_CODE_SUCCESS;
     }
     case TK_FIRST:
@@ -2285,7 +2296,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
         if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) {
           return TSDB_CODE_TSC_INVALID_SQL;
         }
-        tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
+        tscInsertPrimaryTsSourceColumn(pQueryInfo, &index);
         colIndex += 1;  // the first column is ts
 
         pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false);
@@ -2308,12 +2319,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
         SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
         pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pQueryInfo),
                                  TSDB_KEYSIZE, false);
-        tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].aName, sizeof(pExpr->aliasName));
+        tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->aliasName));
 
         const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX;
         SColumnList   ids = getColumnList(1, 0, TS_COLUMN_INDEX);
         insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP,
-                          aAggs[TSDB_FUNC_TS].aName, pExpr);
+                          aAggs[TSDB_FUNC_TS].name, pExpr);
 
         colIndex += 1;  // the first column is ts
 
@@ -2384,7 +2395,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       
       SSchema s = {0};
       if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
-        s = tGetTableNameColumnSchema();
+        s = *tGetTbnameColumnSchema();
       } else {
         s = pTagSchema[index.columnIndex];
       }
@@ -2400,7 +2411,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       s.bytes = bytes;
 
       TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
-      tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG);
+      tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG);
       
       return TSDB_CODE_SUCCESS;
     }
@@ -2778,7 +2789,7 @@ bool validateIpAddress(const char* ip, size_t size) {
   return epAddr != INADDR_NONE;
 }
 
-int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
+int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
   if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@@ -2800,7 +2811,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
     SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex);
     
     if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) ||
-        (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) ||
+        (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_STDDEV_DST) ||
         (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) {
       if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->param[0].i64, &type, &bytes,
                             &interBytes, 0, true) != TSDB_CODE_SUCCESS) {
@@ -2818,7 +2829,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
 }
 
 /* transfer the field-info back to original input format */
-void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
+void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     return;
@@ -2842,6 +2853,8 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
       functionId = TSDB_FUNC_FIRST;
     } else if (functionId == TSDB_FUNC_LAST_DST) {
       functionId = TSDB_FUNC_LAST;
+    } else if (functionId == TSDB_FUNC_STDDEV_DST) {
+      functionId = TSDB_FUNC_STDDEV;
     }
     
     getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &pExpr->resType, &pExpr->resBytes,
@@ -2858,7 +2871,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo)
   size_t size = tscSqlExprNumOfExprs(pQueryInfo);
   for (int32_t i = 0; i < size; ++i) {
     int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
-    if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_STABLE) == 0) {
+    if ((aAggs[functionId].status & TSDB_FUNCSTATE_STABLE) == 0) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       return true;
     }
@@ -2968,7 +2981,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd)
 
   STableMeta* pTableMeta = NULL;
   SSchema*    pSchema = NULL;
-  SSchema     s = tGetTbnameColumnSchema();
+//  SSchema     s = tGetTbnameColumnSchema();
 
   int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
 
@@ -2995,7 +3008,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd)
   
     int32_t numOfCols = tscGetNumOfColumns(pTableMeta);
     if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
-      pSchema = &s;
+      pSchema = tGetTbnameColumnSchema();
     } else {
       pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
     }
@@ -3440,6 +3453,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) {
 
 static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pTableCond, SStringBuilder* sb) {
   const char* msg0 = "invalid table name list";
+  const char* msg1 = "not string following like";
 
   if (pTableCond == NULL) {
     return TSDB_CODE_SUCCESS;
@@ -3457,6 +3471,10 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   if (pTableCond->nSQLOptr == TK_IN) {
     ret = tablenameListToString(pRight, sb);
   } else if (pTableCond->nSQLOptr == TK_LIKE) {
+    if (pRight->nSQLOptr != TK_STRING) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    }
+    
     ret = tablenameCondToString(pRight, sb);
   }
 
@@ -3547,38 +3565,6 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   return TSDB_CODE_SUCCESS;
 }
 
-// todo error handle / such as and /or mixed with +/-/*/
-int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) {
-  tSQLExpr* pLeft = pExpr->pLeft;
-  tSQLExpr* pRight = pExpr->pRight;
-
-  *(*exprString)++ = '(';
-
-  if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) {
-    doArithmeticExprToString(pLeft, exprString);
-  } else {
-    int32_t ret = tSQLExprNodeToString(pLeft, exprString);
-    if (ret != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_TSC_INVALID_SQL;
-    }
-  }
-
-  optrToString(pExpr, exprString);
-
-  if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) {
-    doArithmeticExprToString(pRight, exprString);
-  } else {
-    int32_t ret = tSQLExprNodeToString(pRight, exprString);
-    if (ret != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_TSC_INVALID_SQL;
-    }
-  }
-
-  *(*exprString)++ = ')';
-
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList,
     int32_t* type, uint64_t* uid) {
   if (pExpr->nSQLOptr == TK_ID) {
@@ -5228,7 +5214,7 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
   size_t size = taosArrayGetSize(pQueryInfo->exprList);
   for (int32_t i = 0; i < size; ++i) {
     int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId;
-    if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) {
+    if (!IS_STREAM_QUERY_VALID(aAggs[functId].status)) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
   }
@@ -5251,7 +5237,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu
       bool hasSelectivity = false;
       for (int32_t j = 0; j < size; ++j) {
         SSqlExpr* pEx = tscSqlExprGet(pQueryInfo, j);
-        if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) {
+        if ((aAggs[pEx->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) {
           hasSelectivity = true;
           break;
         }
@@ -5382,13 +5368,15 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) {
   SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8},    {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12},
                                                       {"cDebugFlag", 10}, {"uDebugFlag", 10},   {"debugFlag", 9}};
 
+
   SStrToken* pOptionToken = taosArrayGet(pOptions->a, 0);
 
   if (numOfToken == 1) {
     // reset log does not need value
     for (int32_t i = 0; i < 1; ++i) {
       SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i];
-      if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) {
+      if ((pOption->len == pOptionToken->n) && 
+              (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) {
         return TSDB_CODE_SUCCESS;
       }
     }
@@ -5396,15 +5384,14 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) {
     SStrToken* pValToken = taosArrayGet(pOptions->a, 1);
 
     int32_t val = strtol(pValToken->z, NULL, 10);
-    if (val < 131 || val > 199) {
+    if (!validateDebugFlag(val)) {
-      // options value is out of valid range
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) {
       SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i];
-      if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) {
+      if ((pOption->len == pOptionToken->n) 
-        // options is valid
+              && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) {
         return TSDB_CODE_SUCCESS;
       }
     }
@@ -5705,7 +5692,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) {
   SSchema*     pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex);
   SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex};
 
-  tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL);
+  tscAddFuncInSelectClause(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL);
 
   int32_t numOfFields = tscNumOfFields(pQueryInfo);
   SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1);
@@ -5869,7 +5856,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
       continue;
     }
 
-    if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
+    if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
       numOfSelectivity++;
     } else {
       numOfAggregation++;
@@ -5901,7 +5888,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
       for (int32_t i = 0; i < numOfExprs; ++i) {
         SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
         int16_t functionId = pExpr->functionId;
-        if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) {
+        if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == 0) {
           continue;
         }
 
@@ -5944,7 +5931,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  SSchema s = tGetTableNameColumnSchema();
+  SSchema s = *tGetTbnameColumnSchema();
   SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
   int16_t  bytes = 0;
   int16_t  type = 0;
@@ -6088,7 +6075,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
         }
       }
 
-      if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM &&
+      if (IS_MULTIOUTPUT(aAggs[functId].status) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM &&
           functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_PRJ) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
       }
@@ -6278,7 +6265,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) {
     char    tmpBuf[1024] = {0};
     int32_t tmpLen = 0;
     tmpLen =
-        sprintf(tmpBuf, "%s(uid:%" PRId64 ", %d)", aAggs[pExpr->functionId].aName, pExpr->uid, pExpr->colInfo.colId);
+        sprintf(tmpBuf, "%s(uid:%" PRId64 ", %d)", aAggs[pExpr->functionId].name, pExpr->uid, pExpr->colInfo.colId);
 
     if (tmpLen + offset >= totalBufSize - 1) break;
 
@@ -6979,3 +6966,4 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) {
 
   return false;
 }
+

src/client/src/tscServer.c

@@ -752,6 +752,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->queryType      = htonl(pQueryInfo->type);
   pQueryMsg->vgroupLimit    = htobe64(pQueryInfo->vgroupLimit);
   pQueryMsg->sqlstrLen      = htonl(sqlLen);
+  pQueryMsg->prevResultLen  = htonl(pQueryInfo->bufLen);
 
   size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
   pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);  // this is the stage one output column number
@@ -989,6 +990,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     }
   }
 
+  if (pQueryInfo->bufLen > 0) {
+    memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen);
+    pMsg += pQueryInfo->bufLen;
+  }
+
   SCond* pCond = &pQueryInfo->tagCond.tbnameCond;
   if (pCond->len > 0) {
     strncpy(pMsg, pCond->cond, pCond->len);

src/client/src/tscSql.c

@@ -443,24 +443,6 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   return pFieldInfo->final;
 }
 
-int taos_retrieve(TAOS_RES *res) {
-  if (res == NULL) return 0;
-  SSqlObj *pSql = (SSqlObj *)res;
-  SSqlCmd *pCmd = &pSql->cmd;
-  SSqlRes *pRes = &pSql->res;
-  if (pSql == NULL || pSql->signature != pSql) return 0;
-  if (pRes->qhandle == 0) return 0;
-
-  tscResetForNextRetrieve(pRes);
-
-  if (pCmd->command < TSDB_SQL_LOCAL) {
-    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
-  }
-
-  tscProcessSql(pSql);
-  return pRes->numOfRows;
-}
-
 static bool needToFetchNewBlock(SSqlObj* pSql) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;

src/client/src/tscStream.c

@@ -103,7 +103,7 @@ static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) {
 
   // failed to get table Meta or vgroup list, retry in 10sec.
   if (code == TSDB_CODE_SUCCESS) {
-    tscTansformSQLFuncForSTableQuery(pQueryInfo);
+    tscTansformFuncForSTableQuery(pQueryInfo);
     tscDebug("%p stream:%p, start stream query on:%s", pSql, pStream, tNameGetTableName(&pTableMetaInfo->name));
 
     pSql->fp = tscProcessStreamQueryCallback;

src/client/src/tscSub.c

@@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p == NULL) {
     return dflt;
   }
@@ -76,7 +76,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = ts};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p != NULL) {
     p->key = ts;
     tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key);
@@ -270,7 +270,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
     SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
-    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
       taosArrayPush(pSub->progress, &target);

src/client/src/tscSubquery.c

@@ -23,6 +23,7 @@
 #include "tscSubquery.h"
 #include "tschemautil.h"
 #include "tsclient.h"
+#include "qUtil.h"
 
 typedef struct SInsertSupporter {
   SSqlObj*  pSql;
@@ -501,7 +502,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
 
       int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS;
 
-      tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL);
+      tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL);
       tscPrintSelectClause(pNew, 0);
       tscFieldInfoUpdateOffset(pQueryInfo);
 
@@ -681,7 +682,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
   }
 }
 
-static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
+static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
   SSqlCmd* pCmd = &pSql->cmd;
   tscClearSubqueryInfo(pCmd);
   tscFreeSqlResult(pSql);
@@ -701,7 +702,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
   SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1};
   
   SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
-  tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL);
+  tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL);
   
   // set the tags value for ts_comp function
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@@ -970,7 +971,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
     
     for (int32_t m = 0; m < pParentSql->subState.numOfSub; ++m) {
       SSqlObj* sub = pParentSql->pSubs[m];
-      issueTSCompQuery(sub, sub->param, pParentSql);
+      issueTsCompQuery(sub, sub->param, pParentSql);
     }
   }
 
@@ -1470,7 +1471,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
   }
 
   // restore the offset value for super table query in case of final result.
-  tscRestoreSQLFuncForSTableQuery(pQueryInfo);
+  tscRestoreFuncForSTableQuery(pQueryInfo);
   tscFieldInfoUpdateOffset(pQueryInfo);
 }
 
@@ -1651,7 +1652,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
 
       // set get tags query type
       TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
-      tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG);
+      tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG);
       size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
   
       tscDebug(
@@ -1662,7 +1663,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     } else {
       SSchema      colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1};
       SColumnIndex colIndex = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
-      tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL);
+      tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL);
 
       // set the tags value for ts_comp function
       SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0);
@@ -1821,7 +1822,262 @@ void tscUnlockByThread(int64_t *lockedBy) {
   }
 }
 
+typedef struct SFirstRoundQuerySup {
+  SSqlObj  *pParent;
+  int32_t   numOfRows;
+  SArray   *pColsInfo;
+  int32_t   tagLen;
+  STColumn *pTagCols;
+  SArray   *pResult;   // SArray<SInterResult>
+  int64_t   interval;
+  char*     buf;
+  int32_t   bufLen;
+} SFirstRoundQuerySup;
 
+void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, SQueryInfo* pQueryInfo) {
+  TSKEY key = INT64_MIN;
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) {
+      continue;
+    }
+
+    if (pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+      key = *(TSKEY*) row[i];
+      continue;
+    }
+
+    double v = 0;
+    if (row[i] != NULL) {
+      v = *(double*) row[i];
+    } else {
+      SET_DOUBLE_NULL(&v);
+    }
+
+    int32_t id = pExpr->colInfo.colId;
+    int32_t numOfQueriedCols = (int32_t) taosArrayGetSize(pInterResult->pResult);
+
+    SArray* p = NULL;
+    for(int32_t j = 0; j < numOfQueriedCols; ++j) {
+      SStddevInterResult* pColRes = taosArrayGet(pInterResult->pResult, j);
+      if (pColRes->colId == id) {
+        p = pColRes->pResult;
+        break;
+      }
+    }
+
+    //append a new column
+    if (p == NULL) {
+      SStddevInterResult t = {.colId = id, .pResult = taosArrayInit(10, sizeof(SResPair)),};
+      taosArrayPush(pInterResult->pResult, &t);
+      p = t.pResult;
+    }
+
+    SResPair pair = {.avg = v, .key = key};
+    taosArrayPush(p, &pair);
+  }
+}
+
+void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
+  SSqlObj* pSql = (SSqlObj*)tres;
+  SSqlRes* pRes = &pSql->res;
+
+  SFirstRoundQuerySup* pSup = param;
+  SQueryInfo*          pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
+
+  if (numOfRows > 0) {
+    TAOS_ROW row = NULL;
+    int32_t  numOfCols = taos_field_count(tres);
+
+    if (pSup->tagLen == 0) {  // no tags, all rows belong to one group
+      SInterResult interResult = {.tags = NULL, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))};
+      taosArrayPush(pSup->pResult, &interResult);
+
+      while ((row = taos_fetch_row(tres)) != NULL) {
+        doAppendData(&interResult, row, numOfCols, pQueryInfo);
+      }
+    } else {  // tagLen > 0
+      char* p = calloc(1, pSup->tagLen);
+
+      while ((row = taos_fetch_row(tres)) != NULL) {
+        int32_t* length = taos_fetch_lengths(tres);
+        memset(p, 0, pSup->tagLen);
+
+        int32_t offset = 0;
+        for (int32_t i = 0; i < numOfCols && offset < pSup->tagLen; ++i) {
+          SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+          if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) {
+            memcpy(p + offset, row[i], length[i]);
+            offset += pExpr->resBytes;
+          }
+        }
+
+        assert(offset == pSup->tagLen);
+        size_t size = taosArrayGetSize(pSup->pResult);
+
+        if (size > 0) {
+          SInterResult* pInterResult = taosArrayGetLast(pSup->pResult);
+          if (memcmp(pInterResult->tags, p, pSup->tagLen) == 0) {  // belongs to the same group
+            doAppendData(pInterResult, row, numOfCols, pQueryInfo);
+          } else {
+            char* tags = malloc( pSup->tagLen);
+            memcpy(tags, p, pSup->tagLen);
+
+            SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))};
+            taosArrayPush(pSup->pResult, &interResult);
+            doAppendData(&interResult, row, numOfCols, pQueryInfo);
+          }
+        } else {
+          char* tags = malloc(pSup->tagLen);
+          memcpy(tags, p, pSup->tagLen);
+
+          SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))};
+          taosArrayPush(pSup->pResult, &interResult);
+          doAppendData(&interResult, row, numOfCols, pQueryInfo);
+        }
+      }
+
+      tfree(p);
+    }
+  }
+
+  pSup->numOfRows += numOfRows;
+  if (!pRes->completed) {
+    taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param);
+    return;
+  }
+
+  // set the parameters for the second round query process
+  SSqlObj    *pParent = pSup->pParent;
+  SSqlCmd    *pPCmd   = &pParent->cmd;
+  SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(pPCmd, 0);
+
+  if (pSup->numOfRows > 0) {
+    SBufferWriter bw = tbufInitWriter(NULL, false);
+    interResToBinary(&bw, pSup->pResult, pSup->tagLen);
+
+    pQueryInfo1->bufLen = (int32_t) tbufTell(&bw);
+    pQueryInfo1->buf = tbufGetData(&bw, true);
+
+    // set the serialized binary string as the parameter of arithmetic expression
+    tbufCloseWriter(&bw);
+  }
+
+  taosArrayDestroyEx(pSup->pResult, freeInterResult);
+  taosArrayDestroy(pSup->pColsInfo);
+  tfree(pSup);
+
+  taos_free_result(pSql);
+
+  pQueryInfo1->round = 1;
+  tscDoQuery(pParent);
+}
+
+void tscFirstRoundCallback(void* param, TAOS_RES* tres, int code) {
+  int32_t c = taos_errno(tres);
+  if (c != TSDB_CODE_SUCCESS) {
+    // TODO HANDLE ERROR
+  }
+
+  taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param);
+}
+
+int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) {
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
+  STableMetaInfo* pTableMetaInfo1 = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
+
+  SFirstRoundQuerySup *pSup = calloc(1, sizeof(SFirstRoundQuerySup));
+
+  pSup->pParent  = pSql;
+  pSup->interval = pQueryInfo->interval.interval;
+  pSup->pResult  = taosArrayInit(6, sizeof(SStddevInterResult));
+  pSup->pColsInfo = taosArrayInit(6, sizeof(int16_t)); // result column id
+
+  SSqlObj *pNew = createSubqueryObj(pSql, 0, tscFirstRoundCallback, pSup, TSDB_SQL_SELECT, NULL);
+  SSqlCmd *pCmd = &pNew->cmd;
+
+  tscClearSubqueryInfo(pCmd);
+  tscFreeSqlResult(pSql);
+
+  SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+  assert(pQueryInfo->numOfTables == 1);
+
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
+
+  tscInitQueryInfo(pNewQueryInfo);
+  pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr;
+  if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
+    pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo);
+    if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) {
+      terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+//      goto _error;
+    }
+  }
+
+  if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+//    goto _error;
+  }
+
+  pNewQueryInfo->interval = pQueryInfo->interval;
+
+  pCmd->command = TSDB_SQL_SELECT;
+  pNew->fp = tscFirstRoundCallback;
+
+  int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+
+  int32_t index = 0;
+  int32_t numOfTags = 0;
+  for(int32_t i = 0; i < numOfExprs; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    if (pExpr->functionId == TSDB_FUNC_TS && pQueryInfo->interval.interval > 0) {
+      taosArrayPush(pSup->pColsInfo, &pExpr->resColId);
+
+      SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
+      SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId);
+
+      SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL);
+      p->resColId = pExpr->resColId;  // update the result column id
+    } else if (pExpr->functionId == TSDB_FUNC_STDDEV_DST) {
+      taosArrayPush(pSup->pColsInfo, &pExpr->resColId);
+
+      SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->colInfo.colIndex};
+      SSchema schema = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)};
+      tstrncpy(schema.name, pExpr->aliasName, tListLen(schema.name));
+
+      SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL);
+      p->resColId = pExpr->resColId;  // update the result column id
+    } else if (pExpr->functionId == TSDB_FUNC_TAG) {
+      pSup->tagLen += pExpr->resBytes;
+      SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->colInfo.colIndex};
+
+      SSchema* schema = NULL;
+      if (pExpr->colInfo.colId != TSDB_TBNAME_COLUMN_INDEX) {
+        schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId);
+      } else {
+        schema = tGetTbnameColumnSchema();
+      }
+
+      SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG);
+      p->resColId = pExpr->resColId;
+      numOfTags += 1;
+    }
+  }
+
+  SColumnIndex columnIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
+  tscInsertPrimaryTsSourceColumn(pNewQueryInfo, &columnIndex);
+
+  tscTansformFuncForSTableQuery(pNewQueryInfo);
+
+  tscDebug(
+      "%p first round subquery:%p tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, query to retrieve timestamps, "
+      "numOfExpr:%" PRIzu ", colList:%d, numOfOutputFields:%d, name:%s",
+      pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pNewQueryInfo->type,
+      tscSqlExprNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name));
+
+  tscHandleMasterSTableQuery(pNew);
+  return TSDB_CODE_SUCCESS;
+}
 
 int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
@@ -1833,7 +2089,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
     return pRes->code;
   }
   
-  tExtMemBuffer **  pMemoryBuf = NULL;
+  tExtMemBuffer   **pMemoryBuf = NULL;
   tOrderDescriptor *pDesc  = NULL;
   SColumnModel     *pModel = NULL;
   SColumnModel     *pFinalModel = NULL;
@@ -1863,10 +2119,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
     return ret;
   }
   
-  pSql->pSubs = calloc(pState->numOfSub, POINTER_BYTES);
-
   tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub);
-
+  pSql->pSubs = calloc(pState->numOfSub, POINTER_BYTES);
   if (pSql->pSubs == NULL) {
     tfree(pSql->pSubs);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -2410,7 +2664,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
 
   // record the total inserted rows
   if (numOfRows > 0) {
-    pParentObj->res.numOfRows += numOfRows;
+    atomic_add_fetch_32(&pParentObj->res.numOfRows, numOfRows);
   }
 
   if (taos_errno(tres) != TSDB_CODE_SUCCESS) {
@@ -2739,7 +2993,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
       return;
     }
 
-    tscRestoreSQLFuncForSTableQuery(pQueryInfo);
+    tscRestoreFuncForSTableQuery(pQueryInfo);
   }
 
   assert (pRes->row >= pRes->numOfRows);

src/client/src/tscSystem.c

@@ -18,7 +18,6 @@
 #include "tref.h"
 #include "trpc.h"
 #include "tnote.h"
-#include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
 #include "tsched.h"
@@ -49,7 +48,7 @@ int32_t   tscNumOfThreads = 1;     // num of rpc threads
 static    pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently 
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 
-void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
+void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) {
   taosGetDisk();
   taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
 }
@@ -65,7 +64,7 @@ void tscReleaseRpc(void *param)  {
     return;
   }
   pthread_mutex_lock(&rpcObjMutex);
-  taosCacheRelease(tscRpcCache, (void *)&param, false); 
+  taosCacheRelease(tscRpcCache, (void *)&param, true); 
   pthread_mutex_unlock(&rpcObjMutex);
 } 
 
@@ -88,7 +87,7 @@ int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncry
   rpcInit.sessions = tsMaxConnections;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.user = (char *)user;
-  rpcInit.idleTime = 2000; 
+  rpcInit.idleTime = tsShellActivityTimer * 1000; 
   rpcInit.ckey = "key"; 
   rpcInit.spi = 1; 
   rpcInit.secret = (char *)secretEncrypt;
@@ -216,7 +215,6 @@ void taos_cleanup(void) {
   taosCloseRef(id);
 
   taosCleanupKeywordsTable();
-  taosCloseLog();
 
   p = tscRpcCache; 
   tscRpcCache = NULL;
@@ -226,7 +224,10 @@ void taos_cleanup(void) {
     pthread_mutex_destroy(&rpcObjMutex);
   }
 
-  if (tscEmbedded == 0) rpcCleanup();
+  if (tscEmbedded == 0) {
+    rpcCleanup();
+    taosCloseLog();
+  };
 
   p = tscTmr;
   tscTmr = NULL;

src/client/src/tscUtil.c

@@ -107,11 +107,6 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
     return false;
   }
   
-  // for select query super table, the super table vgroup list can not be null in any cases.
-  // if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
-  //   assert(pTableMetaInfo->vgroupList != NULL);
-  // }
-  
   if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) {
     return false;
   }
@@ -1074,7 +1069,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
   memset(pFieldInfo, 0, sizeof(SFieldInfo));
 }
 
-static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
+static SSqlExpr* doCreateSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
     int16_t size, int16_t resColId, int16_t interSize, int32_t colType) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex);
   
@@ -1127,14 +1122,14 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi
     return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
   }
   
-  SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
+  SSqlExpr* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
   taosArrayInsert(pQueryInfo->exprList, index, &pExpr);
   return pExpr;
 }
 
 SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
     int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) {
-  SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
+  SSqlExpr* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
   taosArrayPush(pQueryInfo->exprList, &pExpr);
   return pExpr;
 }
@@ -1158,6 +1153,22 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi
   return pExpr;
 }
 
+bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) {
+  if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) {
+    return false;
+  }
+
+  int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+  for(int32_t i = 0; i < numOfExprs; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    if (pExpr->functionId == TSDB_FUNC_STDDEV_DST) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) {
   return taosArrayGetSize(pQueryInfo->exprList);
 }
@@ -1422,9 +1433,9 @@ int32_t tscValidateName(SStrToken* pToken) {
   if (sep == NULL) {  // single part
     if (pToken->type == TK_STRING) {
        
-      strdequote(pToken->z);
+      tscDequoteAndTrimToken(pToken);
       tscStrToLower(pToken->z, pToken->n);
-      pToken->n = (uint32_t)strtrim(pToken->z);
+      //pToken->n = (uint32_t)strtrim(pToken->z);
        
       int len = tSQLGetToken(pToken->z, &pToken->type);
 
@@ -1762,6 +1773,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
   pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf);
 
   tfree(pQueryInfo->fillVal);
+  tfree(pQueryInfo->buf);
 }
 
 void tscClearSubqueryInfo(SSqlCmd* pCmd) {
@@ -2068,6 +2080,17 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
   pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit;
   pNewQueryInfo->numOfTables = 0;
   pNewQueryInfo->pTableMetaInfo = NULL;
+  pNewQueryInfo->bufLen = pQueryInfo->bufLen;
+
+  pNewQueryInfo->buf = malloc(pQueryInfo->bufLen);
+  if (pNewQueryInfo->buf == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto _error;
+  }
+
+  if (pQueryInfo->bufLen > 0) {
+    memcpy(pNewQueryInfo->buf, pQueryInfo->buf, pQueryInfo->bufLen);
+  }
 
   pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr;
   if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
@@ -2234,6 +2257,9 @@ void tscDoQuery(SSqlObj* pSql) {
         }
       }
 
+      return;
+    } else if (tscMultiRoundQuery(pQueryInfo, 0) && pQueryInfo->round == 0) {
+      tscHandleFirstRoundStableQuery(pSql);  // todo lock?
       return;
     } else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {  // super table query
       tscLockByThread(&pSql->squeryLock);

src/common/inc/tdataformat.h

@@ -68,9 +68,9 @@ typedef struct {
 typedef struct {
   int      version;    // version
   int      numOfCols;  // Number of columns appended
-  int      tlen;       // maximum length of a SDataRow without the header part
+  int      tlen;       // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
   uint16_t flen;       // First part length in a SDataRow after the header part
-  uint16_t vlen;       // pure value part length, excluded the overhead
+  uint16_t vlen;       // pure value part length, excluded the overhead (bytes only)
   STColumn columns[];
 } STSchema;
 
@@ -134,6 +134,22 @@ typedef uint64_t TKEY;
 #define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
 #define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
 
+#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
+#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff)
+
+#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
+
+static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
+  TSKEY lkey = key;
+  if (key > MAX_TS_KEY) {
+    lkey = MAX_TS_KEY;
+  } else if (key < MIN_TS_KEY) {
+    lkey = MIN_TS_KEY;
+  }
+
+  return tdGetTKEY(lkey);
+}
+
 static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
   TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
   TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
@@ -278,7 +294,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
 int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
-void       tdFreeDataCols(SDataCols *pCols);
+SDataCols *tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
 

src/common/inc/tglobal.h

@@ -16,6 +16,8 @@
 #ifndef TDENGINE_COMMON_GLOBAL_H
 #define TDENGINE_COMMON_GLOBAL_H
 
+#include "taosdef.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -147,7 +149,6 @@ extern char    tsDataDir[];
 extern char    tsLogDir[];
 extern char    tsScriptDir[];
 extern int64_t tsMsPerDay[3];
-extern char    tsVnodeBakDir[];
 
 // system info
 extern char    tsOsName[];
@@ -196,6 +197,14 @@ extern int32_t wDebugFlag;
 extern int32_t cqDebugFlag;
 extern int32_t debugFlag;
 
+typedef struct {
+  char dir[TSDB_FILENAME_LEN];
+  int  level;
+  int  primary;
+} SDiskCfg;
+extern int32_t  tsDiskCfgNum;
+extern SDiskCfg tsDiskCfg[];
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 void    taosInitGlobalCfg();
@@ -204,6 +213,9 @@ void    taosSetAllDebugFlag();
 bool    taosCfgDynamicOptions(char *msg);
 int     taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
 bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
+void    taosAddDataDir(int index, char *v1, int level, int primary);
+void    taosReadDataDirCfg(char *v1, char *v2, char *v3);
+void    taosPrintDataDirCfg();
 
 #ifdef __cplusplus
 }

src/common/inc/tname.h

@@ -36,6 +36,11 @@ typedef struct SColumnInfoData {
   void* pData;    // the corresponding block data in memory
 } SColumnInfoData;
 
+typedef struct SResPair {
+  TSKEY  key;
+  double avg;
+} SResPair;
+
 #define TSDB_DB_NAME_T     1
 #define TSDB_TABLE_NAME_T  2
 
@@ -58,7 +63,7 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len);
 
 void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
 
-SSchema tGetTableNameColumnSchema();
+//SSchema tGetTbnameColumnSchema();
 
 SSchema tGetBlockDistColumnSchema();
 
@@ -68,7 +73,7 @@ bool tscValidateTableNameLength(size_t len);
 
 SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters);
 
-SSchema tGetTbnameColumnSchema();
+SSchema* tGetTbnameColumnSchema();
 
 /**
  * check if the schema is valid or not, including following aspects:

src/common/src/tdataformat.c

@@ -289,24 +289,32 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
     return NULL;
   }
 
+  pCols->maxPoints = maxRows;
+
+  if (maxCols > 0) {
     pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
     if (pCols->cols == NULL) {
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
+             strerror(errno));
       tdFreeDataCols(pCols);
       return NULL;
     }
 
-  pCols->maxRowSize = maxRowSize;
     pCols->maxCols = maxCols;
-  pCols->maxPoints = maxRows;
+  }
+
+  pCols->maxRowSize = maxRowSize;
   pCols->bufSize = maxRowSize * maxRows;
 
+  if (pCols->bufSize > 0) {
     pCols->buf = malloc(pCols->bufSize);
     if (pCols->buf == NULL) {
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
+             strerror(errno));
       tdFreeDataCols(pCols);
       return NULL;
     }
+  }
 
   return pCols;
 }
@@ -337,12 +345,13 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
   return 0;
 }
 
-void tdFreeDataCols(SDataCols *pCols) {
+SDataCols *tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
     tfree(pCols->buf);
     tfree(pCols->cols);
     free(pCols);
   }
+  return NULL;
 }
 
 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {

src/common/src/texpr.c

@@ -407,7 +407,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
   SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
   left->pSchema = pSchema;
 
-  *pSchema = tGetTbnameColumnSchema();
+  *pSchema = *tGetTbnameColumnSchema();
 
   tExprNode* right = exception_calloc(1, sizeof(tExprNode));
   expr->_node.pRight = right;

src/common/src/tglobal.c

@@ -59,7 +59,6 @@ char    tsLocale[TSDB_LOCALE_LEN] = {0};
 char    tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
 int8_t  tsEnableCoreFile = 0;
 int32_t tsMaxBinaryDisplayWidth = 30;
-char    tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
 
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
@@ -182,7 +181,15 @@ char   tsDnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsMnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsDataDir[TSDB_FILENAME_LEN] = {0};
 char   tsScriptDir[TSDB_FILENAME_LEN] = {0};
-char   tsVnodeBakDir[TSDB_FILENAME_LEN] = {0};
+char   tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
+
+int32_t  tsDiskCfgNum = 0;
+
+#ifndef _STORAGE
+SDiskCfg tsDiskCfg[1];
+#else
+SDiskCfg tsDiskCfg[TSDB_MAX_DISKS];
+#endif
 
 /*
  * minimum scale for whole system, millisecond by default
@@ -227,6 +234,7 @@ int32_t sDebugFlag = 135;
 int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 131;
+int32_t fsDebugFlag = 135;
 
 int32_t (*monStartSystemFp)() = NULL;
 void (*monStopSystemFp)() = NULL;
@@ -265,7 +273,7 @@ bool taosCfgDynamicOptions(char *msg) {
   int32_t   vint = 0;
 
   paGetToken(msg, &option, &olen);
-  if (olen == 0) return TSDB_CODE_COM_INVALID_CFG_MSG;
+  if (olen == 0) return false;;
 
   paGetToken(option + olen + 1, &value, &vlen);
   if (vlen == 0)
@@ -308,11 +316,9 @@ bool taosCfgDynamicOptions(char *msg) {
       }
       return true;
     }
-
     if (strncasecmp(cfg->option, "debugFlag", olen) == 0) {
        taosSetAllDebugFlag(); 
     }
-    
     return true;
   }
 
@@ -334,6 +340,39 @@ bool taosCfgDynamicOptions(char *msg) {
   return false;
 }
 
+void taosAddDataDir(int index, char *v1, int level, int primary) {
+  tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
+  tsDiskCfg[index].level = level;
+  tsDiskCfg[index].primary = primary;
+  uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
+}
+
+#ifndef _STORAGE
+void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
+  if (tsDiskCfgNum == 1) {
+    SDiskCfg *cfg = &tsDiskCfg[0];
+    uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
+  }
+  taosAddDataDir(0, v1, 0, 1);
+  tsDiskCfgNum = 1;
+}
+
+void taosPrintDataDirCfg() {
+  for (int i = 0; i < tsDiskCfgNum; ++i) {
+    SDiskCfg *cfg = &tsDiskCfg[i];
+    uInfo(" dataDir: %s", cfg->dir);
+  }
+}
+#endif
+
+static void taosCheckDataDirCfg() {
+  if (tsDiskCfgNum <= 0) {
+    taosAddDataDir(0, tsDataDir, 0, 1);
+    tsDiskCfgNum = 1;
+    uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir);
+  }
+}
+
 static void doInitGlobalConfig(void) {
   osInit();
   srand(taosSafeRand());
@@ -415,7 +454,7 @@ static void doInitGlobalConfig(void) {
 
   cfg.option = "dataDir";
   cfg.ptr = tsDataDir;
-  cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
+  cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -1448,6 +1487,7 @@ int32_t taosCheckGlobalCfg() {
     snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port);
   }
 
+  taosCheckDataDirCfg();
   taosGetSystemInfo();
 
   tsSetLocale();

src/common/src/tname.c

@@ -10,6 +10,7 @@
 
 #define VALID_NAME_TYPE(x)  ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T)
 
+//TODO remove it
 void extractTableName(const char* tableId, char* name) {
   size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]);
   size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]);
@@ -24,6 +25,7 @@ char* extractDBName(const char* tableId, char* name) {
   return strncpy(name, &tableId[offset1 + 1], len);
 }
 
+// todo remove it
 size_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
   tstrncpy(prefix, name, len);
   strcat(prefix, TS_PATH_DELIMITER);
@@ -31,14 +33,6 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
   return strlen(prefix);
 }
 
-SSchema tGetTableNameColumnSchema() {
-  SSchema s = {0};
-  s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE;
-  s.type  = TSDB_DATA_TYPE_BINARY;
-  s.colId = TSDB_TBNAME_COLUMN_INDEX;
-  tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN);
-  return s;
-}
 SSchema tGetBlockDistColumnSchema() {
   SSchema s = {0};
   s.bytes = TSDB_MAX_BINARY_LEN;;
@@ -189,15 +183,15 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
   }
 }
 
-SSchema tGetTbnameColumnSchema() {
+static struct SSchema _s = {
-  struct SSchema s = {
     .colId = TSDB_TBNAME_COLUMN_INDEX,
     .type  = TSDB_DATA_TYPE_BINARY,
-      .bytes = TSDB_TABLE_NAME_LEN
+    .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
-  };
+    .name = TSQL_TBNAME_L,
+};
 
-  strcpy(s.name, TSQL_TBNAME_L);
+SSchema* tGetTbnameColumnSchema() {
-  return s;
+  return &_s;
 }
 
 static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) {

src/common/src/tvariant.c

@@ -86,43 +86,53 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->i64 = GET_INT8_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_UTINYINT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->u64 = GET_UINT8_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_SMALLINT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->i64 = GET_INT16_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_USMALLINT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->u64 = GET_UINT16_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_INT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->i64 = GET_INT32_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_UINT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->u64 = GET_UINT32_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->i64 = GET_INT64_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_UBIGINT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->u64 = GET_UINT64_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_DOUBLE: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->dKey = GET_DOUBLE_VAL(pz);
       break;
     }
     case TSDB_DATA_TYPE_FLOAT: {
+      pVar->nLen = tDataTypes[type].bytes;
       pVar->dKey = GET_FLOAT_VAL(pz);
       break;
     }
@@ -144,6 +154,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
     
     default:
       pVar->i64 = GET_INT32_VAL(pz);
+      pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
   }
   
   pVar->nType = type;

src/connector/jdbc/pom.xml

@@ -36,6 +36,7 @@
         <maven-compiler-plugin.version>3.6.0</maven-compiler-plugin.version>
         <commons-logging.version>1.1.2</commons-logging.version>
         <commons-lang3.version>3.5</commons-lang3.version>
+        <maven.test.jvmargs></maven.test.jvmargs>
     </properties>
     <dependencies>
         <dependency>
@@ -122,11 +123,14 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.12.4</version>
                 <configuration>
+                    <forkMode>pertest</forkMode>
+                    <argLine>${maven.test.jvmargs}</argLine>
                     <includes>
                         <include>**/*Test.java</include>
                     </includes>
                     <excludes>
                         <exclude>**/AppMemoryLeakTest.java</exclude>
+                        <exclude>**/TaosInfoMonitorTest.java</exclude>
                         <exclude>**/FailOverTest.java</exclude>
                     </excludes>
                     <testFailureIgnore>true</testFailureIgnore>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java

@@ -34,13 +34,12 @@ import java.util.*;
 import java.util.concurrent.Executor;
 
 public class TSDBConnection implements Connection {
-    protected Properties props = null;
 
     private TSDBJNIConnector connector = null;
 
     private String catalog = null;
 
-    private TSDBDatabaseMetaData dbMetaData = null;
+    private TSDBDatabaseMetaData dbMetaData;
 
     private Properties clientInfoProps = new Properties();
 
@@ -65,13 +64,7 @@ public class TSDBConnection implements Connection {
     private void connect(String host, int port, String dbName, String user, String password) throws SQLException {
         this.connector = new TSDBJNIConnector();
         this.connector.connect(host, port, dbName, user, password);
-
-        try {
         this.setCatalog(dbName);
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
-
         this.dbMetaData.setConnection(this);
     }
 
@@ -80,68 +73,86 @@ public class TSDBConnection implements Connection {
     }
 
     public Statement createStatement() throws SQLException {
-        if (!this.connector.isClosed()) {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+
         TSDBStatement statement = new TSDBStatement(this, this.connector);
         statement.setConnection(this);
         return statement;
-        } else {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
-        }
     }
 
     public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException {
-        if (this.connector.isClosed()) {
+        if (isClosed()) {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
         }
 
         long id = this.connector.subscribe(topic, sql, restart, 0);
         if (id == 0) {
-            throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription"));
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED);
         }
-
         return new TSDBSubscribe(this.connector, id);
     }
 
     public PreparedStatement prepareStatement(String sql) throws SQLException {
-        if (!this.connector.isClosed()) {
+        if (isClosed()) {
-            return new TSDBPreparedStatement(this, this.connector, sql);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
-        } else {
-            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
         }
+
+        return new TSDBPreparedStatement(this, this.connector, sql);
     }
 
     public CallableStatement prepareCall(String sql) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public String nativeSQL(String sql) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void setAutoCommit(boolean autoCommit) throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+
     }
 
     public boolean getAutoCommit() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+
         return true;
     }
 
     public void commit() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
     }
 
     public void rollback() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void close() throws SQLException {
-        if (this.connector != null && !this.connector.isClosed()) {
+        if (isClosed()) {
-            this.connector.closeConnection();
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
-        } else {
-            throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!"));
         }
+        this.connector.closeConnection();
     }
 
     public boolean isClosed() throws SQLException {
-        return this.connector.isClosed();
+        return this.connector != null && this.connector.isClosed();
     }
 
     /**
@@ -154,6 +165,9 @@ public class TSDBConnection implements Connection {
      * @throws SQLException if a database access error occurs
      */
     public DatabaseMetaData getMetaData() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return this.dbMetaData;
     }
 
@@ -165,17 +179,29 @@ public class TSDBConnection implements Connection {
      * @throws SQLException
      */
     public void setReadOnly(boolean readOnly) throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
     }
 
     public boolean isReadOnly() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return true;
     }
 
     public void setCatalog(String catalog) throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         this.catalog = catalog;
     }
 
     public String getCatalog() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return this.catalog;
     }
 
@@ -187,6 +213,19 @@ public class TSDBConnection implements Connection {
      * @throws SQLException
      */
     public void setTransactionIsolation(int level) throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        switch (level) {
+            case Connection.TRANSACTION_NONE:
+            case Connection.TRANSACTION_READ_COMMITTED:
+            case Connection.TRANSACTION_READ_UNCOMMITTED:
+            case Connection.TRANSACTION_REPEATABLE_READ:
+            case Connection.TRANSACTION_SERIALIZABLE:
+                break;
+            default:
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE);
+        }
     }
 
     /**
@@ -196,33 +235,43 @@ public class TSDBConnection implements Connection {
      * @throws SQLException
      */
     public int getTransactionIsolation() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return Connection.TRANSACTION_NONE;
     }
 
     public SQLWarning getWarnings() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         //todo: implement getWarnings according to the warning messages returned from TDengine
         return null;
-//        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
     }
 
     public void clearWarnings() throws SQLException {
-        // left blank to support HikariCP connection
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         //todo: implement clearWarnings according to the warning messages returned from TDengine
     }
 
     public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency)
             throws SQLException {
         // This method is implemented in the current way to support Spark
         if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) {
-            throw new SQLException(TSDBConstants.INVALID_VARIABLES);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE);
         }
 
         if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) {
-            throw new SQLException(TSDBConstants.INVALID_VARIABLES);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE);
         }
 
         return this.prepareStatement(sql);
@@ -237,19 +286,30 @@ public class TSDBConnection implements Connection {
     }
 
     public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public Map<String, Class<?>> getTypeMap() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void setTypeMap(Map<String, Class<?>> map) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void setHoldability(int holdability) throws SQLException {
-        // intentionally left empty to support druid connection pool.
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
     }
 
     /**
@@ -259,67 +319,111 @@ public class TSDBConnection implements Connection {
      * @throws SQLException
      */
     public int getHoldability() throws SQLException {
-        //intentionally left empty to support HikariCP connection.
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return ResultSet.HOLD_CURSORS_OVER_COMMIT;
     }
 
     public Savepoint setSavepoint() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public Savepoint setSavepoint(String name) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void rollback(Savepoint savepoint) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void releaseSavepoint(Savepoint savepoint) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability)
             throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency,
                                               int resultSetHoldability) throws SQLException {
-        return this.prepareStatement(sql, resultSetType, resultSetConcurrency);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency,
                                          int resultSetHoldability) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public Clob createClob() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public Blob createBlob() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public NClob createNClob() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public SQLXML createSQLXML() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public boolean isValid(int timeout) throws SQLException {
@@ -338,31 +442,52 @@ public class TSDBConnection implements Connection {
     }
 
     public String getClientInfo(String name) throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return clientInfoProps.getProperty(name);
     }
 
     public Properties getClientInfo() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return clientInfoProps;
     }
 
     public Array createArrayOf(String typeName, Object[] elements) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public Struct createStruct(String typeName, Object[] attributes) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void setSchema(String schema) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public String getSchema() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void abort(Executor executor) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD);
     }
 
     public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException {
@@ -370,14 +495,21 @@ public class TSDBConnection implements Connection {
     }
 
     public int getNetworkTimeout() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED);
+        }
         return this.timeoutMilliseconds;
     }
 
     public <T> T unwrap(Class<T> iface) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        try {
+            return iface.cast(this);
+        } catch (ClassCastException cce) {
+            throw new SQLException("Unable to unwrap to " + iface.toString());
+        }
     }
 
     public boolean isWrapperFor(Class<?> iface) throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        return iface.isInstance(this);
     }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java

@@ -19,12 +19,12 @@ import java.util.Map;
 
 public abstract class TSDBConstants {
 
-    public static final String STATEMENT_CLOSED = "Statement already closed.";
+    public static final String STATEMENT_CLOSED = "statement is closed";
-    public static final String DEFAULT_PORT = "6200";
     public static final String UNSUPPORT_METHOD_EXCEPTIONZ_MSG = "this operation is NOT supported currently!";
     public static final String INVALID_VARIABLES = "invalid variables";
-    public static final String RESULT_SET_IS_CLOSED = "resultSet is closed.";
+    public static final String RESULT_SET_IS_CLOSED = "resultSet is closed";
 
+    public static final String DEFAULT_PORT = "6200";
     public static Map<Integer, String> DATATYPE_MAP = null;
 
     public static final long JNI_NULL_POINTER = 0L;

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java

@@ -0,0 +1,31 @@
+package com.taosdata.jdbc;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TSDBError {
+    private static Map<Integer, String> TSDBErrorMap = new HashMap<>();
+
+    static {
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED, "connection already closed");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD, "this operation is NOT supported currently!");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variables");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED, "statement is closed");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED, "resultSet is closed");
+        /**************************************************/
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED, "failed to create subscription");
+    }
+
+    public static String wrapErrMsg(String msg) {
+        return "TDengine Error: " + msg;
+    }
+
+    public static SQLException createSQLException(int errorNumber) {
+        // JDBC exception code is less than 0x2350
+        if (errorNumber <= 0x2350)
+            return new SQLException(TSDBErrorMap.get(errorNumber));
+        // JNI exception code is
+        return new SQLException(wrapErrMsg(TSDBErrorMap.get(errorNumber)));
+    }
+}

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java

@@ -0,0 +1,15 @@
+package com.taosdata.jdbc;
+
+public class TSDBErrorNumbers {
+
+    public static final int ERROR_CONNECTION_CLOSED = 0x2301;   // connection already closed
+    public static final int ERROR_UNSUPPORTED_METHOD = 0x2302;  //this operation is NOT supported currently!
+    public static final int ERROR_INVALID_VARIABLE = 0x2303;    //invalid variables
+    public static final int ERROR_STATEMENT_CLOSED = 0x2304;    //statement already closed
+    public static final int ERROR_RESULTSET_CLOSED = 0x2305;    //resultSet is closed
+
+    public static final int ERROR_SUBSCRIBE_FAILED = 0x2350;    //failed to create subscription
+
+    private TSDBErrorNumbers() {
+    }
+}

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -14,6 +14,8 @@
  *****************************************************************************/
 package com.taosdata.jdbc;
 
+import com.taosdata.jdbc.utils.TaosInfo;
+
 import java.sql.SQLException;
 import java.sql.SQLWarning;
 import java.util.List;
@@ -21,6 +23,8 @@ import java.util.List;
 public class TSDBJNIConnector {
     private static volatile Boolean isInitialized = false;
 
+    private TaosInfo taosInfo = TaosInfo.getInstance();
+
     static {
         System.loadLibrary("taos");
         System.out.println("java.library.path:" + System.getProperty("java.library.path"));
@@ -91,7 +95,8 @@ public class TSDBJNIConnector {
      */
     public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException {
         if (this.taos != TSDBConstants.JNI_NULL_POINTER) {
-            this.closeConnectionImp(this.taos);
+//            this.closeConnectionImp(this.taos);
+            closeConnection();
             this.taos = TSDBConstants.JNI_NULL_POINTER;
         }
 
@@ -99,7 +104,8 @@ public class TSDBJNIConnector {
         if (this.taos == TSDBConstants.JNI_NULL_POINTER) {
             throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(0L)), "", this.getErrCode(0l));
         }
-
+        // invoke connectImp only here
+        taosInfo.conn_open_increment();
         return true;
     }
 
@@ -120,6 +126,7 @@ public class TSDBJNIConnector {
         Long pSql = 0l;
         try {
             pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
+            taosInfo.stmt_count_increment();
         } catch (Exception e) {
             e.printStackTrace();
             this.freeResultSetImp(this.taos, pSql);
@@ -248,6 +255,7 @@ public class TSDBJNIConnector {
     }
 
     private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData);
+
     /**
      * Execute close operation from C to release connection pointer by JNI
      *
@@ -262,6 +270,8 @@ public class TSDBJNIConnector {
         } else {
             throw new SQLException("Undefined error code returned by TDengine when closing a connection");
         }
+        // invoke closeConnectionImpl only here
+        taosInfo.connect_close_increment();
     }
 
     private native int closeConnectionImp(long connection);

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java

@@ -14,13 +14,14 @@
  *****************************************************************************/
 package com.taosdata.jdbc;
 
+import com.taosdata.jdbc.utils.TaosInfo;
+
 import java.sql.*;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 public class TSDBStatement implements Statement {
-    private TSDBJNIConnector connector = null;
+    private TSDBJNIConnector connector;
 
     /**
      * To store batched commands
@@ -67,13 +68,12 @@ public class TSDBStatement implements Statement {
     }
 
     public ResultSet executeQuery(String sql) throws SQLException {
-        if (isClosed) {
+        if (isClosed()) {
-            throw new SQLException("Invalid method call on a closed statement.");
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
         }
 
         // TODO make sure it is not a update query
         pSql = this.connector.executeQuery(sql);
-
         long resultSetPointer = this.connector.getResultSet();
         if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
             this.connector.freeResultSet(pSql);
@@ -98,8 +98,8 @@ public class TSDBStatement implements Statement {
     }
 
     public int executeUpdate(String sql) throws SQLException {
-        if (isClosed) {
+        if (isClosed()) {
-            throw new SQLException("Invalid method call on a closed statement.");
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
         }
 
         // TODO check if current query is update query
@@ -131,25 +131,33 @@ public class TSDBStatement implements Statement {
     }
 
     public int getMaxFieldSize() throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
+        }
+
         return 0;
-//        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
     }
 
     public void setMaxFieldSize(int max) throws SQLException {
-        if (isClosed())
+        if (isClosed()) {
-            throw new SQLException(TSDBConstants.STATEMENT_CLOSED);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
+        }
 
         throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
     }
 
     public int getMaxRows() throws SQLException {
-        if (isClosed())
+        if (isClosed()) {
-            throw new SQLException(TSDBConstants.STATEMENT_CLOSED);
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
+        }
         // always set maxRows to zero, meaning unlimitted rows in a resultSet
         return 0;
     }
 
     public void setMaxRows(int max) throws SQLException {
+        if (isClosed()) {
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
+        }
         // always set maxRows to zero, meaning unlimited rows in a resultSet
     }
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java

@@ -1,272 +0,0 @@
-package com.taosdata.jdbc.utils;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.InputStreamReader;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-public class TDNode {
-    
-    private int index;
-    private int running;
-    private int deployed;
-    private boolean testCluster;    
-    private String path;    
-    private String cfgDir;
-    private String dataDir;
-    private String logDir;
-    private String cfgPath;
-
-    public TDNode(int index) {
-        this.index = index;
-        running = 0;
-        deployed = 0;
-        testCluster = false;            
-    }
-
-    public void setPath(String path) {
-        this.path = path;
-    }
-
-    public void setTestCluster(boolean testCluster) {
-        this.testCluster = testCluster;
-    }
-
-    public void setRunning(int running) {
-        this.running = running;
-    }
-
-    public void searchTaosd(File dir, ArrayList<String> taosdPath) {        
-        File[] fileList = dir.listFiles();
-        
-        if(fileList == null || fileList.length == 0) {
-            return;
-        }
-        
-        for(File file : fileList) {
-            if(file.isFile()) {
-                if(file.getName().equals("taosd")) {                        
-                    taosdPath.add(file.getAbsolutePath());
-                }
-            } else {
-                searchTaosd(file, taosdPath);
-            } 
-        }        
-    }
-
-    public void start() {
-        String selfPath = System.getProperty("user.dir");
-        String binPath = "";
-        String projDir = selfPath + "/../../../";
-
-        try {
-            ArrayList<String> taosdPath = new ArrayList<>();
-            
-            File dir = new File(projDir);
-            String realProjDir = dir.getCanonicalPath();
-            dir = new File(realProjDir);
-            System.out.println("project Dir: " + projDir);
-            searchTaosd(dir, taosdPath);
-                        
-            if(taosdPath.size() == 0) {
-                System.out.println("The project path doens't exist");
-                return;
-            } else {
-                for(String p : taosdPath) {
-                    if(!p.contains("packaging")) {
-                        binPath = p;
-                    }
-                }
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-
-        if(binPath.isEmpty()) {
-            System.out.println("taosd not found");
-            return;
-        } else {
-            System.out.println("taosd found in " + binPath);
-        }
-
-        if(this.deployed == 0) {
-            System.out.println("dnode" + index + "is not deployed");
-            return;
-        }
-
-        String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & "; 
-        System.out.println("start taosd cmd: " + cmd);
-        
-        try{
-            Runtime.getRuntime().exec(cmd);  
-            TimeUnit.SECONDS.sleep(5);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        
-        this.running = 1;        
-    }
-
-    public Integer getTaosdPid() {
-        String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"; 
-        String[] cmds = {"sh", "-c", cmd};
-        try {
-            Process process = Runtime.getRuntime().exec(cmds);
-            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
-            String line = null;
-            Integer res = null;
-            while((line = reader.readLine()) != null) {
-                if(!line.isEmpty()) {
-                    res = Integer.valueOf(line);
-                    break;
-                }
-            }
-            
-            return res;
-        } catch (Exception e) {
-            e.printStackTrace();
-        } 
-        return null;
-    }
-
-    public void stop() {        
-
-        if (this.running != 0) {
-            Integer pid = null;                        
-            while((pid = getTaosdPid()) != null) {
-                
-                String killCmd = "kill -term " + pid;                                      
-                String[] killCmds = {"sh", "-c", killCmd};
-                try {                                
-                    Runtime.getRuntime().exec(killCmds).waitFor();                    
-
-                    TimeUnit.SECONDS.sleep(2);
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }                
-            }
-
-            try {
-                for(int port = 6030; port < 6041; port ++) {
-                    String fuserCmd = "fuser -k -n tcp " + port;
-                    Runtime.getRuntime().exec(fuserCmd).waitFor();
-                }
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-
-            this.running = 0;
-            System.out.println("dnode:" + this.index + " is stopped by kill -term");
-        }
-    }
-
-    public void startIP() {
-        try{
-            String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up";        
-            Runtime.getRuntime().exec(cmd).waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    public void stopIP() {
-        try{
-            String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down";            
-            Runtime.getRuntime().exec(cmd).waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    public void setCfgConfig(String option, String value) {
-        try{
-            String cmd = "echo " + option + " " + value + " >> " + this.cfgPath;
-            String[] cmdLine = {"sh", "-c", cmd};
-            Process ps = Runtime.getRuntime().exec(cmdLine);
-            ps.waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    public String getDnodeRootDir() {
-        String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index;
-        return dnodeRootDir;
-    }
-
-    public String getDnodesRootDir() {
-        String dnodesRootDir = this.path + "/sim/psim" + this.index;
-        return dnodesRootDir;
-    }
-
-    public void deploy() {
-        this.logDir = this.path + "/sim/dnode" + this.index + "/log";
-        this.dataDir = this.path + "/sim/dnode" + this.index + "/data";
-        this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg";
-        this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg";
-
-        try {        
-            String cmd = "rm -rf " + this.logDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-            
-            cmd = "rm -rf " + this.cfgDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "rm -rf " + this.dataDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "mkdir -p " + this.logDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "mkdir -p " + this.cfgDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "mkdir -p " + this.dataDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "touch " + this.cfgPath;
-            Runtime.getRuntime().exec(cmd).waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }        
-
-        if(this.testCluster) {
-            startIP();
-            setCfgConfig("masterIp", "192.168.0.1");
-            setCfgConfig("secondIp", "192.168.0.2");
-            setCfgConfig("publicIp", "192.168.0." + this.index);
-            setCfgConfig("internalIp", "192.168.0." + this.index);
-            setCfgConfig("privateIp", "192.168.0." + this.index);
-        }
-        setCfgConfig("dataDir", this.dataDir);
-        setCfgConfig("logDir", this.logDir);
-        setCfgConfig("numOfLogLines", "1000000/00");
-        setCfgConfig("mnodeEqualVnodeNum", "0");
-        setCfgConfig("walLevel", "1");
-        setCfgConfig("statusInterval", "1");
-        setCfgConfig("numOfMnodes", "3");
-        setCfgConfig("numOfThreadsPerCore", "2.0");
-        setCfgConfig("monitor", "0");
-        setCfgConfig("maxVnodeConnections", "30000");
-        setCfgConfig("maxMgmtConnections", "30000");
-        setCfgConfig("maxMeterConnections", "30000");
-        setCfgConfig("maxShellConns", "30000");
-        setCfgConfig("locale", "en_US.UTF-8");
-        setCfgConfig("charset", "UTF-8");
-        setCfgConfig("asyncLog", "0");
-        setCfgConfig("anyIp", "0");
-        setCfgConfig("dDebugFlag", "135");
-        setCfgConfig("mDebugFlag", "135");
-        setCfgConfig("sdbDebugFlag", "135");
-        setCfgConfig("rpcDebugFlag", "135");
-        setCfgConfig("tmrDebugFlag", "131");
-        setCfgConfig("cDebugFlag", "135");
-        setCfgConfig("httpDebugFlag", "135");
-        setCfgConfig("monitorDebugFlag", "135");
-        setCfgConfig("udebugFlag", "135");
-        setCfgConfig("jnidebugFlag", "135");
-        setCfgConfig("qdebugFlag", "135"); 
-        this.deployed = 1;
-    }
-}

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java

@@ -1,72 +0,0 @@
-package com.taosdata.jdbc.utils;
-
-import java.io.File;
-import java.util.*;
-
-public class TDNodes {
-    private ArrayList<TDNode> tdNodes; 
-    private boolean testCluster;     
-
-    public TDNodes () {
-        tdNodes = new ArrayList<>();
-        for(int i = 1; i < 11; i ++) {            
-            tdNodes.add(new TDNode(i));
-        }               
-    }
-
-    public void setTestCluster(boolean testCluster) {
-        this.testCluster = testCluster;
-    }
-
-    public void check(int index) {
-        if(index < 1 || index > 10) {
-            System.out.println("index: " + index + " should on a scale of [1, 10]");
-            return;
-        }
-    } 
-
-    public void deploy(int index) {
-        try {     
-            File file = new File(System.getProperty("user.dir") + "/../../../");
-            String projectRealPath = file.getCanonicalPath();
-            check(index);        
-            tdNodes.get(index - 1).setTestCluster(this.testCluster);            
-            tdNodes.get(index - 1).setPath(projectRealPath); 
-            tdNodes.get(index - 1).deploy();
-        } catch (Exception e) {
-            e.printStackTrace();
-            System.out.println("deploy Test Exception");
-        }
-    }
-    
-    public void cfg(int index, String option, String value) {
-        check(index);
-        tdNodes.get(index - 1).setCfgConfig(option, value);
-    }
-
-    public TDNode getTDNode(int index) {
-        check(index);
-        return tdNodes.get(index - 1);
-    }
-    
-    public void start(int index) {
-        check(index);
-        tdNodes.get(index - 1).start();
-    }
-
-    public void stop(int index) {
-        check(index);
-        tdNodes.get(index - 1).stop();
-    }
-    
-    public void startIP(int index) {
-        check(index);
-        tdNodes.get(index - 1).startIP();
-    }
-
-    public void stopIP(int index) {
-        check(index);
-        tdNodes.get(index - 1).stopIP();
-    }
-
-}

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java

@@ -0,0 +1,74 @@
+package com.taosdata.jdbc.utils;
+
+import javax.management.*;
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class TaosInfo implements TaosInfoMBean {
+
+    private static volatile TaosInfo instance;
+    private AtomicLong connect_open = new AtomicLong();
+    private AtomicLong connect_close = new AtomicLong();
+    private AtomicLong statement_count = new AtomicLong();
+
+    static {
+        try {
+            MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+            ObjectName name = new ObjectName("TaosInfoMBean:name=TaosInfo");
+            server.registerMBean(TaosInfo.getInstance(), name);
+
+        } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public long getConnect_open() {
+        return connect_open.get();
+    }
+
+    @Override
+    public long getConnect_close() {
+        return connect_close.get();
+    }
+
+    @Override
+    public long getConnect_active() {
+        return connect_open.get() - connect_close.get();
+    }
+
+    @Override
+    public long getStatement_count() {
+        return statement_count.get();
+    }
+
+    /*******************************************************/
+
+    public void conn_open_increment() {
+        connect_open.incrementAndGet();
+    }
+
+    public void connect_close_increment() {
+        connect_close.incrementAndGet();
+    }
+
+    public void stmt_count_increment() {
+        statement_count.incrementAndGet();
+    }
+
+    /********************************************************************************/
+    private TaosInfo() {
+    }
+
+    public static TaosInfo getInstance() {
+        if (instance == null) {
+            synchronized (TaosInfo.class) {
+                if (instance == null) {
+                    instance = new TaosInfo();
+                }
+            }
+        }
+        return instance;
+    }
+
+}

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java

@@ -0,0 +1,13 @@
+package com.taosdata.jdbc.utils;
+
+public interface TaosInfoMBean {
+
+    long getConnect_open();
+
+    long getConnect_close();
+
+    long getConnect_active();
+
+    long getStatement_count();
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java

@@ -0,0 +1,49 @@
+package com.taosdata.jdbc.cases;
+
+import org.junit.Test;
+
+import java.sql.*;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class TaosInfoMonitorTest {
+
+    @Test
+    public void testCreateTooManyConnection() throws ClassNotFoundException {
+        Class.forName("com.taosdata.jdbc.TSDBDriver");
+        final String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata";
+
+        List<Connection> connectionList = IntStream.range(0, 100).mapToObj(i -> {
+            try {
+                TimeUnit.MILLISECONDS.sleep(100);
+                return DriverManager.getConnection(url);
+            } catch (SQLException | InterruptedException e) {
+                e.printStackTrace();
+            }
+            return null;
+        }).collect(Collectors.toList());
+
+        connectionList.stream().forEach(conn -> {
+            try (Statement stmt = conn.createStatement()) {
+                ResultSet rs = stmt.executeQuery("show databases");
+                while (rs.next()) {
+
+                }
+                TimeUnit.MILLISECONDS.sleep(100);
+            } catch (SQLException | InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+
+        connectionList.stream().forEach(conn -> {
+            try {
+                conn.close();
+                TimeUnit.MILLISECONDS.sleep(100);
+            } catch (SQLException | InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+    }
+}

src/connector/python/linux/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.4",
+    version="2.0.5",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python2/taos/cursor.py

@@ -1,7 +1,6 @@
 from .cinterface import CTaosInterface
 from .error import *
 from .constants import FieldType
-import threading
 
 
 class TDengineCursor(object):
@@ -36,7 +35,6 @@ class TDengineCursor(object):
         self._block_iter = 0
         self._affected_rows = 0
         self._logfile = ""
-        self._threadId = threading.get_ident()
 
         if connection is not None:
             self._connection = connection

src/cq/src/cqMain.c

@@ -91,6 +91,20 @@ void cqRmFromList(SCqObj *pObj) {
 
 }
 
+static void freeSCqContext(void *handle) {
+  if (handle == NULL) {
+    return;
+  }
+  SCqContext *pContext = handle;
+  pthread_mutex_destroy(&pContext->mutex);
+    
+  taosTmrCleanUp(pContext->tmrCtrl);
+  pContext->tmrCtrl = NULL;
+  cDebug("vgId:%d, CQ is closed", pContext->vgId);
+  free(pContext);
+}
+
+
 void cqFree(void *handle) {
   if (tsEnableStream == 0) {
     return;
@@ -125,15 +139,7 @@ void cqFree(void *handle) {
   pthread_mutex_unlock(&pContext->mutex);
 
   if (delete) {
-    pthread_mutex_unlock(&pContext->mutex);
+    freeSCqContext(pContext);
-    
-    pthread_mutex_destroy(&pContext->mutex);
-    
-    taosTmrCleanUp(pContext->tmrCtrl);
-    pContext->tmrCtrl = NULL;
-    
-    cDebug("vgId:%d, CQ is closed", pContext->vgId);
-    free(pContext);
   }
 }
 
@@ -186,6 +192,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   return pContext;
 }
 
+
 void cqClose(void *handle) {
   if (tsEnableStream == 0) {
     return;
@@ -194,6 +201,8 @@ void cqClose(void *handle) {
   if (handle == NULL) return;
 
   pContext->delete = 1;
+  int32_t hasCq = 0;
+  int32_t existLoop = 0;
   
   // stop all CQs
   cqStop(pContext);
@@ -208,6 +217,12 @@ void cqClose(void *handle) {
       cqRmFromList(pObj);
 
       rid = pObj->rid;
+
+      hasCq = 1;
+
+      if (pContext->pHead == NULL) {
+        existLoop = 1;
+      }
     } else {      
       pthread_mutex_unlock(&pContext->mutex);
       break;
@@ -216,6 +231,14 @@ void cqClose(void *handle) {
     pthread_mutex_unlock(&pContext->mutex);
     
     taosRemoveRef(cqObjRef, rid);
+
+    if (existLoop) {
+      break;
+    }
+  }
+
+  if (hasCq == 0) {
+    freeSCqContext(pContext);
   }
 }
 

src/dnode/CMakeLists.txt

@@ -27,7 +27,7 @@ IF (TD_GRANT)
   TARGET_LINK_LIBRARIES(taosd grant)
 ENDIF ()
 
-IF ((TD_LINUX OR TD_WINDOWS) AND TD_MQTT)
+IF (TD_MQTT)
   TARGET_LINK_LIBRARIES(taosd mqtt)
 ENDIF ()
 
@@ -43,5 +43,6 @@ ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
     COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
+    COMMAND ${CMAKE_COMMAND} -E echo monitor 0  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMENT "prepare taosd environment")
 ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})

src/dnode/inc/dnodeMgmt.h

@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_DNODE_MGMT_H
-#define TDENGINE_DNODE_MGMT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "trpc.h"
-
-int32_t dnodeInitMgmt();
-void    dnodeCleanupMgmt();
-int32_t dnodeInitMgmtTimer();
-void    dnodeCleanupMgmtTimer();
-void    dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg);
-
-void*   dnodeGetVnode(int32_t vgId);
-int32_t dnodeGetVnodeStatus(void *pVnode);
-void*   dnodeGetVnodeRworker(void *pVnode);
-void*   dnodeGetVnodeWworker(void *pVnode);
-void*   dnodeGetVnodeWal(void *pVnode);
-void*   dnodeGetVnodeTsdb(void *pVnode);
-void    dnodeReleaseVnode(void *pVnode);
-
-void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
-void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/dnode/src/dnodeMain.c

@@ -21,7 +21,7 @@
 #include "tconfig.h"
 #include "tfile.h"
 #include "twal.h"
-// #include "tfs.h"
+#include "tfs.h"
 #include "tsync.h"
 #include "dnodeStep.h"
 #include "dnodePeer.h"
@@ -189,31 +189,34 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
-  if (dnodeCreateDir(tsDataDir) < 0) {
+  if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
+    dError("failed to init TFS since %s", tstrerror(terrno));
     return -1;
   }
+  strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN);
   sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
   sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
   sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
-  sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
+  // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
 
   //TODO(dengyihao): no need to init here 
   if (dnodeCreateDir(tsMnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno));
    return -1;
   } 
-  //TODO(dengyihao): no need to init here
+
-  if (dnodeCreateDir(tsVnodeDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsVnodeDir, strerror(errno));
-   return -1;
-  }
   if (dnodeCreateDir(tsDnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno));
    return -1;
   }
-  if (dnodeCreateDir(tsVnodeBakDir) < 0) {
+
-   dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno));
+  if (tfsMkdir("vnode") < 0) {
+    dError("failed to create vnode dir since %s", tstrerror(terrno));
+    return -1;
+  }
+
+  if (tfsMkdir("vnode_bak") < 0) {
+    dError("failed to create vnode_bak dir since %s", tstrerror(terrno));
     return -1;
   }
 
@@ -223,7 +226,7 @@ static int32_t dnodeInitStorage() {
   return 0;
 }
 
-static void dnodeCleanupStorage() {}
+static void dnodeCleanupStorage() { tfsDestroy(); }
 
 bool  dnodeIsFirstDeploy() {
   return strcmp(tsFirst, tsLocalEp) == 0;

src/dnode/src/dnodeVMgmt.c

@@ -174,7 +174,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
     vnodeRelease(pVnode);
     return code;
   } else {
-    dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
+    dInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
     return TSDB_CODE_VND_INVALID_VGROUP_ID;
   }
 }

src/dnode/src/dnodeVWrite.c

@@ -53,7 +53,7 @@ void dnodeCleanupVWrite() {
   for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
     SVWriteWorker *pWorker = tsVWriteWP.worker + i;
     if (taosCheckPthreadValid(pWorker->thread)) {
-      taosQsetThreadResume(pWorker->qset);
+      if (pWorker->qset) taosQsetThreadResume(pWorker->qset);
     }
   }
 

src/inc/taoserror.h

@@ -192,7 +192,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING,           0, 0x0386, "Database n
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY,         0, 0x0387, "Database unsynced")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS,   0, 0x0390, "Invalid database option: days out of range")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP,   0, 0x0391, "Invalid database option: keep >= keep2 >= keep1 >= days")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP,   0, 0x0391, "Invalid database option: keep >= keep1 >= keep0 >= days")
 
 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "Message not processed")
@@ -242,6 +242,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM,     0, 0x060F, "No table d
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS,      0, 0x0610, "File already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE,        0, 0x0611, "Need to reconfigure table")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO,    0, 0x0612, "Invalid information to create table")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK,            0, 0x0613, "No available disk")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG,               0, 0x0614, "TSDB messed message")
 
 // query
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE,          0, 0x0700, "Invalid handle")

src/inc/taosmsg.h

@@ -496,6 +496,7 @@ typedef struct {
   int32_t     tsOrder;          // ts comp block order
   int32_t     numOfTags;        // number of tags columns involved
   int32_t     sqlstrLen;        // sql query string
+  int32_t     prevResultLen;    // previous result length
   SColumnInfo colList[];
 } SQueryTableMsg;
 

src/inc/tfs.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFS_H
+#define TD_TFS_H
+
+#include "tglobal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int level;
+  int id;
+} SDiskID;
+
+#define TFS_UNDECIDED_LEVEL -1
+#define TFS_UNDECIDED_ID -1
+#define TFS_PRIMARY_LEVEL 0
+#define TFS_PRIMARY_ID 0
+
+// FS APIs ====================================
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SFSMeta;
+
+int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
+void tfsDestroy();
+void tfsUpdateInfo(SFSMeta *pFSMeta);
+void tfsGetMeta(SFSMeta *pMeta);
+void tfsAllocDisk(int expLevel, int *level, int *id);
+
+const char *TFS_PRIMARY_PATH();
+const char *TFS_DISK_PATH(int level, int id);
+
+// TFILE APIs ====================================
+typedef struct {
+  int  level;
+  int  id;
+  char rname[TSDB_FILENAME_LEN];  // REL name
+  char aname[TSDB_FILENAME_LEN];  // ABS name
+} TFILE;
+
+#define TFILE_LEVEL(pf) ((pf)->level)
+#define TFILE_ID(pf) ((pf)->id)
+#define TFILE_NAME(pf) ((pf)->aname)
+#define TFILE_REL_NAME(pf) ((pf)->rname)
+
+#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
+#define tfsclose(fd) close(fd)
+#define tfsremove(pf) remove(TFILE_NAME(pf))
+#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df))
+#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
+
+void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
+bool  tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
+int   tfsEncodeFile(void **buf, TFILE *pf);
+void *tfsDecodeFile(void *buf, TFILE *pf);
+void  tfsbasename(const TFILE *pf, char *dest);
+void  tfsdirname(const TFILE *pf, char *dest);
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id);
+int tfsMkdirRecurAt(const char *rname, int level, int id);
+int tfsMkdir(const char *rname);
+int tfsRmdir(const char *rname);
+int tfsRename(char *orname, char *nrname);
+
+typedef struct TDIR TDIR;
+
+TDIR *       tfsOpendir(const char *rname);
+const TFILE *tfsReaddir(TDIR *tdir);
+void         tfsClosedir(TDIR *tdir);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/tsdb.h

@@ -40,7 +40,8 @@ extern "C" {
 
 // TSDB STATE DEFINITION
 #define TSDB_STATE_OK 0x0
-#define TSDB_STATE_BAD_FILE 0x1
+#define TSDB_STATE_BAD_META 0x1
+#define TSDB_STATE_BAD_DATA 0x2
 
 // --------- TSDB APPLICATION HANDLE DEFINITION
 typedef struct {
@@ -48,7 +49,7 @@ typedef struct {
   void *cqH;
   int (*notifyStatus)(void *, int status, int eno);
   int (*eventCallBack)(void *);
-  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
+  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema);
   void (*cqDropFunc)(void *handle);
 } STsdbAppH;
 
@@ -76,17 +77,17 @@ typedef struct {
   int64_t pointsWritten;  // total data points written
 } STsdbStat;
 
-typedef void TSDB_REPO_T;  // use void to hide implementation details from outside
+typedef struct STsdbRepo STsdbRepo;
 
-STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
+STsdbCfg *tsdbGetCfg(const STsdbRepo *repo);
 
 // --------- TSDB REPOSITORY DEFINITION
-int          tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
+int32_t    tsdbCreateRepo(int repoid);
-int32_t      tsdbDropRepo(char *rootDir);
+int32_t    tsdbDropRepo(int repoid);
-TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
+STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH);
-int          tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
+int        tsdbCloseRepo(STsdbRepo *repo, int toCommit);
-int32_t      tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
+int32_t    tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg);
-int          tsdbGetState(TSDB_REPO_T *repo);
+int        tsdbGetState(STsdbRepo *repo);
 
 // --------- TSDB TABLE DEFINITION
 typedef struct {
@@ -110,8 +111,8 @@ typedef struct {
 
 void tsdbClearTableCfg(STableCfg *config);
 
-void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes);
+void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes);
-char* tsdbGetTableName(void *pTable);
+char *tsdbGetTableName(void *pTable);
 
 #define TSDB_TABLEID(_table) ((STableId*) (_table))
 #define TSDB_PREV_ROW  0x1
@@ -119,12 +120,11 @@ char* tsdbGetTableName(void *pTable);
 
 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
 
-int   tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
+int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg);
-int   tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId);
+int tsdbDropTable(STsdbRepo *pRepo, STableId tableId);
-int   tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg);
+int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg);
-// TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid);
 
-uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
+uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
@@ -134,7 +134,7 @@ typedef struct STsdbRepoInfo {
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
 } STsdbRepoInfo;
-STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
+STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo);
 
 // the meter information report structure
 typedef struct {
@@ -152,7 +152,7 @@ typedef struct {
  *
  * @return the number of points inserted, -1 for failure and the error number is set
  */
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
+int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
 
 // -- FOR QUERY TIME SERIES DATA
 
@@ -169,8 +169,8 @@ typedef struct STsdbQueryCond {
 
 typedef struct SMemRef {
   int32_t ref;
-  void    *mem;
+  void *  mem;
-  void    *imem;
+  void *  imem;
 } SMemRef;
 
 typedef struct SDataBlockInfo {
@@ -188,7 +188,7 @@ typedef struct {
 
 typedef struct {
   size_t    numOfTables;
-  SArray   *pGroupList;
+  SArray *  pGroupList;
   SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
@@ -202,7 +202,8 @@ typedef struct {
  * @param qinfo      query info handle from query processor
  * @return
  */
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -214,14 +215,15 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
  * @param tableInfo  table list.
  * @return
  */
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * get the queried table object list
  * @param pHandle
  * @return
  */
-SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
+SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
 
 /**
  * get the group list according to table id from client
@@ -231,8 +233,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
  * @param qinfo
  * @return
  */
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
-                                               void *qinfo, SMemRef* pRef);
+                                               void *qinfo, SMemRef *pRef);
 
 
 /**
@@ -268,7 +270,7 @@ SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t
  * @param pBlockInfo
  * @return
  */
-void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo);
+void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo);
 
 /**
  *
@@ -299,7 +301,7 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL
  * @param stableid. super table sid
  * @param pTagCond. tag query condition
  */
-int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
                                  int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
                                  SColIndex *pColIndex, int32_t numOfCols);
 
@@ -317,7 +319,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
  * @param pGroupInfo  the generated result
  * @return
  */
-int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
+int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
 
 /**
  *
@@ -326,7 +328,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, ST
  * @param pGroupInfo
  * @return
  */
-int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
+int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
 
 /**
  * clean up the query handle
@@ -345,10 +347,14 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int
 
 int  tsdbInitCommitQueue();
 void tsdbDestroyCommitQueue();
-int  tsdbSyncCommit(TSDB_REPO_T *repo);
+int  tsdbSyncCommit(STsdbRepo *repo);
 void tsdbIncCommitRef(int vgId);
 void tsdbDecCommitRef(int vgId);
 
+// For TSDB file sync
+int tsdbSyncSend(void *pRepo, SOCKET socketFd);
+int tsdbSyncRecv(void *pRepo, SOCKET socketFd);
+
 #ifdef __cplusplus
 }
 #endif

src/inc/tsync.h

@@ -56,16 +56,6 @@ typedef struct {
   int32_t  role[TAOS_SYNC_MAX_REPLICA];
 } SNodesRole;
 
-/* 
-  if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file
-  is found between index and eindex, index shall be updated, name shall be set, size shall be set to 
-  file size, and file magic number shall be returned. 
-
-  if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return
-  zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated.
-*/
-typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); 
-
 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
 typedef int32_t  (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); 
@@ -83,24 +73,31 @@ typedef void     (*FNotifyRole)(int32_t vgId, int8_t role);
 typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
 
 // when data file is synced successfully, notity app
-typedef int32_t  (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion);
+typedef void     (*FStartSyncFile)(int32_t vgId);
+typedef void     (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
 
 // get file version
 typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
 
+typedef int32_t  (*FSendFile)(void *tsdb, SOCKET socketFd);
+typedef int32_t  (*FRecvFile)(void *tsdb, SOCKET socketFd);
+
 typedef struct {
   int32_t  vgId;       // vgroup ID
   uint64_t version;    // initial version
   SSyncCfg syncCfg;    // configuration from mgmt
   char     path[TSDB_FILENAME_LEN];  // path to the file
-  FGetFileInfo      getFileInfo;
+  void *   pTsdb;
-  FGetWalInfo       getWalInfo;
+  FGetWalInfo       getWalInfoFp;
-  FWriteToCache     writeToCache;
+  FWriteToCache     writeToCacheFp;
   FConfirmForward   confirmForward;
-  FNotifyRole       notifyRole;
+  FNotifyRole       notifyRoleFp;
-  FNotifyFlowCtrl   notifyFlowCtrl;
+  FNotifyFlowCtrl   notifyFlowCtrlFp;
-  FNotifyFileSynced notifyFileSynced;
+  FStartSyncFile    startSyncFileFp;
-  FGetVersion       getVersion;
+  FStopSyncFile     stopSyncFileFp;
+  FGetVersion       getVersionFp;
+  FSendFile         sendFileFp;
+  FRecvFile         recvFileFp;
 } SSyncInfo;
 
 typedef void *tsync_h;

src/kit/shell/src/shellEngine.c

@@ -470,7 +470,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
 
   wordexp_t full_path;
 
-  if (wordexp(fname, &full_path, 0) != 0) {
+  if (wordexp((char *)fname, &full_path, 0) != 0) {
     fprintf(stderr, "ERROR: invalid file name: %s\n", fname);
     return -1;
   }

src/kit/taosdemo/taosdemo.c

@@ -67,6 +67,7 @@ typedef struct DemoArguments {
   char *   sqlFile;
   bool     use_metric;
   bool     insert_only;
+  bool     answer_yes;
   char *   output_file;
   int      mode;
   char *   datatype[MAX_NUM_DATATYPE + 1];
@@ -114,6 +115,7 @@ typedef struct DemoArguments {
     {0, 'c', "config_directory",         0, "Configuration directory. Default is '/etc/taos/'.",                                                                14},
     #endif
     {0, 'x', 0,                          0, "Insert only flag.",                                                                                                13},
+    {0, 'y', 0,                          0, "Default input yes for prompt",                                                                                                13},
     {0, 'O', "order",                    0, "Insert mode--0: In order, 1: Out of order. Default is in order.",                                                  14},
     {0, 'R', "rate",                     0, "Out of order data's rate--if order=1 Default 10, min: 0, max: 50.",                                                14},
     {0, 'D', "delete table",             0, "Delete data methods——0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database",             14},
@@ -210,6 +212,9 @@ typedef struct DemoArguments {
       case 'x':
         arguments->insert_only = true;
         break;
+      case 'y':
+        arguments->answer_yes = true;
+        break;
       case 'c':
         if (wordexp(arg, &full_path, 0) != 0) {
           fprintf(stderr, "Invalid path %s\n", arg);
@@ -328,6 +333,8 @@ typedef struct DemoArguments {
     #endif
     printf("%s%s\n", indent, "-x");
     printf("%s%s%s\n", indent, indent, "flag, Insert only flag.");
+    printf("%s%s\n", indent, "-y");
+    printf("%s%s%s\n", indent, indent, "flag, Anser Yes for prompt.");
     printf("%s%s\n", indent, "-O");
     printf("%s%s%s\n", indent, indent, "order, Insert mode--0: In order, 1: Out of order. Default is in order.");
     printf("%s%s\n", indent, "-R");
@@ -409,6 +416,8 @@ typedef struct DemoArguments {
         arguments->use_metric = true;
       } else if (strcmp(argv[i], "-x") == 0) {
         arguments->insert_only = true;
+      } else if (strcmp(argv[i], "-y") == 0) {
+        arguments->answer_yes = true;
       } else if (strcmp(argv[i], "-c") == 0) {
         strcpy(configDir, argv[++i]);
       } else if (strcmp(argv[i], "-O") == 0) {
@@ -549,6 +558,7 @@ int main(int argc, char *argv[]) {
                                 NULL,
                                 false,           // use_metric
                                 false,           // insert_only
+                                false,           // answer_yes
                                 "./output.txt",  // output_file
                                 0,               // mode
                                 {
@@ -584,6 +594,7 @@ int main(int argc, char *argv[]) {
   arguments.num_of_RPR = 1000;
   arguments.use_metric = true;
   arguments.insert_only = false;
+  arguments.answer_yes = false;
   // end change
 
   parse_args(argc, argv, &arguments);
@@ -606,6 +617,7 @@ int main(int argc, char *argv[]) {
   int nrecords_per_request = arguments.num_of_RPR;
   bool use_metric = arguments.use_metric;
   bool insert_only = arguments.insert_only;
+  bool answer_yes = arguments.answer_yes;
   char **data_type = arguments.datatype;
   int count_data_type = 0;
   char dataString[STRING_LEN];
@@ -666,9 +678,12 @@ int main(int argc, char *argv[]) {
   printf("# Delete method:                     %d\n", method_of_delete);
   printf("# Test time:                         %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
           tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
+
+  if (!answer_yes) {
     printf("###################################################################\n\n");
     printf("Press enter key to continue");
     (void)getchar();
+  }
 
   fprintf(fp, "###################################################################\n");
   fprintf(fp, "# Server IP:                         %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);
@@ -908,6 +923,7 @@ int main(int argc, char *argv[]) {
     }
     pthread_join(read_id, NULL);
     taos_close(rInfo->taos);
+    free(rInfo);
   }
 
   taos_cleanup();

src/kit/taosdemox/taosdemox.c

@@ -93,6 +93,8 @@ extern char configDir[];
 #define   MAX_QUERY_SQL_COUNT    10
 #define   MAX_QUERY_SQL_LENGTH   256
 
+#define   MAX_DATABASE_COUNT     256
+
 typedef enum CREATE_SUB_TALBE_MOD_EN {
   PRE_CREATE_SUBTBL,
   AUTO_CREATE_SUBTBL,
@@ -117,6 +119,40 @@ enum QUERY_TYPE {
   QUERY_TYPE_BUT
 } ;  
 
+enum _show_db_index {
+  TSDB_SHOW_DB_NAME_INDEX,
+  TSDB_SHOW_DB_CREATED_TIME_INDEX,
+  TSDB_SHOW_DB_NTABLES_INDEX,
+  TSDB_SHOW_DB_VGROUPS_INDEX,
+  TSDB_SHOW_DB_REPLICA_INDEX,
+  TSDB_SHOW_DB_QUORUM_INDEX,  
+  TSDB_SHOW_DB_DAYS_INDEX,
+  TSDB_SHOW_DB_KEEP_INDEX,
+  TSDB_SHOW_DB_CACHE_INDEX,
+  TSDB_SHOW_DB_BLOCKS_INDEX,
+  TSDB_SHOW_DB_MINROWS_INDEX,
+  TSDB_SHOW_DB_MAXROWS_INDEX,
+  TSDB_SHOW_DB_WALLEVEL_INDEX,
+  TSDB_SHOW_DB_FSYNC_INDEX,
+  TSDB_SHOW_DB_COMP_INDEX,
+  TSDB_SHOW_DB_CACHELAST_INDEX,
+  TSDB_SHOW_DB_PRECISION_INDEX,  
+  TSDB_SHOW_DB_UPDATE_INDEX,
+  TSDB_SHOW_DB_STATUS_INDEX,
+  TSDB_MAX_SHOW_DB
+};
+
+// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
+enum _show_stables_index {
+  TSDB_SHOW_STABLES_NAME_INDEX,
+  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
+  TSDB_SHOW_STABLES_COLUMNS_INDEX,
+  TSDB_SHOW_STABLES_METRIC_INDEX,  
+  TSDB_SHOW_STABLES_UID_INDEX,  
+  TSDB_SHOW_STABLES_TID_INDEX,
+  TSDB_SHOW_STABLES_VGID_INDEX,  
+  TSDB_MAX_SHOW_STABLES
+};  
 enum _describe_table_index {
   TSDB_DESCRIBE_METRIC_FIELD_INDEX,
   TSDB_DESCRIBE_METRIC_TYPE_INDEX,
@@ -173,6 +209,7 @@ typedef struct SSuperTable_S {
   int          childTblCount;
   bool         superTblExists;    // 0: no, 1: yes
   bool         childTblExists;    // 0: no, 1: yes  
+  int          batchCreateTableNum;  // 0: no batch,  > 0: batch table number in one sql
   int8_t       autoCreateTable;                  // 0: create sub table, 1: auto create sub table
   char         childTblPrefix[MAX_TB_NAME_SIZE];
   char         dataSource[MAX_TB_NAME_SIZE];  // rand_gen or sample
@@ -218,6 +255,28 @@ typedef struct SSuperTable_S {
   int64_t    totalAffectedRows;
 } SSuperTable;
 
+typedef struct {
+  char     name[TSDB_DB_NAME_LEN + 1];
+  char     create_time[32];
+  int32_t  ntables;
+  int32_t  vgroups;  
+  int16_t  replica;
+  int16_t  quorum;
+  int16_t  days;  
+  char     keeplist[32];
+  int32_t  cache; //MB
+  int32_t  blocks;
+  int32_t  minrows;
+  int32_t  maxrows;
+  int8_t   wallevel;
+  int32_t  fsync;
+  int8_t   comp;
+  int8_t   cachelast;
+  char     precision[8];   // time resolution
+  int8_t   update;
+  char     status[16];
+} SDbInfo;
+
 typedef struct SDbCfg_S { 
 //  int       maxtablesPerVnode;
   int       minRows; 
@@ -812,7 +871,8 @@ static void printfInsertMeta() {
   printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
   printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
   printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
-  printf("thread count:            \033[33m%d\033[0m\n", g_Dbs.threadCount);
+  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
+  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
 
   printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
   for (int i = 0; i < g_Dbs.dbCount; i++) {
@@ -948,7 +1008,8 @@ static void printfInsertMetaToFile(FILE* fp) {
   fprintf(fp, "user:                       %s\n", g_Dbs.user);
   fprintf(fp, "password:                   %s\n", g_Dbs.password);
   fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
-  fprintf(fp, "thread count:            %d\n", g_Dbs.threadCount);
+  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
+  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
 
   fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
   for (int i = 0; i < g_Dbs.dbCount; i++) {
@@ -1123,6 +1184,272 @@ static void printfQueryMeta() {
   printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n");
 }
 
+
+static char* xFormatTimestamp(char* buf, int64_t val, int precision) {
+  time_t tt;
+  if (precision == TSDB_TIME_PRECISION_MICRO) {
+    tt = (time_t)(val / 1000000);
+  } else {
+    tt = (time_t)(val / 1000);
+  }
+
+/* comment out as it make testcases like select_with_tags.sim fail.
+  but in windows, this may cause the call to localtime crash if tt < 0,
+  need to find a better solution.
+  if (tt < 0) {
+    tt = 0;
+  }
+  */
+
+#ifdef WINDOWS
+  if (tt < 0) tt = 0;
+#endif
+
+  struct tm* ptm = localtime(&tt);
+  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);
+
+  if (precision == TSDB_TIME_PRECISION_MICRO) {
+    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
+  } else {
+    sprintf(buf + pos, ".%03d", (int)(val % 1000));
+  }
+
+  return buf;
+}
+
+static void xDumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) {
+  if (val == NULL) {
+    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
+    return;
+  }
+
+  char buf[TSDB_MAX_BYTES_PER_ROW];
+  switch (field->type) {
+    case TSDB_DATA_TYPE_BOOL:
+      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      fprintf(fp, "%d", *((int8_t *)val));
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      fprintf(fp, "%d", *((int16_t *)val));
+      break;
+    case TSDB_DATA_TYPE_INT:
+      fprintf(fp, "%d", *((int32_t *)val));
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      fprintf(fp, "%" PRId64, *((int64_t *)val));
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
+      break;
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      memcpy(buf, val, length);
+      buf[length] = 0;
+      fprintf(fp, "\'%s\'", buf);
+      break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      xFormatTimestamp(buf, *(int64_t*)val, precision);
+      fprintf(fp, "'%s'", buf);
+      break;
+    default:
+      break;
+  }
+}
+
+static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
+  TAOS_ROW row = taos_fetch_row(tres);
+  if (row == NULL) {
+    return 0;
+  }
+
+  FILE* fp = fopen(fname, "at");
+  if (fp == NULL) {
+    fprintf(stderr, "ERROR: failed to open file: %s\n", fname);
+    return -1;
+  }
+
+  int num_fields = taos_num_fields(tres);
+  TAOS_FIELD *fields = taos_fetch_fields(tres);
+  int precision = taos_result_precision(tres);
+
+  for (int col = 0; col < num_fields; col++) {
+    if (col > 0) {
+      fprintf(fp, ",");
+    }
+    fprintf(fp, "%s", fields[col].name);
+  }
+  fputc('\n', fp);
+  
+  int numOfRows = 0;
+  do {
+    int32_t* length = taos_fetch_lengths(tres);
+    for (int i = 0; i < num_fields; i++) {
+      if (i > 0) {
+        fputc(',', fp);
+      }
+      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
+    }
+    fputc('\n', fp);
+
+    numOfRows++;
+    row = taos_fetch_row(tres);
+  } while( row != NULL);
+
+  fclose(fp);
+
+  return numOfRows;
+}
+
+static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {  
+  TAOS_RES * res;  
+  TAOS_ROW row = NULL;
+  int count = 0;
+  
+  res = taos_query(taos, "show databases;");  
+  int32_t code = taos_errno(res);
+  
+  if (code != 0) {
+    fprintf(stderr, "failed to run <show databases>, reason: %s\n", taos_errstr(res));
+    return -1;
+  }
+
+  TAOS_FIELD *fields = taos_fetch_fields(res);
+
+  while ((row = taos_fetch_row(res)) != NULL) {
+    // sys database name : 'log'
+    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue;
+
+    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
+    if (dbInfos[count] == NULL) {
+      fprintf(stderr, "failed to allocate memory for some dbInfo[%d]\n", count);
+      return -1;
+    }
+
+    strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
+    xFormatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI);
+    dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
+    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);  
+    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
+    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
+    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);  
+
+    strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);      
+    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
+    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
+    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
+    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
+    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
+    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
+    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
+    dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
+
+    strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);  
+    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
+    strncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX], fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); 
+    
+    count++;
+    if (count > MAX_DATABASE_COUNT) {
+      fprintf(stderr, "The database count overflow than %d\n", MAX_DATABASE_COUNT);      
+      break;
+    }
+  }
+
+  return count;
+}
+
+static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) {
+  FILE *fp = NULL;
+  if (filename[0] != 0) {
+    fp = fopen(filename, "at");
+    if (fp == NULL) {
+      fprintf(stderr, "failed to open file: %s\n", filename);
+      return;
+    }
+  }
+
+  fprintf(fp, "================ database[%d] ================\n", index);
+  fprintf(fp, "name: %s\n", dbInfos->name);
+  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
+  fprintf(fp, "ntables: %d\n", dbInfos->ntables);
+  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);  
+  fprintf(fp, "replica: %d\n", dbInfos->replica);
+  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
+  fprintf(fp, "days: %d\n", dbInfos->days);    
+  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);      
+  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
+  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
+  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
+  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
+  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
+  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
+  fprintf(fp, "comp: %d\n", dbInfos->comp);
+  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);  
+  fprintf(fp, "precision: %s\n", dbInfos->precision);  
+  fprintf(fp, "update: %d\n", dbInfos->update);
+  fprintf(fp, "status: %s\n", dbInfos->status); 
+  fprintf(fp, "\n");
+
+  fclose(fp);
+}
+
+static void printfQuerySystemInfo(TAOS * taos) {
+  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
+  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
+  TAOS_RES* res;
+
+  time_t t;
+  struct tm* lt;
+  time(&t);
+  lt = localtime(&t);
+  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d", lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec);
+  
+  // show variables
+  res = taos_query(taos, "show variables;");
+  //getResult(res, filename);
+  xDumpResultToFile(filename, res);
+
+  // show dnodes
+  res = taos_query(taos, "show dnodes;");
+  xDumpResultToFile(filename, res);
+  //getResult(res, filename);
+  
+  // show databases
+  res = taos_query(taos, "show databases;");
+  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
+  if (dbInfos == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return;
+  }
+  int dbCount = getDbFromServer(taos, dbInfos);
+  if (dbCount <= 0) return;
+
+  for (int i = 0; i < dbCount; i++) {
+    // printf database info 
+    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
+    
+    // show db.vgroups
+    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
+    res = taos_query(taos, buffer);
+    xDumpResultToFile(filename, res);
+  
+    // show db.stables
+    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
+    res = taos_query(taos, buffer);
+    xDumpResultToFile(filename, res);
+
+    free(dbInfos[i]);
+  }
+
+  free(dbInfos);
+  
+}
+
+
 #ifdef TD_LOWA_CURL
 static size_t responseCallback(void *contents, size_t size, size_t nmemb, void *userp)
 {
@@ -1731,18 +2058,26 @@ static int createDatabases() {
 
 void * createTable(void *sarg) 
 {  
-  char command[BUFFER_SIZE] = "\0";
-  
   threadInfo *winfo = (threadInfo *)sarg; 
   SSuperTable* superTblInfo = winfo->superTblInfo;
 
   int64_t  lastPrintTime = taosGetTimestampMs();
 
+  char* buffer = calloc(superTblInfo->maxSqlLen, 1);
+
+  int len = 0;
+  int batchNum = 0;
   //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id);
   for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) {
     if (0 == g_Dbs.use_metric) {
-      snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable);
+      snprintf(buffer, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable);
     } else {
+      if (0 == len) {  
+        batchNum = 0;
+        memset(buffer, 0, superTblInfo->maxSqlLen);
+        len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "create table ");
+      }
+      
       char* tagsValBuf = NULL;
       if (0 == superTblInfo->tagSource) {
         tagsValBuf = generateTagVaulesForStb(superTblInfo);
@@ -1750,13 +2085,22 @@ void * createTable(void *sarg)
         tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount);
       }
       if (NULL == tagsValBuf) {
+        free(buffer);
         return NULL;
       }
-      snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.%s tags %s;", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf);
+      
+      len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "if not exists %s.%s%d using %s.%s tags %s ", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf);
       free(tagsValBuf);
+      batchNum++;
+
+      if ((batchNum < superTblInfo->batchCreateTableNum) && ((superTblInfo->maxSqlLen - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) {
+        continue;
+      }
     }
 
-    if (0 != queryDbExec(winfo->taos, command, NO_INSERT_TYPE)){
+    len = 0;
+    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){
+      free(buffer);
       return NULL;
     }
 
@@ -1767,6 +2111,11 @@ void * createTable(void *sarg)
     }
   }
   
+  if (0 != len) {
+    (void)queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE);
+  }
+  
+  free(buffer);
   return NULL;
 }
 
@@ -2423,6 +2772,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
         goto PARSE_OVER;
       }
       
+      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
+      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
+        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
+      } else if (!batchCreateTbl) {
+        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 2000;
+      } else {
+        printf("failed to read json, batch_create_tbl_num not found");
+        goto PARSE_OVER;
+      }      
+
       cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
       if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) {
         if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) {
@@ -3679,14 +4038,14 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
     b = ntables % threads;
   }
 
-  TAOS* taos;
+  //TAOS* taos;
-  if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
+  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
-    taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
+  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
-    if (NULL == taos) {
+  //  if (NULL == taos) {
-      printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
+  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
-      exit(-1);
+  //    exit(-1);
-    }
+  //  }
-  }
+  //}
 
   int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
   if (0 != precision[0]) {
@@ -3719,7 +4078,12 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
     t_info->start_time = start_time;
 
     if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
-      t_info->taos = taos;
+      //t_info->taos = taos;
+      t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
+      if (NULL == t_info->taos) {
+        printf("connect to server fail from insert sub thread, reason: %s\n", taos_errstr(NULL));
+        exit(-1);
+      }
     } else {
       t_info->taos = NULL;
       #ifdef TD_LOWA_CURL
@@ -3754,6 +4118,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
     threadInfo *t_info = infos + i;
 
     tsem_destroy(&(t_info->lock_sem));
+    taos_close(t_info->taos);
 
     superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
     superTblInfo->totalRowsInserted += t_info->totalRowsInserted;
@@ -3766,7 +4131,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
 
   double end = getCurrentTime();
   
-  taos_close(taos);
+  //taos_close(taos);
 
   free(pids);
   free(infos);  
@@ -4093,7 +4458,7 @@ void *subQueryProcess(void *sarg) {
 int queryTestProcess() {
   TAOS * taos = NULL;  
   taos_init();
-  taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port);
+  taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, NULL, g_queryInfo.port);
   if (taos == NULL) {
     fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
     exit(-1);
@@ -4107,6 +4472,8 @@ int queryTestProcess() {
   printf("Press enter key to continue\n\n");
   (void)getchar();
 
+  printfQuerySystemInfo(taos);
+  
   pthread_t  *pids  = NULL;
   threadInfo *infos = NULL;
   //==== create sub threads for query from specify table

src/mnode/inc/mnodeDnode.h

@@ -67,6 +67,7 @@ void    mnodeCleanupDnodes();
 int32_t mnodeGetDnodesNum();
 int32_t mnodeGetOnlinDnodesCpuCoreNum();
 int32_t mnodeGetOnlineDnodesNum();
+void    mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum);
 void *  mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
 void    mnodeCancelGetNextDnode(void *pIter);
 void    mnodeIncDnodeRef(SDnodeObj *pDnode);

src/mnode/src/mnodeDb.c

@@ -568,7 +568,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
 
   pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "keep1,keep2,keep(D)");
+  strcpy(pSchema[cols].name, "keep0,keep1,keep(D)");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -679,7 +679,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
     pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb);
 
     if (pDb == NULL) break;
-    if (pDb->pAcct != pUser->pAcct) {
+    if (pDb->pAcct != pUser->pAcct || pDb->status != TSDB_DB_STATUS_READY) {
       mnodeDecDbRef(pDb);
       continue;
     }

src/mnode/src/mnodeDnode.c

@@ -263,6 +263,28 @@ int32_t mnodeGetOnlineDnodesNum() {
   return onlineDnodes;
 }
 
+void mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum) {
+  SDnodeObj *pDnode = NULL;
+  void *     pIter = NULL;
+  int32_t    onlineDnodes = 0, totalDnodes = 0;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status != TAOS_DN_STATUS_OFFLINE) ++onlineDnodes;
+    ++totalDnodes;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  if (onlineNum) {
+    *onlineNum = onlineDnodes;
+  }
+
+  if (totalNum) {
+    *totalNum = totalDnodes;
+  }
+}
+
 void *mnodeGetDnode(int32_t dnodeId) {
   return sdbGetRow(tsDnodeSdb, &dnodeId);
 }

src/mnode/src/mnodePeer.c

@@ -17,7 +17,6 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tsched.h"
-#include "tsystem.h"
 #include "tutil.h"
 #include "tgrant.h"
 #include "tbn.h"

src/mnode/src/mnodeSdb.c

@@ -242,11 +242,6 @@ void sdbUpdateMnodeRoles() {
   mnodeUpdateMnodeEpSet(NULL);
 }
 
-static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
-  sdbUpdateMnodeRoles();
-  return 0;
-}
-
 static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
   return walGetWalFile(tsSdbMgmt.wal, fileName, fileId);
 }
@@ -262,7 +257,9 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) {
   sdbUpdateMnodeRoles();
 }
 
-static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; }
+static void sdbStartFileSync(int32_t vgId) {}
+
+static void sdbStopFileSync(int32_t vgId, uint64_t fversion) {}
 
 static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {}
 
@@ -396,14 +393,14 @@ int32_t sdbUpdateSync(void *pMnodes) {
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;
   sprintf(syncInfo.path, "%s", tsMnodeDir);
-  syncInfo.getFileInfo = sdbGetFileInfo;
+  syncInfo.getWalInfoFp = sdbGetWalInfo;
-  syncInfo.getWalInfo = sdbGetWalInfo;
+  syncInfo.writeToCacheFp = sdbWriteFwdToQueue;
-  syncInfo.writeToCache = sdbWriteFwdToQueue;
   syncInfo.confirmForward = sdbConfirmForward;
-  syncInfo.notifyRole = sdbNotifyRole;
+  syncInfo.notifyRoleFp = sdbNotifyRole;
-  syncInfo.notifyFileSynced = sdbNotifyFileSynced;
+  syncInfo.startSyncFileFp = sdbStartFileSync;
-  syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl;
+  syncInfo.stopSyncFileFp = sdbStopFileSync;
-  syncInfo.getVersion = sdbGetSyncVersion;
+  syncInfo.notifyFlowCtrlFp = sdbNotifyFlowCtrl;
+  syncInfo.getVersionFp = sdbGetSyncVersion;
   tsSdbMgmt.cfg = syncCfg;
 
   if (tsSdbMgmt.sync) {

src/mnode/src/mnodeShow.c

@@ -280,8 +280,11 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
     }
   }
 
-  pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
+  int32_t    onlineDnodes = 0, totalDnodes = 0;
-  pRsp->totalDnodes = htonl(mnodeGetDnodesNum());
+  mnodeGetOnlineAndTotalDnodesNum(&onlineDnodes, &totalDnodes);
+
+  pRsp->onlineDnodes = htonl(onlineDnodes);
+  pRsp->totalDnodes = htonl(totalDnodes);
   mnodeGetMnodeEpSetForShell(&pRsp->epSet, false);
 
   pMsg->rpcRsp.rsp = pRsp;

src/mnode/src/mnodeTable.c

@@ -1081,20 +1081,13 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
   if (code != TSDB_CODE_SUCCESS) {
     mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
-  } else {
-    mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
-  }
 
     return code;
-}
+  }
 
-static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
+  mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
-  if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
 
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-  mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle,
-        pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash));
-
   if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
     int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL);
     while (pVgId) {
@@ -1122,6 +1115,16 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
     mnodeDropAllChildTablesInStable(pStable);
   }
 
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
+  if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
+
+  SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
+  mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle,
+        pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash));
+
   SSdbRow row = {
     .type    = SDB_OPER_GLOBAL,
     .pTable  = tsSuperTableSdb,
@@ -1461,9 +1464,9 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
 
-  SSchema tbnameSchema = tGetTableNameColumnSchema();
+  SSchema* tbnameSchema = tGetTbnameColumnSchema();
-  pShow->bytes[cols] = tbnameSchema.bytes;
+  pShow->bytes[cols] = tbnameSchema->bytes;
-  pSchema[cols].type = tbnameSchema.type;
+  pSchema[cols].type = tbnameSchema->type;
   strcpy(pSchema[cols].name, "name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
@@ -2821,9 +2824,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
 
-  SSchema s = tGetTableNameColumnSchema();
+  SSchema* s = tGetTbnameColumnSchema();
-  pShow->bytes[cols] = s.bytes;
+  pShow->bytes[cols] = s->bytes;
-  pSchema[cols].type = s.type;
+  pSchema[cols].type = s->type;
   strcpy(pSchema[cols].name, "table_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
@@ -2840,9 +2843,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
-  SSchema tbCol = tGetTableNameColumnSchema();
+  SSchema* tbCol = tGetTbnameColumnSchema();
-  pShow->bytes[cols] = tbCol.bytes + VARSTR_HEADER_SIZE;
+  pShow->bytes[cols] = tbCol->bytes + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = tbCol.type;
+  pSchema[cols].type = tbCol->type;
   strcpy(pSchema[cols].name, "stable_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
@@ -3076,9 +3079,9 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
 
-  SSchema tbnameColSchema = tGetTableNameColumnSchema();
+  SSchema* tbnameColSchema = tGetTbnameColumnSchema();
-  pShow->bytes[cols] = tbnameColSchema.bytes;
+  pShow->bytes[cols] = tbnameColSchema->bytes;
-  pSchema[cols].type = tbnameColSchema.type;
+  pSchema[cols].type = tbnameColSchema->type;
   strcpy(pSchema[cols].name, "table_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;

src/os/inc/osDarwin.h

@@ -85,6 +85,7 @@ extern "C" {
 #define TAOS_OS_FUNC_STRING_STR2INT64
 #define TAOS_OS_FUNC_SYSINFO
 #define TAOS_OS_FUNC_TIMER
+#define TAOS_OS_FUNC_SEMPHONE_PTHREAD
 
 // specific
 #define htobe64 htonll
@@ -105,6 +106,8 @@ typedef int(*__compar_fn_t)(const void *, const void *);
   #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
 #endif
 
+#define TAOS_OS_FUNC_PTHREAD_RWLOCK
+
 int64_t tsosStr2int64(char *str);
 
 #include "eok.h"

src/os/inc/osDef.h

@@ -26,10 +26,6 @@ extern "C" {
   #endif
 #endif
 
-#ifndef STDERR_FILENO
-#define STDERR_FILENO (2)
-#endif
-
 #define FD_VALID(x) ((x) > STDERR_FILENO)
 #define FD_INITIALIZER  ((int32_t)-1)
 

src/os/inc/osDir.h

@@ -25,8 +25,8 @@ extern "C" {
 // TAOS_OS_FUNC_DIR
 void taosRemoveDir(char *rootDir);
 int  taosMkDir(const char *pathname, mode_t mode); 
-void taosRename(char* oldName, char *newName);
 void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays);
+int32_t taosRename(char* oldName, char *newName);
 int32_t taosCompressFile(char *srcFileName, char *destFileName);
 
 #ifdef __cplusplus

src/os/inc/osFile.h

@@ -26,6 +26,7 @@ int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
 int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
 int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);
+int64_t taosCopy(char *from, char *to);
 
 #define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
 #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)

src/os/inc/osMemory.h

@@ -35,7 +35,7 @@ void   taosDumpMemoryLeak();
 void * taosTMalloc(size_t size);
 void * taosTCalloc(size_t nmemb, size_t size);
 void * taosTRealloc(void *ptr, size_t size);
-void   taosTZfree(void *ptr);
+void * taosTZfree(void *ptr);
 size_t taosTSizeof(void *ptr);
 void   taosTMemset(void *ptr, int c);
 

src/os/inc/osSemphone.h

@@ -28,6 +28,21 @@ extern "C" {
   #define tsem_destroy sem_destroy
 #endif
 
+#ifdef TAOS_OS_FUNC_PTHREAD_RWLOCK
+  #define pthread_rwlock_t pthread_mutex_t
+  #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock)
+
+  #define pthread_spinlock_t pthread_mutex_t
+  #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_spin_lock(lock) pthread_mutex_lock(lock)
+  #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock)
+#endif
+
 // TAOS_OS_FUNC_SEMPHONE_PTHREAD
 bool    taosCheckPthreadValid(pthread_t thread);
 int64_t taosGetSelfPthreadId();

src/os/inc/osSysinfo.h

@@ -21,10 +21,16 @@ extern "C" {
 #endif
 
 // TAOS_OS_FUNC_SYSINFO
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SysDiskSize;
+
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
 void taosGetSystemInfo();
 bool taosGetProcIO(float *readKB, float *writeKB);
 bool taosGetBandSpeed(float *bandSpeedKb);
-bool taosGetDisk();
+void taosGetDisk();
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ;
 bool taosGetProcMemory(float *memoryUsedMB) ;
 bool taosGetSysMemory(float *memoryUsedMB);

src/os/inc/osWindows.h

@@ -46,6 +46,8 @@
 #include "msvcFcntl.h"
 #include "msvcLibgen.h"
 #include "msvcStdio.h"
+#include "msvcUnistd.h"
+#include "msvcLibgen.h"
 #include "sys/msvcStat.h"
 #include "sys/msvcTypes.h"
 
@@ -67,6 +69,8 @@ extern "C" {
 #define TAOS_OS_FUNC_FILE_GETTMPFILEPATH
 #define TAOS_OS_FUNC_FILE_FTRUNCATE 
 
+#define TAOS_OS_FUNC_DIR
+
 #define TAOS_OS_FUNC_MATH
   #define SWAP(a, b, c)      \
     do {                     \
@@ -144,7 +148,6 @@ typedef int (*__compar_fn_t)(const void *, const void *);
 #define in_addr_t unsigned long
 #define socklen_t int
 #define htobe64 htonll
-#define getpid _getpid
 
 struct tm *localtime_r(const time_t *timep, struct tm *result);
 char *     strptime(const char *buf, const char *fmt, struct tm *tm);
@@ -153,15 +156,8 @@ char *     getpass(const char *prefix);
 int        flock(int fd, int option);
 int        fsync(int filedes);
 char *     strndup(const char *s, size_t n);
-char *     dirname(char *pszPathname);
 int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
-// for access function in io.h
-#define F_OK 00  //Existence only
-#define W_OK 02  //Write - only
-#define R_OK 04  //Read - only
-#define X_OK 06  //Read and write
-
 // for send function in tsocket.c
 #define MSG_NOSIGNAL             0
 #define SO_NO_CHECK              0x1234
@@ -201,11 +197,11 @@ int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
 typedef struct {
   int    we_wordc;
-  char **we_wordv;
+  char  *we_wordv[1];
   int    we_offs;
-  char   wordPos[20];
+  char   wordPos[1025];
 } wordexp_t;
-int  wordexp(const char *words, wordexp_t *pwordexp, int flags);
+int  wordexp(char *words, wordexp_t *pwordexp, int flags);
 void wordfree(wordexp_t *pwordexp);
 
 #define openlog(a, b, c)

src/os/src/darwin/darwinEnv.c

@@ -17,17 +17,27 @@
 #include "os.h"
 #include "tglobal.h"
 
+static const char* expand_like_shell(const char *path) {
+  static __thread char buf[TSDB_FILENAME_LEN];
+  buf[0] = '\0';
+  wordexp_t we;
+  if (wordexp(path, &we, 0)) return "/tmp/taosd";
+  if (sizeof(buf)<=snprintf(buf, sizeof(buf), "%s", we.we_wordv[0])) return "/tmp/taosd";
+  wordfree(&we);
+  return buf;
+}
+
 void osInit() {
   if (configDir[0] == 0) {
-    strcpy(configDir, "~/TDengine/cfg");
+    strcpy(configDir, expand_like_shell("/usr/local/etc/taos"));
   }
+  strcpy(tsDataDir,   expand_like_shell("/usr/local/var/lib/taos"));
+  strcpy(tsLogDir,    expand_like_shell("/usr/local/var/log/taos"));
+  strcpy(tsScriptDir, expand_like_shell("/usr/local/etc/taos"));
 
   strcpy(tsVnodeDir, "");
   strcpy(tsDnodeDir, "");
   strcpy(tsMnodeDir, "");
-  strcpy(tsDataDir, "~/TDengine/data");
-  strcpy(tsLogDir, "~/TDengine/log");
-  strcpy(tsScriptDir, "~/TDengine/cfg");
   strcpy(tsOsName, "Darwin");
 }
 

src/os/src/darwin/darwinFile.c

@@ -17,71 +17,49 @@
 #include "os.h"
 #include "tulog.h"
 
-#define _SEND_FILE_STEP_ 1000
-
 int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
-  fseek(in_file, (int32_t)(*offset), 0);
+  int r = 0;
-  int     writeLen = 0;
+  if (offset) {
-  uint8_t buffer[_SEND_FILE_STEP_] = {0};
+    r = fseek(in_file, *offset, SEEK_SET);
-
+    if (r==-1) return -1;
-  for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-    size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
-    if (rlen <= 0) {
-      return writeLen;
-    } else if (rlen < _SEND_FILE_STEP_) {
-      fwrite(buffer, 1, rlen, out_file);
-      return (int)(writeLen + rlen);
-    } else {
-      fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
-      writeLen += _SEND_FILE_STEP_;
   }
+  off_t len = count;
+  while (len>0) {
+    char buf[1024*16];
+    off_t n = sizeof(buf);
+    if (len<n) n = len;
+    size_t m = fread(buf, 1, n, in_file);
+    if (m<n) {
+      int e = ferror(in_file);
+      if (e) return -1;
     }
-
+    if (m==0) break;
-  int remain = count - writeLen;
+    if (m!=fwrite(buf, 1, m, out_file)) {
-  if (remain > 0) {
+      return -1;
-    size_t rlen = fread(buffer, 1, remain, in_file);
-    if (rlen <= 0) {
-      return writeLen;
-    } else {
-      fwrite(buffer, 1, remain, out_file);
-      writeLen += remain;
     }
+    len -= m;
   }
-
+  return count - len;
-  return writeLen;
 }
 
 int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) {
-  lseek(sfd, (int32_t)(*offset), 0);
+  int r = 0;
-  int64_t writeLen = 0;
+  if (offset) {
-  uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
+    r = lseek(sfd, *offset, SEEK_SET);
-  
+    if (r==-1) return -1;
-  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-    int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
-    if (rlen <= 0) {
-      return writeLen;
   }
-    else if (rlen < _SEND_FILE_STEP_) {
+  off_t len = count;
-      taosWriteSocket(dfd, buffer, rlen);
+  while (len>0) {
-      return (int64_t)(writeLen + rlen);
+    char buf[1024*16];
+    off_t n = sizeof(buf);
+    if (len<n) n = len;
+    size_t m = read(sfd, buf, n);
+    if (m==-1) return -1;
+    if (m==0) break;
+    size_t l = write(dfd, buf, m);
+    if (l==-1) return -1;
+    len -= l;
   }
-    else {
+  return count - len;
-      taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
-      writeLen += _SEND_FILE_STEP_;
-    }
-  }
-
-  int64_t remain = count - writeLen;
-  if (remain > 0) {
-    int32_t rlen = read(sfd, buffer, (int32_t)remain);
-    if (rlen <= 0) {
-      return writeLen;
-    }
-    else {
-      taosWriteSocket(sfd, buffer, (int32_t)remain);
-      writeLen += remain;
-    }
-  }
-
-  return writeLen;
 }
+

src/os/src/darwin/darwinSemphone.c

@@ -21,6 +21,8 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 
+#include <libproc.h>
+
 // #define SEM_USE_PTHREAD
 // #define SEM_USE_POSIX
 #define SEM_USE_SEM
@@ -279,3 +281,41 @@ int tsem_destroy(tsem_t *sem) {
   return 0;
 }
 
+bool taosCheckPthreadValid(pthread_t thread) {
+  uint64_t id = 0;
+  int r = pthread_threadid_np(thread, &id);
+  return r ? false : true;
+}
+
+int64_t taosGetSelfPthreadId() {
+  return (int64_t)pthread_self();
+}
+
+int64_t taosGetPthreadId(pthread_t thread) {
+  return (int64_t)thread;
+}
+
+void taosResetPthread(pthread_t* thread) {
+  *thread = NULL;
+}
+
+bool taosComparePthread(pthread_t first, pthread_t second) {
+  return pthread_equal(first, second) ? true : false;
+}
+
+int32_t taosGetPId() {
+  return (int32_t)getpid();
+}
+
+int32_t taosGetCurrentAPPName(char* name, int32_t* len) {
+  char buf[PATH_MAX+1];
+  buf[0] = '\0';
+  proc_name(getpid(), buf, sizeof(buf)-1);
+  buf[PATH_MAX] = '\0';
+  size_t n = strlen(buf);
+  if (len) *len = n;
+  if (name) strcpy(name, buf);
+  return 0;
+}
+
+

src/os/src/darwin/darwinSysInfo.c

@@ -18,44 +18,140 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
+#include <errno.h>
+#include <libproc.h>
+
 
 static void taosGetSystemTimezone() {
-  // get and set default timezone
   SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
-  if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
+  if (cfg_timezone == NULL) return;
-    char *tz = getenv("TZ");
+  if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) {
-    if (tz == NULL || strlen(tz) == 0) {
+    return;
-      strcpy(tsTimezone, "not configured");
   }
-    else {
+
-      strcpy(tsTimezone, tz);
+  /* load time zone string from /etc/localtime */
+  char buf[4096];
+  char *tz = NULL; {
+    int n = readlink("/etc/localtime", buf, sizeof(buf));
+    if (n<0) {
+      uError("read /etc/localtime error, reason:%s", strerror(errno));
+      return;
     }
-    cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
+    buf[n] = '\0';
-    uInfo("timezone not configured, use default");
+    for(int i=n-1; i>=0; --i) {
+      if (buf[i]=='/') {
+        if (tz) {
+          tz = buf + i + 1;
+          break;
         }
+        tz = buf + i + 1;
+      }
+    }
+    if (!tz || 0==strchr(tz, '/')) {
+      uError("parsing /etc/localtime failed");
+      return;
+    }
+
+    setenv("TZ", tz, 1);
+    tzset();
+  }
+
+  /*
+   * NOTE: do not remove it.
+   * Enforce set the correct daylight saving time(DST) flag according
+   * to current time
+   */
+  time_t    tx1 = time(NULL);
+  struct tm tm1;
+  localtime_r(&tx1, &tm1);
+
+  /*
+   * format example:
+   *
+   * Asia/Shanghai   (CST, +0800)
+   * Europe/London   (BST, +0100)
+   */
+  snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)",
+           tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone/3600);
+
+  // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
+  uWarn("timezone not configured, set to system default:%s", tsTimezone);
 }
 
-static void taosGetSystemLocale() {
+/*
-  // get and set default locale
+ * originally from src/os/src/detail/osSysinfo.c
+ * POSIX format locale string:
+ * (Language Strings)_(Country/Region Strings).(code_page)
+ *
+ * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
+ *
+ * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
+ *
+ * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
+ * for libiconv that is employed to convert string in this system. This program will automatically use
+ * UTF-8 instead as the charset.
+ *
+ * In case of windows client, the locale string is not valid POSIX format, user needs to set the
+ * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
+ * CP936, CP437 for English charset.
+ *
+ */
+static void taosGetSystemLocale() {  // get and set default locale
+  char  sep = '.';
+  char *locale = NULL;
+
   SGlobalCfg *cfg_locale = taosGetConfigOption("locale");
   if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
-    char *locale = setlocale(LC_CTYPE, "chs");
+    locale = setlocale(LC_CTYPE, "");
-    if (locale != NULL) {
+    if (locale == NULL) {
+      uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
+      strcpy(tsLocale, "en_US.UTF-8");
+    } else {
       tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
-      cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
+      uWarn("locale not configured, set to system default:%s", tsLocale);
-      uInfo("locale not configured, set to default:%s", tsLocale);
     }
   }
 
+  /* if user does not specify the charset, extract it from locale */
   SGlobalCfg *cfg_charset = taosGetConfigOption("charset");
   if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
-    strcpy(tsCharset, "cp936");
+    char *str = strrchr(tsLocale, sep);
-    cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
+    if (str != NULL) {
-    uInfo("charset not configured, set to default:%s", tsCharset);
+      str++;
+
+      char *revisedCharset = taosCharsetReplace(str);
+      tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN);
+
+      free(revisedCharset);
+      uWarn("charset not configured, set to system default:%s", tsCharset);
+    } else {
+      strcpy(tsCharset, "UTF-8");
+      uWarn("can't get locale and charset from system, set it to UTF-8");
+    }
   }
 }
 
-void taosPrintOsInfo() {}
+void taosPrintOsInfo() {
+  uInfo(" os pageSize:            %" PRId64 "(KB)", tsPageSize / 1024);
+  // uInfo(" os openMax:             %" PRId64, tsOpenMax);
+  // uInfo(" os streamMax:           %" PRId64, tsStreamMax);
+  uInfo(" os numOfCores:          %d", tsNumOfCores);
+  uInfo(" os totalDisk:           %f(GB)", tsTotalDataDirGB);
+  uInfo(" os totalMemory:         %d(MB)", tsTotalMemoryMB);
+
+  struct utsname buf;
+  if (uname(&buf)) {
+    uInfo(" can't fetch os info");
+    return;
+  }
+  uInfo(" os sysname:             %s", buf.sysname);
+  uInfo(" os nodename:            %s", buf.nodename);
+  uInfo(" os release:             %s", buf.release);
+  uInfo(" os version:             %s", buf.version);
+  uInfo(" os machine:             %s", buf.machine);
+  uInfo("==================================");
+}
 
 void taosKillSystem() {
   uError("function taosKillSystem, exit!");
@@ -63,12 +159,26 @@ void taosKillSystem() {
 }
 
 void taosGetSystemInfo() {
+  // taosGetProcInfos();
+
+  tsNumOfCores        = sysconf(_SC_NPROCESSORS_ONLN);
+  long physical_pages = sysconf(_SC_PHYS_PAGES);
+  long page_size      = sysconf(_SC_PAGESIZE);
+  tsTotalMemoryMB     = physical_pages * page_size / (1024 * 1024);
+  tsPageSize          = page_size;
+
+  // float tmp1, tmp2;
+  // taosGetSysMemory(&tmp1);
+  // taosGetProcMemory(&tmp2);
+  // taosGetDisk();
+  // taosGetBandSpeed(&tmp1);
+  // taosGetCpuUsage(&tmp1, &tmp2);
+  // taosGetProcIO(&tmp1, &tmp2);
+
   taosGetSystemTimezone();
   taosGetSystemLocale();
 }
 
-bool taosGetDisk() { return true; }
-
 bool taosGetProcIO(float *readKB, float *writeKB) {
   *readKB = 0;
   *writeKB = 0;
@@ -103,14 +213,37 @@ int taosSystem(const char *cmd) {
 
 void taosSetCoreDump() {}
 
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
+  struct statvfs info;
+  if (statvfs(tsDataDir, &info)) {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  } else {
+    diskSize->tsize = info.f_blocks * info.f_frsize;
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
+  }
+}
+
+char cmdline[1024];
+
 char *taosGetCmdlineByPID(int pid) {
-  return "[not supported yet]";
+  errno = 0;
+
+  if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) {
+    fprintf(stderr, "PID is %d, %s", pid, strerror(errno));
+    return strerror(errno);
+  }
+
+  return cmdline;
 }
 
 bool taosGetSystemUid(char *uid) {
   uuid_t uuid = {0};
   uuid_generate(uuid);
   // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null
-  uuid_unparse(uuid, uid);
+  uuid_unparse_lower(uuid, uid);
   return true;
 }
+

src/os/src/detail/osDir.c

@@ -51,19 +51,22 @@ int taosMkDir(const char *path, mode_t mode) {
   return code;
 }
 
-void taosRename(char* oldName, char *newName) {
+
-  // if newName in not empty, rename return fail. 
+#ifndef TAOS_OS_FUNC_DIR
-  // the newName must be empty or does not exist
+
-#ifdef WINDOWS
+int32_t taosRename(char* oldName, char *newName) {
-  remove(newName);
+  int32_t code = rename(oldName, newName);
-#endif
+  if (code < 0) {
-  if (rename(oldName, newName)) {
     uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
   } else {
-    uInfo("successfully to rename file %s to %s", oldName, newName);
+    uTrace("successfully to rename file %s to %s", oldName, newName);
   }
+
+  return code;
 }
 
+#endif
+
 void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) {
   DIR *dir = opendir(rootDir);
   if (dir == NULL) return;

src/os/src/detail/osFile.c

@@ -119,6 +119,42 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
   return (int64_t)lseek(fd, (long)offset, whence);
 }
 
+int64_t taosCopy(char *from, char *to) {
+  char    buffer[4096];
+  int     fidto = -1, fidfrom = -1;
+  int64_t size = 0;
+  int64_t bytes;
+
+  fidfrom = open(from, O_RDONLY | O_BINARY);
+  if (fidfrom < 0) goto _err;
+
+  fidto = open(to, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
+  if (fidto < 0) goto _err;
+
+  while (true) {
+    bytes = taosRead(fidfrom, buffer, sizeof(buffer));
+    if (bytes < 0) goto _err;
+    if (bytes == 0) break;
+
+    size += bytes;
+
+    if (taosWrite(fidto, (void *)buffer, bytes) < bytes) goto _err;
+    if (bytes < sizeof(buffer)) break;
+  }
+
+  fsync(fidto);
+
+  close(fidfrom);
+  close(fidto);
+  return size;
+
+_err:
+  if (fidfrom >= 0) close(fidfrom);
+  if (fidto >= 0) close(fidto);
+  remove(to);
+  return -1;
+}
+
 #ifndef TAOS_OS_FUNC_FILE_SENDIFLE
 
 int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) {

src/os/src/detail/osMemory.c

@@ -512,8 +512,9 @@ void * taosTRealloc(void *ptr, size_t size) {
   return (void *)((char *)tptr + sizeof(size_t));
 }
 
-void taosTZfree(void *ptr) {
+void* taosTZfree(void* ptr) {
   if (ptr) {
-    free((void *)((char *)ptr - sizeof(size_t)));
+    free((void*)((char*)ptr - sizeof(size_t)));
   }
+  return NULL;
 }

src/os/src/detail/osSysinfo.c

@@ -18,6 +18,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
 
 #ifndef TAOS_OS_FUNC_SYSINFO
 
@@ -316,37 +317,17 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   struct statvfs info;
-  const double   unit = 1024 * 1024 * 1024;
-
-  if (tscEmbedded) {
   if (statvfs(tsDataDir, &info)) {
     uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
-      return false;
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   } else {
-      tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
+    diskSize->tsize = info.f_blocks * info.f_frsize;
-      tsAvailDataDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
   }
-  }
-
-  if (statvfs(tsLogDir, &info)) {
-    uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno));
-    return false;
-  } else {
-    tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-    tsAvailLogDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-  }
-
-  if (statvfs("/tmp", &info)) {
-    uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno));
-    return false;
-  } else {
-    tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-    tsAvailTmpDirectorySpace = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-  }
-
-  return true;
 }
 
 static bool taosGetCardInfo(int64_t *bytes) {
@@ -510,7 +491,7 @@ void taosGetSystemInfo() {
   float tmp1, tmp2;
   taosGetSysMemory(&tmp1);
   taosGetProcMemory(&tmp2);
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);
@@ -537,7 +518,6 @@ void taosPrintOsInfo() {
   uInfo(" os release:             %s", buf.release);
   uInfo(" os version:             %s", buf.version);
   uInfo(" os machine:             %s", buf.machine);
-  uInfo("==================================");
 }
 
 void taosKillSystem() {

src/os/src/windows/wFile.c

@@ -160,7 +160,6 @@ int32_t taosFtruncate(int32_t fd, int64_t l_size) {
   return 0;
 }
 
-
 int fsync(int filedes) { 
   if (filedes < 0) {
     errno = EBADF;
@@ -172,3 +171,14 @@ int fsync(int filedes) {
   
   return FlushFileBuffers(h);
 }
+
+int32_t taosRename(char* oldName, char *newName) {
+  int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
+  if (code < 0) {
+    uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+  } else {
+    uTrace("successfully to rename file %s to %s", oldName, newName);
+  }
+
+  return code;
+}

src/os/src/windows/wString.c

@@ -75,18 +75,6 @@ char *getpass(const char *prefix) {
   return passwd;
 }
 
-char *strndup(const char *s, size_t n) {
-  size_t len = strlen(s);
-  if (len >= n) {
-    len = n;
-  }
-
-  char *r = calloc(len + 1, 1);
-  memcpy(r, s, len);
-  r[len] = 0;
-  return r;
-}
-
 int twcslen(const wchar_t *wcs) {
   int *wstr = (int *)wcs;
   if (NULL == wstr) {

src/os/src/windows/wSysinfo.c

@@ -21,6 +21,7 @@
 #include "ttimer.h"
 #include "tulog.h"
 #include "tutil.h"
+#include "taoserror.h"
 #if (_WIN64)
 #include <iphlpapi.h>
 #include <mswsock.h>
@@ -126,37 +127,22 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
-  const double    unit = 1024 * 1024 * 1024;
-  BOOL            fResult;
   unsigned _int64 i64FreeBytesToCaller;
   unsigned _int64 i64TotalBytes;
   unsigned _int64 i64FreeBytes;
 
-  if (tscEmbedded) {
+  BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-    fResult = GetDiskFreeSpaceExA(tsDataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
                                      (PULARGE_INTEGER)&i64FreeBytes);
   if (fResult) {
-      tsTotalDataDirGB = (float)(i64TotalBytes / unit);
+    diskSize->tsize = (int64_t)(i64TotalBytes);
-      tsAvailDataDirGB = (float)(i64FreeBytes / unit);
+    diskSize->avail = (int64_t)(i64FreeBytes);
+    return 0;
+  } else {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   }
-  }
-
-  fResult = GetDiskFreeSpaceExA(tsLogDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
-  if (fResult) {
-    tsTotalLogDirGB = (float)(i64TotalBytes / unit);
-    tsAvailLogDirGB = (float)(i64FreeBytes / unit);
-  }
-
-  fResult = GetDiskFreeSpaceExA(tsTempDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
-  if (fResult) {
-    tsTotalTmpDirGB = (float)(i64TotalBytes / unit);
-    tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit);
-  }
-
-  return true;
 }
 
 bool taosGetBandSpeed(float *bandSpeedKb) {
@@ -207,7 +193,7 @@ void taosGetSystemInfo() {
   tsTotalMemoryMB = taosGetTotalMemory();
 
   float tmp1, tmp2;
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);

src/os/src/windows/wWordexp.c

@@ -21,13 +21,20 @@
 #include "tulog.h"
 #include "tutil.h"
 
-int wordexp(const char *words, wordexp_t *pwordexp, int flags) {
+int wordexp(char *words, wordexp_t *pwordexp, int flags) {
   pwordexp->we_offs = 0;
   pwordexp->we_wordc = 1;
-  pwordexp->we_wordv = (char **)(pwordexp->wordPos);
+  pwordexp->we_wordv[0] = pwordexp->wordPos;
-  pwordexp->we_wordv[0] = (char *)words;
+
+  memset(pwordexp->wordPos, 0, 1025);
+  if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) {
+    pwordexp->we_wordv[0] = words;
+    uError("failed to parse relative path:%s to abs path", words);
+    return -1;
+  }
+
+  uTrace("parse relative path:%s to abs path:%s", words, pwordexp->wordPos);
   return 0;
 }
 
 void wordfree(wordexp_t *pwordexp) {}
-

src/plugins/http/src/httpRestJson.c

@@ -35,7 +35,7 @@ void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t
   // data row array end
   httpJsonToken(jsonBuf, JsonArrEnd);
 
-  cmd->numOfRows = affect_rows;
+  cmd->numOfRows = 1;
 }
 
 void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) {

src/plugins/http/src/httpSystem.c

@@ -92,6 +92,11 @@ void httpStopSystem() {
   tsHttpServer.stop = 1;
 #ifdef WINDOWS
   closesocket(tsHttpServer.fd);
+#elif __APPLE__
+  if (tsHttpServer.fd!=-1) {
+    close(tsHttpServer.fd);
+    tsHttpServer.fd = -1;
+  }
 #else
   shutdown(tsHttpServer.fd, SHUT_RD);
 #endif

src/plugins/monitor/src/monMain.c

@@ -20,7 +20,6 @@
 #include "tlog.h"
 #include "ttimer.h"
 #include "tutil.h"
-#include "tsystem.h"
 #include "tscUtil.h"
 #include "tsclient.h"
 #include "dnode.h"

src/query/inc/qAggMain.h

@@ -59,25 +59,27 @@ extern "C" {
 
 #define TSDB_FUNC_FIRST_DST    25
 #define TSDB_FUNC_LAST_DST     26
-#define TSDB_FUNC_INTERP       27
+#define TSDB_FUNC_STDDEV_DST   27
+#define TSDB_FUNC_INTERP       28
 
-#define TSDB_FUNC_RATE         28
+#define TSDB_FUNC_RATE         29
-#define TSDB_FUNC_IRATE        29
+#define TSDB_FUNC_IRATE        30
-#define TSDB_FUNC_SUM_RATE     30
+#define TSDB_FUNC_SUM_RATE     31
-#define TSDB_FUNC_SUM_IRATE    31
+#define TSDB_FUNC_SUM_IRATE    32
-#define TSDB_FUNC_AVG_RATE     32
+#define TSDB_FUNC_AVG_RATE     33
-#define TSDB_FUNC_AVG_IRATE    33
+#define TSDB_FUNC_AVG_IRATE    34
+
+#define TSDB_FUNC_TID_TAG      35
+#define TSDB_FUNC_HISTOGRAM    36
+#define TSDB_FUNC_HLL          37
+#define TSDB_FUNC_MODE         38
+#define TSDB_FUNC_SAMPLE       39
+#define TSDB_FUNC_CEIL         40
+#define TSDB_FUNC_FLOOR        41
+#define TSDB_FUNC_ROUND        42
+#define TSDB_FUNC_MAVG         43
+#define TSDB_FUNC_CSUM         44
 
-#define TSDB_FUNC_TID_TAG      34
-#define TSDB_FUNC_HISTOGRAM    35
-#define TSDB_FUNC_HLL          36
-#define TSDB_FUNC_MODE         37
-#define TSDB_FUNC_SAMPLE       38
-#define TSDB_FUNC_CEIL         39
-#define TSDB_FUNC_FLOOR        40
-#define TSDB_FUNC_ROUND        41
-#define TSDB_FUNC_MAVG         42
-#define TSDB_FUNC_CSUM         43
 
 #define TSDB_FUNCSTATE_SO           0x1u    // single output
 #define TSDB_FUNCSTATE_MO           0x2u    // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
@@ -90,15 +92,12 @@ extern "C" {
 #define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF
 #define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF
 
-
 #define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16
 #define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50
 
 #define DATA_SET_FLAG ','  // to denote the output area has data, not null value
 #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG)
 
-
-
 #define QUERY_ASC_FORWARD_STEP   1
 #define QUERY_DESC_FORWARD_STEP -1
 
@@ -167,8 +166,9 @@ typedef struct SExtTagsInfo {
 
 // sql function runtime context
 typedef struct SQLFunctionCtx {
-  int32_t      startOffset;
+  int32_t      startOffset;  // todo remove it
   int32_t      size;      // number of rows
+  void *       pInput;    //
   uint32_t     order;     // asc|desc
   int16_t      inputType;
   int16_t      inputBytes;
@@ -180,10 +180,9 @@ typedef struct SQLFunctionCtx {
   bool         requireNull;   // require null in some function
   bool         stableQuery;
   int16_t      functionId;    // function id
-  void *       aInputElemBuf;
+  char *       pOutput;       // final result output buffer, point to sdata->data
-  char *       aOutputBuf;            // final result output buffer, point to sdata->data
   uint8_t      currentStage;  // record current running step, default: 0
-  int64_t      nStartQueryTimestamp;  // timestamp range of current query when function is executed on a specific data block
+  int64_t      startTs;       // timestamp range of current query when function is executed on a specific data block
   int32_t      numOfParams;
   tVariant     param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */
   int64_t     *ptsList;       // corresponding timestamp array list
@@ -198,17 +197,16 @@ typedef struct SQLFunctionCtx {
   SPoint1      end;
 } SQLFunctionCtx;
 
-typedef struct SQLAggFuncElem {
+typedef struct SAggFunctionInfo {
-  char     aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH];
+  char     name[TSDB_FUNCTIONS_NAME_MAX_LENGTH];
-
+  uint8_t  index;       // index of function in aAggs
-  uint8_t  nAggIdx;       // index of function in aAggs
   int8_t   stableFuncId;  // transfer function for super table query
-  uint16_t nStatus;
+  uint16_t status;
 
   bool (*init)(SQLFunctionCtx *pCtx);  // setup the execute environment
 
   void (*xFunction)(SQLFunctionCtx *pCtx);                     // blocks version function
-  void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position);  // single-row function version
+  void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position);  // single-row function version, todo merge with blockwise function
 
   // some sql function require scan data twice or more, e.g.,stddev, percentile
   void (*xNextStep)(SQLFunctionCtx *pCtx);
@@ -218,7 +216,7 @@ typedef struct SQLAggFuncElem {
   void (*mergeFunc)(SQLFunctionCtx *pCtx);
 
   int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId);
-} SQLAggFuncElem;
+} SAggFunctionInfo;
 
 #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
 
@@ -246,7 +244,7 @@ typedef struct STwaInfo {
 } STwaInfo;
 
 /* global sql function array */
-extern struct SQLAggFuncElem aAggs[];
+extern struct SAggFunctionInfo aAggs[];
 
 extern int32_t functionCompatList[]; // compatible check array list
 

src/query/inc/qExecutor.h

@@ -143,6 +143,11 @@ typedef struct {
   int64_t ts;
 } SOrderedPrjQueryInfo;
 
+typedef struct {
+  char*   tags;
+  SArray* pResult;  // SArray<SStddevInterResult>
+} SInterResult;
+
 typedef struct SQuery {
   int16_t          numOfCols;
   int16_t          numOfTags;
@@ -154,7 +159,12 @@ typedef struct SQuery {
   int16_t          fillType;
   int16_t          checkResultBuf;   // check if the buffer is full during scan each block
   SLimitVal        limit;
-  int32_t          rowSize;
+
+  int32_t          srcRowSize;       // todo extract struct
+  int32_t          resultRowSize;
+  int32_t          maxSrcColumnSize;
+  int32_t          tagLen;           // tag value length of current query
+
   SSqlGroupbyExpr* pGroupbyExpr;
   SExprInfo*       pExpr1;
   SExprInfo*       pExpr2;
@@ -184,8 +194,7 @@ typedef struct SQueryRuntimeEnv {
   uint16_t             scanFlag;         // denotes reversed scan of data or not
   SFillInfo*           pFillInfo;
   SResultRowInfo       windowResInfo;
-  STSBuf*              pTsBuf;
+
-  STSCursor            cur;
   SQueryCostInfo       summary;
   void*                pQueryHandle;
   void*                pSecQueryHandle;  // another thread for
@@ -205,8 +214,12 @@ typedef struct SQueryRuntimeEnv {
 
   int32_t*             rowCellInfoOffset;// offset value for each row result cell info
   char**               prevRow;
-  char**               nextRow;
 
+  SArray*              prevResult;       // intermediate result, SArray<SInterResult>
+  STSBuf*              pTsBuf;           // timestamp filter list
+  STSCursor            cur;
+
+  char*                tagVal;           // tag value of current data block
   SArithmeticSupport  *sasArray;
 } SQueryRuntimeEnv;
 

src/query/inc/qFill.h

@@ -68,7 +68,7 @@ typedef struct SPoint {
   void *  val;
 } SPoint;
 
-SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
+SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
                             int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
                             SFillColInfo* pFillCol, void* handle);
 
@@ -78,7 +78,7 @@ void* taosDestroyFillInfo(SFillInfo *pFillInfo);
 
 void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
 
-void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput);
+void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput);
 
 void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput);
 

src/query/inc/qUtil.h

@@ -15,6 +15,8 @@
 #ifndef TDENGINE_QUERYUTIL_H
 #define TDENGINE_QUERYUTIL_H
 
+#include "tbuffer.h"
+
 #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid)     \
   do {                                               \
     assert(sizeof(_uid) == sizeof(uint64_t));        \
@@ -74,5 +76,13 @@ int32_t getNumOfUsedResultRows(SResultRowPool* p);
 
 bool isPointInterpoQuery(SQuery *pQuery);
 
+typedef struct {
+  SArray* pResult;     // SArray<SResPair>
+  int32_t colId;
+} SStddevInterResult;
+
+void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen);
+SArray* interResFromBinary(const char* data, int32_t len);
+void freeInterResult(void* param);
 
 #endif  // TDENGINE_QUERYUTIL_H

src/query/inc/queryLog.h

@@ -30,6 +30,7 @@ extern uint32_t qDebugFlag;
 #define qInfo(...)  do { if (qDebugFlag & DEBUG_INFO)  { taosPrintLog("QRY ", 255, __VA_ARGS__); }}        while(0)
 #define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
 #define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP)  { taosDumpData((unsigned char *)a, l); }} while(0)
 
 #ifdef __cplusplus
 }

src/query/src/qFill.c

@@ -321,7 +321,7 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
   return pFillInfo->numOfRows - pFillInfo->index;
 }
 
-SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
+SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
                             int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
                             SFillColInfo* pCol, void* handle) {
   if (fillType == TSDB_FILL_NONE) {
@@ -414,7 +414,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
 }
 
 // copy the data into source data buffer
-void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) {
+void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) {
   for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
     memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes);
   }