Merge branch 'develop' into feature/query

.gitignore

@@ -11,6 +11,7 @@ debs/
 rpms/
 mac/
 *.pyc
+.mypy_cache
 *.tmp
 *.swp
 src/connector/nodejs/node_modules/
@@ -33,5 +34,10 @@ Target/
 *.failed
 *.sql
 sim/
+psim/
+pysim/
+*.out
 *DS_Store
 
+# Doxygen Generated files
+html/

.travis.yml

@@ -6,6 +6,7 @@
 #
 matrix:
   - os: linux
+    dist: bionic
     language: c
 
     git:
@@ -49,7 +50,7 @@ matrix:
           ./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $?
 
           cd ${TRAVIS_BUILD_DIR}/tests/pytest
-          ./valgrind-test.sh -g 2>&1 | tee mem-error-out.txt
+          ./valgrind-test.sh 2>&1 > mem-error-out.txt
           sleep 1
 
           # Color setting
@@ -59,9 +60,9 @@ matrix:
           GREEN_UNDERLINE='\033[4;32m'
           NC='\033[0m'
 
-          grep 'ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt
+          grep 'start to execute\|ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt
 
-          for memError in `cat uniq-mem-error-out.txt | awk '{print $4}'` 
+          for memError in `grep 'ERROR SUMMARY' uniq-mem-error-out.txt | awk '{print $4}'` 
           do
             if [ -n "$memError" ]; then
               if [ "$memError" -gt 12 ]; then
@@ -72,8 +73,8 @@ matrix:
             fi
           done
 
-          grep 'definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt
+          grep 'start to execute\|definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt
-          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'`
+          for defiMemError in `grep 'definitely lost:' uniq-definitely-lost-out.txt | awk '{print $7}'`
           do
             if [ -n "$defiMemError" ]; then
               if [ "$defiMemError" -gt 13 ]; then
@@ -203,6 +204,29 @@ matrix:
           ;;
         esac
 
+  - os: linux
+    dist: trusty
+    language: c
+    git:
+      - depth: 1
+
+    addons:
+      apt:
+        packages:
+          - build-essential
+          - cmake
+    env: 
+      - DESC="trusty/gcc-4.8 build"
+
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
+      - mkdir debug
+      - cd debug
+
+    script:
+      - cmake .. > /dev/null
+      - make > /dev/null
+
   - os: linux
     language: c
     compiler: clang

README.md

@@ -115,251 +115,6 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo
 - [RESTful API](https://www.taosdata.com/en/documentation/connector/#RESTful-Connector)
 - [Node.js](https://www.taosdata.com/en/documentation/connector/#Node.js-Connector)
 
-# How to run the test cases and how to add a new test case?
-
-### Prepare development environment
-
-1.  sudo apt install
-    build-essential cmake net-tools python-pip python-setuptools python3-pip
-    python3-setuptools valgrind psmisc curl
-
-2.  git clone <https://github.com/taosdata/TDengine>; cd TDengine
-
-3.  mkdir debug; cd debug; cmake ..; make ; sudo make install
-
-4.  pip install src/connector/python/linux/python2 ; pip3 install
-    src/connector/python/linux/python3
-
-### How to run TSIM test suite
-
-1.  cd \<TDengine\>/tests/script
-
-2.  sudo ./test.sh
-
-### How to run Python test suite
-
-1.  cd \<TDengine\>/tests/pytest
-
-2.  ./smoketest.sh \# for smoke test
-
-3.  ./smoketest.sh -g \# for memory leak detection test with valgrind
-
-4.  ./fulltest.sh \# for full test
-
->   Note1: TDengine daemon's configuration and data files are stored in
->   \<TDengine\>/sim directory. As a historical design, it's same place with
->   TSIM script. So after the TSIM script ran with sudo privilege, the directory
->   has been used by TSIM then the python script cannot write it by a normal
->   user. You need to remove the directory completely first before running the
->   Python test case. We should consider using two different locations to store
->   for TSIM and Python script.
-
->   Note2: if you need to debug crash problem with a core dump, you need
->   manually edit smoketest.sh or fulltest.sh to add "ulimit -c unlimited"
->   before the script line. Then you can look for the core file in
->   \<TDengine\>/tests/pytest after the program crash.
-
-### How to add a new test case
-
-**1. add a new TSIM test cases:**
-
-TSIM test cases are now included in the new development branch and can be
-added to the TDengine/tests/script/test.sh script based on the manual test
-methods necessary to add test cases as described above.
-
-**2. add a new Python test cases:**
-
-**2.1 Please refer to \<TDengine\>/tests/pytest/insert/basic.py to add a new
-test case.** The new test case must implement 3 functions, where self.init()
-and self.stop() simply copy the contents of insert/basic.py and the test
-logic is implemented in self.run(). You can refer to the code in the util
-directory for more information.
-
-**2.2 Edit smoketest.sh to add the path and filename of the new test case**
-
-Note: The Python test framework may continue to be improved in the future,
-hopefully, to provide more functionality and ease of writing test cases. The
-method of writing the test case above does not exclude that it will also be
-affected.
-
-**2.3 What test.py does in detail:**
-
-test.py is the entry program for test case execution and monitoring.
-
-test.py has the following functions.
-
-\-f --file, Specifies the test case file name to be executed
--p --path, Specifies deployment path
-
-\-m --master, Specifies the master server IP for cluster deployment 
--c--cluster, test cluster function
--s--stop, terminates all running nodes
-
-\-g--valgrind, load valgrind for memory leak detection test
-
-\-h--help, display help
-
-**2.4 What util/log.py does in detail:**
-
-log.py is quite simple, the main thing is that you can print the output in
-different colors as needed. The success() should be called for successful
-test case execution and the success() will print green text. The exit() will
-print red text and exit the program, exit() should be called for test
-failure.
-
-**util/log.py**
-
-...
-
-    def info(self, info):
-
-        printf("%s %s" % (datetime.datetime.now(), info))
-
- 
-
-    def sleep(self, sec):
-
-        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec))
-
-        time.sleep(sec)
-
- 
-
-    def debug(self, err):
-
-        printf("\\033[1;36m%s %s\\033[0m" % (datetime.datetime.now(), err))
-
- 
-
-    def success(self, info):
-
-        printf("\\033[1;32m%s %s\\033[0m" % (datetime.datetime.now(), info))
-
- 
-
-    def notice(self, err):
-
-        printf("\\033[1;33m%s %s\\033[0m" % (datetime.datetime.now(), err))
-
- 
-
-    def exit(self, err):
-
-        printf("\\033[1;31m%s %s\\033[0m" % (datetime.datetime.now(), err))
-
-        sys.exit(1)
-
- 
-
-    def printNoPrefix(self, info):
-
-        printf("\\033[1;36m%s\\033[0m" % (info)
-
-...
-
-**2.5 What util/sql.py does in detail:**
-
-SQL.py is mainly used to execute SQL statements to manipulate the database,
-and the code is extracted and commented as follows:
-
-**util/sql.py**
-
-\# prepare() is mainly used to set up the environment for testing table and
-data, and to set up the database db for testing. do not call prepare() if you
-need to test the database operation command.
-
-def prepare(self):
-
-tdLog.info("prepare database:db")
-
-self.cursor.execute('reset query cache')
-
-self.cursor.execute('drop database if exists db')
-
-self.cursor.execute('create database db')
-
-self.cursor.execute('use db')
-
-...
-
-\# query() is mainly used to execute select statements for normal syntax input
-
-def query(self, sql):
-
-...
-
-\# error() is mainly used to execute the select statement with the wrong syntax
-input, the error will be caught as a reasonable behavior, if not caught it will
-prove that the test failed
-
-def error()
-
-...
-
-\# checkRows() is used to check the number of returned lines after calling
-query(select ...) after calling the query(select ...) to check the number of
-rows of returned results.
-
-def checkRows(self, expectRows):
-
-...
-
-\# checkData() is used to check the returned result data after calling
-query(select ...) after the query(select ...) is called, failure to meet
-expectation is
-
-def checkData(self, row, col, data):
-
-...
-
-\# getData() returns the result data after calling query(select ...) to return
-the resulting data after calling query(select ...)
-
-def getData(self, row, col):
-
-...
-
-\# execute() used to execute sql and return the number of affected rows
-
-def execute(self, sql):
-
-...
-
-\# executeTimes() Multiple executions of the same sql statement
-
-def executeTimes(self, sql, times):
-
-...
-
-\# CheckAffectedRows() Check if the number of affected rows is as expected
-
-def checkAffectedRows(self, expectAffectedRows):
-
-...
-
->   Note: Both Python2 and Python3 are currently supported by the Python test
->   case. Since Python2 is no longer officially supported by January 1, 2020, it
->   is recommended that subsequent test case development be guaranteed to run
->   correctly on Python3. For Python2, please consider being compatible if
->   appropriate without additional
->   burden. <https://nakedsecurity.sophos.com/2020/01/03/python-is-dead-long-live-python/> 
-
-### CI Covenant submission adoption principle.
-
--   Every commit / PR compilation must pass. Currently, the warning is treated
-    as an error, so the warning must also be resolved.
-
--   Test cases that already exist must pass.
-
--   Because CI is very important to support build and automatically test
-    procedure, it is necessary to manually test the test case before adding it
-    and do as many iterations as possible to ensure that the test case provides
-    stable and reliable test results when added.
-
->   Note: In the future, according to the requirements and test development
->   progress will add stress testing, performance testing, code style, 
->   and other features based on functional testing.
-
 ### Third Party Connectors
 
 The TDengine community has also kindly built some of their own connectors! Follow the links below to find the source code for them.
@@ -367,6 +122,10 @@ The TDengine community has also kindly built some of their own connectors! Follo
 - [Rust Connector](https://github.com/taosdata/TDengine/tree/master/tests/examples/rust)
 - [.Net Core Connector](https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos)
 
+# How to run the test cases and how to add a new test case?
+  TDengine's test framework and all test cases are fully open source.
+  Please refer to [this document](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md) for how to run test and develop new test case.
+
 # TDengine Roadmap
 - Support event-driven stream computing
 - Support user defined functions

src/client/inc/tscUtil.h

@@ -43,7 +43,7 @@ extern "C" {
 
 typedef struct SParsedColElem {
   int16_t colIndex;
-  int16_t offset;
+  uint16_t offset;
 } SParsedColElem;
 
 typedef struct SParsedDataColInfo {
@@ -264,6 +264,7 @@ bool hasMoreVnodesToTry(SSqlObj *pSql);
 void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
 void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
 void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)());
+int  tscSetMgmtIpListFromCfg(const char *first, const char *second);
 
 void* malloc_throw(size_t size);
 void* calloc_throw(size_t nmemb, size_t size);

src/client/inc/tsclient.h

@@ -49,7 +49,7 @@ typedef struct STableComInfo {
   uint8_t numOfTags;
   uint8_t precision;
   int16_t numOfColumns;
-  int16_t rowSize;
+  int32_t rowSize;
 } STableComInfo;
 
 typedef struct STableMeta {

src/client/src/tscFunctionImpl.c

@@ -3620,299 +3620,6 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
   doFinalizer(pCtx);
 }
 
-static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                          int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  //  int64_t lastKey = 0;
-  //  int8_t  lastVal = TSDB_DATA_TINYINT_NULL;
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((char *)&data[i], type)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (type != TSDB_DATA_TYPE_BOOL) { // ignore the bool data type pre-calculation
-    //      if (isNull((char *)&lastVal, type)) {
-    //        lastKey = primaryKey[i];
-    //        lastVal = data[i];
-    //      } else {
-    //        *wsum = lastVal * (primaryKey[i] - lastKey);
-    //        lastKey = primaryKey[i];
-    //        lastVal = data[i];
-    //      }
-    //    }
-  }
-}
-
-static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  //  int64_t lastKey = 0;
-  //  int16_t lastVal = TSDB_DATA_SMALLINT_NULL;
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-}
-
-static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  //  int64_t lastKey = 0;
-  //  int32_t lastVal = TSDB_DATA_INT_NULL;
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-}
-
-static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-}
-
-static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, double *min, double *max, double *sum,
-                         int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  float fmin      = DBL_MAX;
-  float fmax      = -DBL_MAX;
-  double dsum     = 0;
-  *minIndex       = 0;
-  *maxIndex       = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    float fv = 0;
-    fv = GET_FLOAT_VAL(&(data[i]));
-    dsum += fv;
-    if (fmin > fv) {
-      fmin = fv;
-      *minIndex = i;
-    }
-    
-    if (fmax < fv) {
-      fmax = fv;
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-  
-  double csum = 0;
-  csum = GET_DOUBLE_VAL(sum);
-  csum += dsum;
-#ifdef _TD_ARM_32_
-  SET_DOUBLE_VAL_ALIGN(sum, &csum);
-  SET_DOUBLE_VAL_ALIGN(max, &fmax);
-  SET_DOUBLE_VAL_ALIGN(min, &fmin);
-#else
-  *sum = csum;
-  *max = fmax;
-  *min = fmin;
-#endif
-}
-
-static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, double *min, double *max, double *sum,
-                         int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  double dmin      = DBL_MAX;
-  double dmax      = -DBL_MAX;
-  double dsum      = 0;
-  *minIndex        = 0;
-  *maxIndex        = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    double dv = 0;
-    dv = GET_DOUBLE_VAL(&(data[i]));
-    dsum += dv;
-    if (dmin > dv) {
-      dmin = dv;
-      *minIndex = i;
-    }
-    
-    if (dmax < dv) {
-      dmax = dv;
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-  
-  double csum = 0;
-  csum = GET_DOUBLE_VAL(sum);
-  csum += dsum;
-
-
-#ifdef _TD_ARM_32_
-  SET_DOUBLE_VAL_ALIGN(sum, &csum);
-    SET_DOUBLE_VAL_ALIGN(max, &dmax);
-    SET_DOUBLE_VAL_ALIGN(min, &dmin);
-#else
-  *sum = csum;
-  *max = dmax;
-  *min = dmin;
-#endif
-}
-
-void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, int32_t type, int64_t *min, int64_t *max,
-                   int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  int64_t *primaryKey = (int64_t *)priData;
-  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-    for (int32_t i = 0; i < numOfRow; ++i) {
-      if (isNull(data + i * size, type)) {
-        (*numOfNull) += 1;
-        continue;
-      }
-    }
-  } else {
-    if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
-      getStatics_i8(primaryKey, type, (int8_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_SMALLINT) {
-      getStatics_i16(primaryKey, (int16_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_INT) {
-      getStatics_i32(primaryKey, (int32_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) {
-      getStatics_i64(primaryKey, (int64_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_DOUBLE) {
-      getStatics_d(primaryKey, (double *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_FLOAT) {
-      getStatics_f(primaryKey, (float *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull);
-    }
-  }
-}
 
 /**
  * param[1]: start time

src/client/src/tscParseInsert.c

@@ -1016,7 +1016,9 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   }
 
-  if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) {
+  // TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536
+  // but TSDB_PAYLOAD_SIZE is 65380
+  if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
     return code;
   }
 

src/client/src/tscPrepare.c

@@ -194,7 +194,7 @@ static int normalStmtPrepare(STscStmt* stmt) {
 
 static char* normalStmtBuildSql(STscStmt* stmt) {
   SNormalStmt* normal = &stmt->normal;
-  SStringBuilder sb = {0};
+  SStringBuilder sb; memset(&sb, 0, sizeof(sb));
 
   if (taosStringBuilderSetJmp(&sb) != 0) {
     taosStringBuilderDestroy(&sb);

src/client/src/tscSQLParser.c

@@ -3558,7 +3558,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac
     return TSDB_CODE_SUCCESS;
   }
 
-  SStringBuilder sb1 = {0};
+  SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1));
   taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN);
 
   char db[TSDB_TABLE_ID_LEN] = {0};
@@ -3813,7 +3813,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
   pQueryInfo->window.ekey = INT64_MAX;
 
   // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space
-  SStringBuilder sb = {0};
+  SStringBuilder sb; memset(&sb, 0, sizeof(sb));
   SCondExpr      condExpr = {0};
 
   if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) {

src/client/src/tscServer.c

@@ -27,6 +27,7 @@
 #include "ttimer.h"
 #include "tutil.h"
 #include "tscLog.h"
+#include "qsqltype.h"
 
 #define TSC_MGMT_VNODE 999
 
@@ -67,7 +68,7 @@ void tscPrintMgmtIp() {
   }
 }
 
-void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) {
+void tscSetMgmtIpList(SRpcIpSet *pIpList) {
   tscMgmtIpSet.numOfIps = pIpList->numOfIps;
   tscMgmtIpSet.inUse = pIpList->inUse;
   for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
@@ -75,16 +76,6 @@ void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) {
   }
 }
 
-void tscSetMgmtIpListFromEdge() {
-  if (tscMgmtIpSet.numOfIps != 1) {
-    tscMgmtIpSet.numOfIps = 1;
-    tscMgmtIpSet.inUse = 0;
-    taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
-    tscTrace("edge mgmt IP list:");
-    tscPrintMgmtIp();
-  }
-}
-
 void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
   tscMgmtIpSet = *pIpSet;
   tscTrace("mgmt IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse);
@@ -93,18 +84,6 @@ void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
   }
 }
 
-void tscSetMgmtIpList(SRpcIpSet *pIpList) {
-  /*
-    * The iplist returned by the cluster edition is the current management nodes
-    * and the iplist returned by the edge edition is empty
-    */
-  if (pIpList->numOfIps != 0) {
-    tscSetMgmtIpListFromCluster(pIpList);
-  } else {
-    tscSetMgmtIpListFromEdge();
-  }
-}
-
 /*
  * For each management node, try twice at least in case of poor network situation.
  * If the client start to connect to a non-management node from the client, and the first retry may fail due to
@@ -132,7 +111,8 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
   if (code == 0) {
     SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp;
     SRpcIpSet *      pIpList = &pRsp->ipList;
-    tscSetMgmtIpList(pIpList);
+    if (pIpList->numOfIps > 0) 
+      tscSetMgmtIpList(pIpList);
 
     if (pRsp->killConnection) {
       tscKillConnection(pObj);
@@ -207,7 +187,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
     memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
   }
 
-  tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
+  // tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
 
   SRpcMsg rpcMsg = {
       .msgType = pSql->cmd.msgType,
@@ -235,7 +215,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
   STscObj *pObj = pSql->pTscObj;
-  tscTrace("%p msg:%p is received from server", pSql, rpcMsg->pCont);
+  // tscTrace("%p msg:%s is received from server", pSql, taosMsg[rpcMsg->msgType]);
 
   if (pSql->freed || pObj->signature != pObj) {
     tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
@@ -340,10 +320,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
       pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks);
 
       pRes->numOfRows += pMsg->affectedRows;
-      tscTrace("%p cmd:%d code:%s, inserted rows:%d, rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code),
+      tscTrace("%p SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql, sqlCmd[pCmd->command], 
-          pMsg->affectedRows, pRes->rspLen);
+          tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen);
     } else {
-      tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen);
+      tscTrace("%p SQL cmd:%s, code:%s rspLen:%d", pSql, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen);
     }
   }
   
@@ -426,7 +406,7 @@ int tscProcessSql(SSqlObj *pSql) {
     assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0);
   }
 
-  tscTrace("%p SQL cmd:%d will be processed, name:%s, type:%d", pSql, pCmd->command, name, type);
+  tscTrace("%p SQL cmd:%s will be processed, name:%s, type:%d", pSql, sqlCmd[pCmd->command], name, type);
   if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL
     if (pTableMetaInfo == NULL) {
       pSql->res.code = TSDB_CODE_OTHERS;
@@ -1487,15 +1467,16 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   char *         pMsg;
   int            msgLen = 0;
 
-  char *tmpData = 0;
+  char *tmpData = NULL;
-  if (pSql->cmd.allocSize > 0) {
+  uint32_t len = pSql->cmd.payloadLen;
-    tmpData = calloc(1, pSql->cmd.allocSize);
+  if (len > 0) {
+    tmpData = calloc(1, len);
     if (NULL == tmpData) {
       return TSDB_CODE_CLI_OUT_OF_MEMORY;
     }
 
     // STagData is in binary format, strncpy is not available
-    memcpy(tmpData, pSql->cmd.payload, pSql->cmd.allocSize);
+    memcpy(tmpData, pSql->cmd.payload, len);
   }
 
   SSqlCmd *   pCmd = &pSql->cmd;
@@ -1509,9 +1490,9 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg);
 
-  if (pSql->cmd.autoCreated) {
+  if (pSql->cmd.autoCreated && len > 0) {
-    memcpy(pInfoMsg->tags, tmpData, sizeof(STagData));
+    memcpy(pInfoMsg->tags, tmpData, len);
-    pMsg += sizeof(STagData);
+    pMsg += len;
   }
 
   pCmd->payloadLen = pMsg - (char*)pInfoMsg;;
@@ -2224,7 +2205,8 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   assert(len <= tListLen(pObj->db));
   strncpy(pObj->db, temp, tListLen(pObj->db));
   
-  tscSetMgmtIpList(&pConnect->ipList);
+  if (pConnect->ipList.numOfIps > 0) 
+    tscSetMgmtIpList(&pConnect->ipList);
 
   strcpy(pObj->sversion, pConnect->serverVersion);
   pObj->writeAuth = pConnect->writeAuth;
@@ -2394,7 +2376,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo);
 
   pNew->cmd.autoCreated = pSql->cmd.autoCreated;  // create table if not exists
-  if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
+  if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
     tscError("%p malloc failed for payload to get table meta", pSql);
     free(pNew);
 
@@ -2405,7 +2387,8 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
 
   strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name));
-  memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE);  // tag information if table does not exists.
+  memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen);  // tag information if table does not exists.
+  pNew->cmd.payloadLen = pSql->cmd.payloadLen;
   tscTrace("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
 
   pNew->fp = tscTableMetaCallBack;

src/client/src/tscSql.c

@@ -66,31 +66,17 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     return NULL;
   }
 
+  if (ip) {
+    if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL;
+    if (port) tscMgmtIpSet.port[0] = port;
+  } 
+ 
   void *pDnodeConn = NULL;
   if (tscInitRpc(user, pass, &pDnodeConn) != 0) {
     terrno = TSDB_CODE_NETWORK_UNAVAIL;
     return NULL;
   }
  
-  tscMgmtIpSet.numOfIps = 0;
-
-  if (ip && ip[0]) {
-    tscMgmtIpSet.inUse = 0;
-    tscMgmtIpSet.numOfIps = 1;
-    strcpy(tscMgmtIpSet.fqdn[0], ip);
-    tscMgmtIpSet.port[0] = port? port: tsDnodeShellPort;
-  } else {
-    if (tsFirst[0] != 0) {
-      taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
-      tscMgmtIpSet.numOfIps++;
-    }
-
-    if (tsSecond[0] != 0) {
-      taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
-      tscMgmtIpSet.numOfIps++;
-    }
-  }
-  
   STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
   if (NULL == pObj) {
     terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;

src/client/src/tscSub.c

@@ -14,8 +14,6 @@
  */
 
 #include "os.h"
-
-#include "shash.h"
 #include "taos.h"
 #include "trpc.h"
 #include "tsclient.h"

src/client/src/tscSubquery.c

@@ -488,7 +488,7 @@ void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables) {
     if( prev == NULL || tt->vgId != prev->vgId ) {
       SVgroupsInfo* pvg = pTableMetaInfo->vgroupList;
 
-      SVgroupTableInfo info = { 0 };
+      SVgroupTableInfo info = {{ 0 }};
       for( int32_t m = 0; m < pvg->numOfVgroups; ++m ) {
         if( tt->vgId == pvg->vgroups[m].vgId ) {
           info.vgInfo = pvg->vgroups[m];

src/client/src/tscSystem.c

@@ -23,6 +23,7 @@
 #include "tutil.h"
 #include "tsched.h"
 #include "tscLog.h"
+#include "tscUtil.h"
 #include "tsclient.h"
 #include "tglobal.h"
 #include "tconfig.h"
@@ -114,13 +115,9 @@ void taos_init_imp() {
     taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
   }
 
-  tscMgmtIpSet.inUse = 0;
+  if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) {
-  tscMgmtIpSet.numOfIps = 1;
+    tscError("failed to init mgmt IP list");
-  taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
+    return;
-
-  if (tsSecond[0] && strcmp(tsSecond, tsFirst) != 0) {
-    tscMgmtIpSet.numOfIps = 2;
-    taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]);
   } 
 
   tscInitMsgsFp();

src/client/src/tscUtil.c

@@ -2163,3 +2163,33 @@ char* strdup_throw(const char* str) {
   }
   return p;
 }
+
+int tscSetMgmtIpListFromCfg(const char *first, const char *second) {
+  tscMgmtIpSet.numOfIps = 0;
+  tscMgmtIpSet.inUse = 0;
+
+  if (first && first[0] != 0) {
+    if (strlen(first) >= TSDB_EP_LEN) {
+      terrno = TSDB_CODE_INVALID_FQDN;
+      return -1;
+    }
+    taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
+    tscMgmtIpSet.numOfIps++;
+  }
+
+  if (second && second[0] != 0) {
+    if (strlen(second) >= TSDB_EP_LEN) {
+      terrno = TSDB_CODE_INVALID_FQDN;
+      return -1;
+    }
+    taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
+    tscMgmtIpSet.numOfIps++;
+  }
+
+  if ( tscMgmtIpSet.numOfIps == 0) {
+    terrno = TSDB_CODE_INVALID_FQDN;
+    return -1;
+  }
+
+  return 0;
+}

src/common/inc/qsqltype.h

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_QSQLCMD_H
+#define TDENGINE_QSQLCMD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// sql type
+
+#ifdef TSDB_SQL_C
+#define TSDB_DEFINE_SQL_TYPE( name, msg ) msg, 
+char *sqlCmd[] = {
+  "null",
+#else
+#define TSDB_DEFINE_SQL_TYPE( name, msg ) name,
+enum {
+  TSDB_SQL_NULL = 0,
+#endif
+
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" )
+
+  // the SQL below is for mgmt node
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_ACCT, "create-acct" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_USER, "create-user" ) 
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_ACCT, "drop-acct" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_USER, "drop-user" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_USER, "alter-user" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_ACCT, "alter-acct" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DNODE, "drop-dnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_DNODE, "cfg-dnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_MNODE, "cfg-mnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW, "show" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE, "retrieve" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_QUERY, "kill-query" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_STREAM, "kill-stream" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_CONNECTION, "kill-connection" )
+
+  // SQL below is for read operation
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_READ, "read" )  
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CONNECT, "connect" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_USE_DB, "use-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_META, "meta" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_STABLEVGROUP, "stable-vgroup" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MULTI_META, "multi-meta" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_HB, "heart-beat" )
+
+  // SQL below for client local 
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_LOCAL, "local" ) 
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DESCRIBE_TABLE, "describe-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_LOCALMERGE, "retrieve-localmerge" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_TABLE_JOIN_RETRIEVE, "join-retrieve" )
+
+  /*
+   * build empty result instead of accessing dnode to fetch result
+   * reset the client cache
+   */
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_EMPTY_RESULT, "retrieve-empty-result" )
+
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RESET_CACHE, "reset-cache" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_STATUS, "serv-status" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_DB, "current-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_VERSION, "serv-version" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CLI_VERSION, "cli-version" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_USER, "current-user ")
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_LOCAL, "cfg-local" )
+
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MAX, "max" )
+};
+
+// create table operation type
+enum TSQL_TYPE {
+  TSQL_CREATE_TABLE = 0x1,
+  TSQL_CREATE_STABLE = 0x2,
+  TSQL_CREATE_TABLE_FROM_STABLE = 0x3,
+  TSQL_CREATE_STREAM = 0x4,
+};
+
+extern char *sqlCmd[];
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_QSQLCMD_H

src/common/inc/tglobal.h

@@ -175,7 +175,7 @@ void taosInitGlobalCfg();
 bool taosCheckGlobalCfg();
 void taosSetAllDebugFlag();
 bool taosCfgDynamicOptions(char *msg);
-int  taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port);
+int  taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
  
 #ifdef __cplusplus
 }

src/common/src/sqlcmdstr.c

@@ -13,14 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _sdb_str_hash_header_
+#define TSDB_SQL_C
-#define _sdb_str_hash_header_
 
-void *sdbOpenStrHash(int maxSessions, int dataSize);
+#include "qsqltype.h"
-void sdbCloseStrHash(void *handle);
-void *sdbAddStrHash(void *handle, void *key, void *pData);
-void sdbDeleteStrHash(void *handle, void *key);
-void *sdbGetStrHashData(void *handle, void *key);
-void *sdbFetchStrHashData(void *handle, void *ptr, void **ppMeta);
-
-#endif

src/common/src/tglobal.c

@@ -61,10 +61,10 @@ int32_t tscEmbedded = 0;
  */
 int64_t tsMsPerDay[] = {86400000L, 86400000000L};
 
-char  tsFirst[TSDB_FQDN_LEN] = {0};
+char  tsFirst[TSDB_EP_LEN] = {0};  
-char  tsSecond[TSDB_FQDN_LEN] = {0};
+char  tsSecond[TSDB_EP_LEN] = {0};
-char  tsArbitrator[TSDB_FQDN_LEN] = {0};
+char  tsArbitrator[TSDB_EP_LEN] = {0};
-char  tsLocalEp[TSDB_FQDN_LEN] = {0};  // Local End Point, hostname:port
+char  tsLocalEp[TSDB_EP_LEN] = {0};  // Local End Point, hostname:port
 uint16_t tsServerPort = 6030;
 uint16_t tsDnodeShellPort = 6030;  // udp[6035-6039] tcp[6035]
 uint16_t tsDnodeDnodePort = 6035;   // udp/tcp
@@ -284,7 +284,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.ptrLength = TSDB_EP_LEN;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -294,7 +294,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.ptrLength = TSDB_EP_LEN;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -356,7 +356,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.ptrLength = TSDB_EP_LEN;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -1252,7 +1252,7 @@ bool taosCheckGlobalCfg() {
   return true;
 }
 
-int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port) {
+int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
   *port = 0;
   strcpy(fqdn, ep);
 

src/common/src/ttypes.c

@@ -32,18 +32,280 @@ const int32_t TYPE_BYTES[11] = {
     sizeof(VarDataOffsetT)   // TSDB_DATA_TYPE_NCHAR
 };
 
+static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                          int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int8_t *data = (int8_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  //  int64_t lastKey = 0;
+  //  int8_t  lastVal = TSDB_DATA_TINYINT_NULL;
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((char *)&data[i], TSDB_DATA_TYPE_TINYINT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+  }
+}
+
+static void getStatics_i16(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int16_t *data = (int16_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  //  int64_t lastKey = 0;
+  //  int16_t lastVal = TSDB_DATA_SMALLINT_NULL;
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+    
+    //    if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) {
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    } else {
+    //      *wsum = lastVal * (primaryKey[i] - lastKey);
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    }
+  }
+}
+
+static void getStatics_i32(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int32_t *data = (int32_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  //  int64_t lastKey = 0;
+  //  int32_t lastVal = TSDB_DATA_INT_NULL;
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+    
+    //    if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) {
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    } else {
+    //      *wsum = lastVal * (primaryKey[i] - lastKey);
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    }
+  }
+}
+
+static void getStatics_i64(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int64_t *data = (int64_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+    
+    //    if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) {
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    } else {
+    //      *wsum = lastVal * (primaryKey[i] - lastKey);
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    }
+  }
+}
+
+static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                         int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  float *data = (float *)pData;
+  float fmin      = DBL_MAX;
+  float fmax      = -DBL_MAX;
+  double dsum     = 0;
+  *minIndex       = 0;
+  *maxIndex       = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    float fv = 0;
+    fv = GET_FLOAT_VAL(&(data[i]));
+    dsum += fv;
+    if (fmin > fv) {
+      fmin = fv;
+      *minIndex = i;
+    }
+    
+    if (fmax < fv) {
+      fmax = fv;
+      *maxIndex = i;
+    }
+    
+    //    if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) {
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    } else {
+    //      *wsum = lastVal * (primaryKey[i] - lastKey);
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    }
+  }
+  
+  double csum = 0;
+  csum = GET_DOUBLE_VAL(sum);
+  csum += dsum;
+#ifdef _TD_ARM_32_
+  SET_DOUBLE_VAL_ALIGN(sum, &csum);
+  SET_DOUBLE_VAL_ALIGN(max, &fmax);
+  SET_DOUBLE_VAL_ALIGN(min, &fmin);
+#else
+  *sum = csum;
+  *max = fmax;
+  *min = fmin;
+#endif
+}
+
+static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                         int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  double *data = (double *)pData;
+  double dmin      = DBL_MAX;
+  double dmax      = -DBL_MAX;
+  double dsum      = 0;
+  *minIndex        = 0;
+  *maxIndex        = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    double dv = 0;
+    dv = GET_DOUBLE_VAL(&(data[i]));
+    dsum += dv;
+    if (dmin > dv) {
+      dmin = dv;
+      *minIndex = i;
+    }
+    
+    if (dmax < dv) {
+      dmax = dv;
+      *maxIndex = i;
+    }
+    
+    //    if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) {
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    } else {
+    //      *wsum = lastVal * (primaryKey[i] - lastKey);
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    }
+  }
+  
+  double csum = 0;
+  csum = GET_DOUBLE_VAL(sum);
+  csum += dsum;
+
+
+#ifdef _TD_ARM_32_
+  SET_DOUBLE_VAL_ALIGN(sum, &csum);
+    SET_DOUBLE_VAL_ALIGN(max, &dmax);
+    SET_DOUBLE_VAL_ALIGN(min, &dmin);
+#else
+  *sum = csum;
+  *max = dmax;
+  *min = dmin;
+#endif
+}
+
 tDataTypeDescriptor tDataTypeDesc[11] = {
-  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL},
+  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL,                  NULL},
-  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool},
+  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool,      NULL},
-  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint},
+  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
-  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint},
+  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
-  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt},
+  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt,       getStatics_i32},
-  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT",    tsCompressBigint,    tsDecompressBigint},
+  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT",    tsCompressBigint,    tsDecompressBigint,    getStatics_i64},
-  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat},
+  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat,     getStatics_f},
-  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble},
+  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble,    getStatics_d},
-  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString},
+  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString,    NULL},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, NULL},
-  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString},
+  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString,    NULL},
 };
 
 char tTokenTypeSwitcher[13] = {

src/connector/nodejs/nodetaos/cinterface.js

@@ -241,17 +241,12 @@ function CTaosInterface (config = null, pass = false) {
     'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]],
 
     // Subscription
-    //TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, long time, int mseconds)
+    //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval)
-    ////TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, int64_t time, int mseconds);
+    'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ],
-    'taos_subscribe': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int64, ref.types.int] ],
+    // TAOS_RES *taos_consume(TAOS_SUB *tsub)
-    //TAOS_ROW taos_consume(TAOS_SUB *tsub);
+    'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ],
-    'taos_consume': [ ref.refType(ref.types.void_ptr2), [ref.types.void_ptr] ],
     //void taos_unsubscribe(TAOS_SUB *tsub);
     'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ],
-    //int taos_subfields_count(TAOS_SUB *tsub);
-    'taos_subfields_count': [ ref.types.int, [ref.types.void_ptr ] ],
-    //TAOS_FIELD *taos_fetch_subfields(TAOS_SUB *tsub);
-    'taos_fetch_subfields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ],
 
     // Continuous Query
     //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
@@ -362,7 +357,7 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
   blocks.fill(null);
   num_of_rows = Math.abs(num_of_rows);
   let offset = 0;
-  pblock = pblock.deref()
+  pblock = pblock.deref();
   for (let i = 0; i < fields.length; i++) {
 
     if (!convertFunctions[fields[i]['type']] ) {
@@ -472,64 +467,40 @@ CTaosInterface.prototype.getClientInfo = function getClientInfo() {
 }
 
 // Subscription
-CTaosInterface.prototype.subscribe = function subscribe(host=null, user="root", password="taosdata", db=null, table=null, time=null, mseconds=null) {
+CTaosInterface.prototype.subscribe = function subscribe(connection, restart, topic, sql, interval) {
-  let dbOrig = db;
+  let topicOrig = topic;
-  let tableOrig = table;
+  let sqlOrig = sql;
-  try  {
+  try {
-    host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL);
+    sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL);
   }
   catch(err) {
-    throw "Attribute Error: host is expected as a str";
+    throw "Attribute Error: sql is expected as a str";
   }
   try {
-    user = ref.allocCString(user)
+    topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL);
   }
   catch(err) {
-    throw "Attribute Error: user is expected as a str";
+    throw TypeError("topic is expected as a str");
   }
-  try {
+
-    password = ref.allocCString(password);
+  restart = ref.alloc(ref.types.int, restart);
-  }
+
-  catch(err) {
+  let subscription = this.libtaos.taos_subscribe(connection, restart, topic, sql, null, null, interval);
-    throw "Attribute Error: password is expected as a str";
-  }
-  try {
-    db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL);
-  }
-  catch(err) {
-    throw "Attribute Error: db is expected as a str";
-  }
-  try {
-    table = table != null ? ref.allocCString(table) : ref.alloc(ref.types.char_ptr, ref.NULL);
-  }
-  catch(err) {
-    throw TypeError("table is expected as a str");
-  }
-  try {
-    mseconds = ref.alloc(ref.types.int, mseconds);
-  }
-  catch(err) {
-    throw TypeError("mseconds is expected as an int");
-  }
-  //TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, int64_t time, int mseconds);
-  let subscription = this.libtaos.taos_subscribe(host, user, password, db, table, time, mseconds);
   if (ref.isNull(subscription)) {
     throw new errors.TDError('Failed to subscribe to TDengine | Database: ' + dbOrig + ', Table: ' + tableOrig);
   }
   else {
-    console.log('Successfully subscribed to TDengine | Database: ' + dbOrig + ', Table: ' + tableOrig);
+    console.log('Successfully subscribed to TDengine - Topic: ' + topicOrig);
   }
   return subscription;
 }
-CTaosInterface.prototype.subFieldsCount = function subFieldsCount(subscription) {
+
-  return this.libtaos.taos_subfields_count(subscription);
+CTaosInterface.prototype.consume = function consume(subscription) {
-}
+  let result = this.libtaos.taos_consume(subscription);
-CTaosInterface.prototype.fetchSubFields = function fetchSubFields(subscription) {
-  let pfields = this.libtaos.taos_fetch_subfields(subscription);
-  let pfieldscount = this.subFieldsCount(subscription);
   let fields = [];
+  let pfields = this.fetchFields(result);
   if (ref.isNull(pfields) == false) {
-    pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0);
+    pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0);
     for (let i = 0; i < pfields.length; i += 68) {
       //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type
       fields.push( {
@@ -539,27 +510,23 @@ CTaosInterface.prototype.fetchSubFields = function fetchSubFields(subscription)
       })
     }
   }
-  return fields;
+
-}
+  let data = [];
-CTaosInterface.prototype.consume = function consume(subscription) {
+  while(true) {
-  let row = this.libtaos.taos_consume(subscription);
+    let { blocks, num_of_rows } = this.fetchBlock(result, fields);
-  let fields = this.fetchSubFields(subscription);
+    if (num_of_rows == 0) {
-  //let isMicro = (cti.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO);
+      break;
-  let isMicro = false; //no supported function for determining precision?
+    }
-  let blocks = new Array(fields.length);
+    for (let i = 0; i < num_of_rows; i++) {
-  blocks.fill(null);
+      data.push([]);
-  let numOfRows2 = 1; //Math.abs(numOfRows2);
+      let rowBlock = new Array(fields.length);
-  let offset = 0;
+      for (let j = 0; j < fields.length; j++) {
-  if (numOfRows2 > 0){
+        rowBlock[j] = blocks[j][i];
-    for (let i = 0; i < fields.length; i++) {
-      if (!convertFunctions[fields[i]['type']] ) {
-        throw new errors.DatabaseError("Invalid data type returned from database");
       }
-      blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro);
+      data[data.length-1] = (rowBlock);
-      offset += fields[i]['bytes'] * numOfRows2;
     }
   }
-  return {blocks:blocks, fields:fields};
+  return { data: data, fields: fields, result: result };
 }
 CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) {
   //void taos_unsubscribe(TAOS_SUB *tsub);

src/connector/nodejs/nodetaos/cursor.js

@@ -405,18 +405,16 @@ TDengineCursor.prototype.getClientInfo = function getClientInfo() {
 /**
  * Subscribe to a table from a database in TDengine.
  * @param {Object} config - A configuration object containing the configuration options for the subscription
- * @param {string} config.host - The host to subscribe to
+ * @param {string} config.restart - whether or not to continue a subscription if it already exits, otherwise start from beginning
- * @param {string} config.user - The user to subscribe as
+ * @param {string} config.topic - The unique identifier of a subscription
- * @param {string} config.password - The password for the said user
+ * @param {string} config.sql - A sql statement for data query
- * @param {string} config.db - The db containing the table to subscribe to
+ * @param {string} config.interval - The pulling interval
- * @param {string} config.table - The name of the table to subscribe to
- * @param {number} config.time - The start time to start a subscription session
- * @param {number} config.mseconds - The pulling period of the subscription session
  * @return {Buffer} A buffer pointing to the subscription session handle
  * @since 1.3.0
  */
 TDengineCursor.prototype.subscribe = function subscribe(config) {
-  return this._chandle.subscribe(config.host, config.user, config.password, config.db, config.table, config.time, config.mseconds);
+  let restart = config.restart ? 1 : 0;
+  return this._chandle.subscribe(this._connection._conn, restart, config.topic, config.sql, config.interval);
 };
 /**
  * An infinite loop that consumes the latest data and calls a callback function that is provided.
@@ -426,18 +424,8 @@ TDengineCursor.prototype.subscribe = function subscribe(config) {
  */
 TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) {
   while (true) {
-    let res = this._chandle.consume(subscription);
+    let { data, fields, result} = this._chandle.consume(subscription);
-    let data = [];
+    callback(data, fields, result);
-    let num_of_rows = res.blocks[0].length;
-    for (let j = 0; j < num_of_rows; j++) {
-      data.push([]);
-      let rowBlock = new Array(res.fields.length);
-      for (let k = 0; k < res.fields.length; k++) {
-        rowBlock[k] = res.blocks[k][j];
-      }
-      data[data.length-1] = rowBlock;
-    }
-    callback(data, res.fields, subscription);
   }
 }
 /**

src/connector/nodejs/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "td-connector",
-  "version": "1.5.0",
+  "version": "1.6.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

src/connector/nodejs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "td-connector",
-  "version": "1.5.0",
+  "version": "1.6.1",
   "description": "A Node.js connector for TDengine.",
   "main": "tdengine.js",
   "scripts": {

src/connector/nodejs/test/test.js

@@ -33,12 +33,12 @@ for (let i = 0; i < 10000; i++) {
                     parseInt( R(-Math.pow(2,31) + 1 , Math.pow(2,31) - 1) ), // Int
                     parseInt( R(-Math.pow(2,31) + 1 , Math.pow(2,31) - 1) ), // BigInt
                     parseFloat( R(-3.4E38, 3.4E38) ), // Float
-                    parseFloat( R(-1.7E308, 1.7E308) ), // Double
+                    parseFloat( R(-1.7E30, 1.7E30) ), // Double
                     "\"Long Binary\"", // Binary
                     parseInt( R(-32767, 32767) ), // Small Int
                     parseInt( R(-127, 127) ), // Tiny Int
                     randomBool(),
-                    "\"Nchars 一些中文字幕\""]; // Bool
+                    "\"Nchars\""]; // Bool
   c1.execute('insert into td_connector_test.all_types values(' + insertData.join(',') + ' );', {quiet:true});
   if (i % 1000 == 0) {
     console.log("Insert # " , i);

src/connector/nodejs/test/testSubscribe.js

@@ -0,0 +1,16 @@
+const taos = require('../tdengine');
+var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:10});
+var c1 = conn.cursor();
+let stime = new Date();
+let interval = 1000;
+c1.execute('use td_connector_test');
+let sub = c1.subscribe({
+  restart: true,
+  sql: "select AVG(_int) from td_connector_test.all_Types;",
+  topic: 'all_Types',
+  interval: 1000
+});
+
+c1.consumeData(sub, (data, fields) => {
+  console.log(data);
+});

src/connector/python/linux/python3/taos/cursor.py

@@ -1,6 +1,8 @@
 from .cinterface import CTaosInterface
 from .error import *
 
+# querySeqNum = 0
+
 class TDengineCursor(object):
     """Database cursor which is used to manage the context of a fetch operation.
 
@@ -109,7 +111,14 @@ class TDengineCursor(object):
         if params is not None:
             pass
         
+
+        # global querySeqNum
+        # querySeqNum += 1
+        # localSeqNum = querySeqNum # avoid raice condition
+        # print("   >> Exec Query ({}): {}".format(localSeqNum, str(stmt)))
         res = CTaosInterface.query(self._connection._conn, stmt)
+        # print("   << Query ({}) Exec Done".format(localSeqNum))
+
         if res == 0:
             if CTaosInterface.fieldsCount(self._connection._conn) == 0:
                 self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)

src/dnode/inc/dnodeMain.h

@@ -13,18 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _rpc_hash_ip_header_
+#ifndef TDENGINE_DNODE_MAIN_H
-#define _rpc_hash_ip_header_
+#define TDENGINE_DNODE_MAIN_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-void *rpcOpenIpHash(int maxSessions);
+int32_t dnodeInitSystem();
-void  rpcCloseIpHash(void *handle);
+void dnodeCleanUpSystem();
-void *rpcAddIpHash(void *handle, void *pData, uint32_t ip, uint16_t port);
-void  rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port);
-void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port);
 
 #ifdef __cplusplus
 }

src/dnode/src/dnodeMain.c

@@ -16,8 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taos.h"
-#include "tglobal.h"
-#include "trpc.h"
 #include "tutil.h"
 #include "tconfig.h"
 #include "tglobal.h"
@@ -29,112 +27,14 @@
 #include "dnodeVRead.h"
 #include "dnodeShell.h"
 #include "dnodeVWrite.h"
-#include "tgrant.h"
 
-static int32_t dnodeInitSystem();
 static int32_t dnodeInitStorage();
-extern void grantParseParameter();
 static void dnodeCleanupStorage();
-static void dnodeCleanUpSystem();
 static void dnodeSetRunStatus(SDnodeRunStatus status);
-static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
 static void dnodeCheckDataDirOpenned(char *dir);
 static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
 
-int32_t main(int32_t argc, char *argv[]) {
+int32_t dnodeInitSystem() {
-  // Set global configuration file
-  for (int32_t i = 1; i < argc; ++i) {
-    if (strcmp(argv[i], "-c") == 0) {
-      if (i < argc - 1) {
-        strcpy(configDir, argv[++i]);
-      } else {
-        printf("'-c' requires a parameter, default:%s\n", configDir);
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-V") == 0) {
-#ifdef _SYNC      
-      char *versionStr = "enterprise";
-#else      
-      char *versionStr = "community";
-#endif      
-      printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version);
-      printf("gitinfo: %s\n", gitinfo);
-      printf("gitinfoI: %s\n", gitinfoOfInternal);
-      printf("buildinfo: %s\n", buildinfo);
-      exit(EXIT_SUCCESS);
-    } else if (strcmp(argv[i], "-k") == 0) {
-      grantParseParameter();
-      exit(EXIT_SUCCESS);
-    }
-#ifdef TAOS_MEM_CHECK
-    else if (strcmp(argv[i], "--alloc-random-fail") == 0) {
-      if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
-        taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true);
-      } else {
-        taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true);
-      }
-    } else if (strcmp(argv[i], "--detect-mem-leak") == 0) {
-      if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
-        taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true);
-      } else {
-        taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true);
-      }
-    }
-#endif
-  }
-
-  /* Set termination handler. */
-  struct sigaction act = {0};
-  act.sa_flags = SA_SIGINFO;
-  act.sa_sigaction = signal_handler;
-  sigaction(SIGTERM, &act, NULL);
-  sigaction(SIGHUP, &act, NULL);
-  sigaction(SIGINT, &act, NULL);
-  sigaction(SIGUSR1, &act, NULL);
-  sigaction(SIGUSR2, &act, NULL);
-
-  // Open /var/log/syslog file to record information.
-  openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1);
-  syslog(LOG_INFO, "Starting TDengine service...");
-
-  // Initialize the system
-  if (dnodeInitSystem() < 0) {
-    syslog(LOG_ERR, "Error initialize TDengine system");
-    closelog();
-
-    dnodeCleanUpSystem();
-    exit(EXIT_FAILURE);
-  }
-
-  syslog(LOG_INFO, "Started TDengine service successfully.");
-
-  while (1) {
-    sleep(1000);
-  }
-}
-
-static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) {
-  if (signum == SIGUSR1) {
-    taosCfgDynamicOptions("debugFlag 135");
-    return;
-  }
-  if (signum == SIGUSR2) {
-    taosCfgDynamicOptions("resetlog");
-    return;
-  }
-  syslog(LOG_INFO, "Shut down signal is %d", signum);
-  syslog(LOG_INFO, "Shutting down TDengine service...");
-  // clean the system.
-  dPrint("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid);
-  dnodeCleanUpSystem();
-  // close the syslog
-  syslog(LOG_INFO, "Shut down TDengine service successfully");
-  dPrint("TDengine is shut down!");
-  closelog();
-  exit(EXIT_SUCCESS);
-}
-
-static int32_t dnodeInitSystem() {
   dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
   tscEmbedded  = 1;
   taosResolveCRC();
@@ -180,7 +80,7 @@ static int32_t dnodeInitSystem() {
   return 0;
 }
 
-static void dnodeCleanUpSystem() {
+void dnodeCleanUpSystem() {
   if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
     dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
     dnodeCleanupShell();

src/dnode/src/dnodeMgmt.c

@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "ihash.h"
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "ttime.h"
@@ -412,7 +411,7 @@ static bool dnodeReadMnodeInfos() {
       dError("failed to read mnode mgmtIpList.json, nodeName not found");
       goto PARSE_OVER;
     }
-    strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_FQDN_LEN);
+    strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
  }
 
   ret = true;

src/dnode/src/dnodeModule.c

@@ -33,7 +33,7 @@ typedef struct {
   void    (*stopFp)();
 } SModule;
 
-static SModule  tsModule[TSDB_MOD_MAX] = {0};
+static SModule  tsModule[TSDB_MOD_MAX] = {{0}};
 static uint32_t tsModuleStatus = 0;
 
 static void dnodeSetModuleStatus(int32_t module) {

src/dnode/src/dnodeShell.c

@@ -157,8 +157,8 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
   if (rpcRsp.code != 0) {
     dError("user:%s, auth msg received from mnode, error:%s", user, tstrerror(rpcRsp.code));
   } else {
-    dTrace("user:%s, auth msg received from mnode", user);
     SDMAuthRsp *pRsp = rpcRsp.pCont;
+    dTrace("user:%s, auth msg received from mnode", user);
     memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
     memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
     *spi = pRsp->spi;

src/dnode/src/dnodeSystem.c

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tgrant.h"
+#include "tutil.h"
+#include "tglobal.h"
+#include "dnodeInt.h"
+#include "dnodeMain.h"
+
+static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
+
+int32_t main(int32_t argc, char *argv[]) {
+  // Set global configuration file
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        strcpy(configDir, argv[++i]);
+      } else {
+        printf("'-c' requires a parameter, default:%s\n", configDir);
+        exit(EXIT_FAILURE);
+      }
+    } else if (strcmp(argv[i], "-V") == 0) {
+#ifdef _SYNC      
+      char *versionStr = "enterprise";
+#else      
+      char *versionStr = "community";
+#endif      
+      printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version);
+      printf("gitinfo: %s\n", gitinfo);
+      printf("gitinfoI: %s\n", gitinfoOfInternal);
+      printf("buildinfo: %s\n", buildinfo);
+      exit(EXIT_SUCCESS);
+    } else if (strcmp(argv[i], "-k") == 0) {
+      grantParseParameter();
+      exit(EXIT_SUCCESS);
+    }
+#ifdef TAOS_MEM_CHECK
+    else if (strcmp(argv[i], "--alloc-random-fail") == 0) {
+      if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
+        taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true);
+      } else {
+        taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true);
+      }
+    } else if (strcmp(argv[i], "--detect-mem-leak") == 0) {
+      if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
+        taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true);
+      } else {
+        taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true);
+      }
+    }
+#endif
+  }
+
+  /* Set termination handler. */
+  struct sigaction act = {{0}};
+  act.sa_flags = SA_SIGINFO;
+  act.sa_sigaction = signal_handler;
+  sigaction(SIGTERM, &act, NULL);
+  sigaction(SIGHUP, &act, NULL);
+  sigaction(SIGINT, &act, NULL);
+  sigaction(SIGUSR1, &act, NULL);
+  sigaction(SIGUSR2, &act, NULL);
+
+  // Open /var/log/syslog file to record information.
+  openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1);
+  syslog(LOG_INFO, "Starting TDengine service...");
+
+  // Initialize the system
+  if (dnodeInitSystem() < 0) {
+    syslog(LOG_ERR, "Error initialize TDengine system");
+    closelog();
+
+    dnodeCleanUpSystem();
+    exit(EXIT_FAILURE);
+  }
+
+  syslog(LOG_INFO, "Started TDengine service successfully.");
+
+  while (1) {
+    sleep(1000);
+  }
+}
+
+static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) {
+  if (signum == SIGUSR1) {
+    taosCfgDynamicOptions("debugFlag 135");
+    return;
+  }
+  if (signum == SIGUSR2) {
+    taosCfgDynamicOptions("resetlog");
+    return;
+  }
+  syslog(LOG_INFO, "Shut down signal is %d", signum);
+  syslog(LOG_INFO, "Shutting down TDengine service...");
+  // clean the system.
+  dPrint("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid);
+  dnodeCleanUpSystem();
+  // close the syslog
+  syslog(LOG_INFO, "Shut down TDengine service successfully");
+  dPrint("TDengine is shut down!");
+  closelog();
+  exit(EXIT_SUCCESS);
+}

src/dnode/src/dnodeVRead.c

@@ -92,8 +92,6 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
   char        *pCont       = (char *) pMsg->pCont;
   void        *pVnode;  
 
-  dTrace("dnode %s msg incoming, thandle:%p", taosMsg[pMsg->msgType], pMsg->handle);
-
   while (leftLen > 0) {
     SMsgHead *pHead = (SMsgHead *) pCont;
     pHead->vgId    = htonl(pHead->vgId);
@@ -214,6 +212,7 @@ static void *dnodeProcessReadQueue(void *param) {
       continue;
     }
 
+    dTrace("%p, msg:%s will be processed", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);
     int32_t code = vnodeProcessRead(pVnode, pReadMsg->rpcMsg.msgType, pReadMsg->pCont, pReadMsg->contLen, &pReadMsg->rspRet);
     dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
     taosFreeQitem(pReadMsg);

src/dnode/src/dnodeVWrite.c

@@ -200,6 +200,7 @@ static void *dnodeProcessWriteQueue(void *param) {
         pHead->msgType = pWrite->rpcMsg.msgType;
         pHead->version = 0;
         pHead->len = pWrite->contLen;
+        dTrace("%p, msg:%s will be processed", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]);
       } else {
         pHead = (SWalHead *)item;
       }

src/inc/taosdef.h

@@ -147,6 +147,8 @@ typedef struct tDataTypeDescriptor {
                   char algorithm, char *const buffer, int bufferSize);
   int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output,
                     int outputSize, char algorithm, char *const buffer, int bufferSize);
+  void (*getStatisFunc)(const TSKEY *primaryKey, const void *pData, int32_t numofrow, int64_t *min, int64_t *max,
+                         int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 
 extern tDataTypeDescriptor tDataTypeDesc[11];
@@ -191,20 +193,20 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_ACCT_LEN             TSDB_UNI_LEN
 #define TSDB_PASSWORD_LEN         TSDB_UNI_LEN
 
-#define TSDB_MAX_COLUMNS          256
+#define TSDB_MAX_COLUMNS          1024
 #define TSDB_MIN_COLUMNS          2       //PRIMARY COLUMN(timestamp) + other columns
 
 #define TSDB_NODE_NAME_LEN        64
 #define TSDB_TABLE_NAME_LEN       192
 #define TSDB_DB_NAME_LEN          32
 #define TSDB_COL_NAME_LEN         64
-#define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 16
+#define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
 #define TSDB_MAX_ALLOWED_SQL_LEN  (8*1024*1024U)          // sql length should be less than 6mb
 
-#define TSDB_MAX_BYTES_PER_ROW    TSDB_MAX_COLUMNS * 16
+#define TSDB_MAX_BYTES_PER_ROW    TSDB_MAX_COLUMNS * 64
-#define TSDB_MAX_TAGS_LEN         512
+#define TSDB_MAX_TAGS_LEN         65536
-#define TSDB_MAX_TAGS             32
+#define TSDB_MAX_TAGS             128
 
 #define TSDB_AUTH_LEN             16
 #define TSDB_KEY_LEN              16
@@ -216,7 +218,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_LOCALE_LEN           64
 #define TSDB_TIMEZONE_LEN         64
 
-#define TSDB_FQDN_LEN             72
+#define TSDB_FQDN_LEN             128
+#define TSDB_EP_LEN               (TSDB_FQDN_LEN+6)
 #define TSDB_IPv4ADDR_LEN      	  16
 #define TSDB_FILENAME_LEN         128
 #define TSDB_METER_VNODE_BITS     20
@@ -232,9 +235,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_DEFAULT_PKT_SIZE     65480  //same as RPC_MAX_UDP_SIZE
 
 #define TSDB_PAYLOAD_SIZE         (TSDB_DEFAULT_PKT_SIZE - 100)
-#define TSDB_DEFAULT_PAYLOAD_SIZE 1024   // default payload size
+#define TSDB_DEFAULT_PAYLOAD_SIZE 2048   // default payload size
 #define TSDB_EXTRA_PAYLOAD_SIZE   128    // extra bytes for auth
-#define TSDB_SQLCMD_SIZE          1024
+#define TSDB_CQ_SQL_SIZE          1024
 #define TSDB_MAX_VNODES           256
 #define TSDB_MIN_VNODES           50
 #define TSDB_INVALID_VNODE_NUM    0

src/inc/taoserror.h

@@ -170,6 +170,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE,            0, 459, "invalid handle"
 TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED,            0, 460, "query cancelled")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE,                 0, 461, "invalid ie")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE,              0, 462, "invalid value")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FQDN,               0, 463, "invalid FQDN")
 
 // others
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT,        0, 500, "invalid file format")

src/inc/taosmsg.h

@@ -187,13 +187,13 @@ typedef struct SMsgHead {
 
 // Submit message for one table
 typedef struct SSubmitBlk {
-  int64_t uid;        // table unique id
+  uint64_t uid;        // table unique id
-  int32_t tid;        // table id
+  int32_t  tid;        // table id
-  int32_t padding;    // TODO just for padding here
+  int32_t  padding;    // TODO just for padding here
-  int32_t sversion;   // data schema version
+  int32_t  sversion;   // data schema version
-  int32_t len;        // data part length, not including the SSubmitBlk head
+  int32_t  len;        // data part length, not including the SSubmitBlk head
-  int16_t numOfRows;  // total number of rows in current submit block
+  int16_t  numOfRows;  // total number of rows in current submit block
-  char    data[];
+  char     data[];
 } SSubmitBlk;
 
 // Submit message for this TSDB
@@ -236,6 +236,7 @@ typedef struct {
   int16_t  numOfTags;
   int32_t  sid;
   int32_t  sversion;
+  int32_t  tversion;
   int32_t  tagDataLen;
   int32_t  sqlDataLen;
   uint64_t uid;
@@ -327,9 +328,9 @@ typedef struct {
 } SMDDropTableMsg;
 
 typedef struct {
-  int32_t contLen;
+  int32_t  contLen;
-  int32_t vgId;
+  int32_t  vgId;
-  int64_t uid;
+  uint64_t uid;
   char    tableId[TSDB_TABLE_ID_LEN + 1];
 } SMDDropSTableMsg;
 
@@ -404,9 +405,9 @@ typedef struct SColumnInfo {
 } SColumnInfo;
 
 typedef struct STableIdInfo {
-  int64_t uid;
+  uint64_t uid;
-  int32_t tid;
+  int32_t  tid;
-  TSKEY   key;  // last accessed ts, for subscription
+  TSKEY    key;  // last accessed ts, for subscription
 } STableIdInfo;
 
 typedef struct STimeWindow {
@@ -529,7 +530,7 @@ typedef struct {
 
 typedef struct {
   int32_t   nodeId;
-  char      nodeEp[TSDB_FQDN_LEN];
+  char      nodeEp[TSDB_EP_LEN];
 } SDMMnodeInfo;
 
 typedef struct {
@@ -541,7 +542,7 @@ typedef struct {
 typedef struct {
   uint32_t   version;
   int32_t    dnodeId;
-  char       dnodeEp[TSDB_FQDN_LEN];
+  char       dnodeEp[TSDB_EP_LEN];
   uint32_t   moduleStatus;
   uint32_t   lastReboot;        // time stamp for last reboot
   uint16_t   numOfTotalVnodes;  // from config file
@@ -583,7 +584,7 @@ typedef struct {
 
 typedef struct {
   int32_t  nodeId;
-  char     nodeEp[TSDB_FQDN_LEN];
+  char     nodeEp[TSDB_EP_LEN];
 } SMDVnodeDesc;
 
 typedef struct {
@@ -632,6 +633,7 @@ typedef struct STableMetaMsg {
   uint8_t       tableType;
   int16_t       numOfColumns;
   int16_t       sversion;
+  int16_t       tversion;
   int32_t       sid;
   uint64_t      uid;
   SCMVgroupInfo vgroup;
@@ -667,7 +669,7 @@ typedef struct SCMShowRsp {
 } SCMShowRsp;
 
 typedef struct {
-  char     ep[TSDB_FQDN_LEN];  // end point, hostname:port
+  char     ep[TSDB_EP_LEN];  // end point, hostname:port
 } SCMCreateDnodeMsg, SCMDropDnodeMsg;
 
 typedef struct {
@@ -682,7 +684,7 @@ typedef struct {
 } SDMConfigVnodeMsg;
 
 typedef struct {
-  char ep[TSDB_FQDN_LEN];  // end point, hostname:port
+  char ep[TSDB_EP_LEN];  // end point, hostname:port
   char config[64];
 } SMDCfgDnodeMsg, SCMCfgDnodeMsg;
 

src/inc/trpc.h

@@ -48,6 +48,7 @@ typedef struct {
   int     contLen;
   int32_t code;
   void   *handle;
+  void   *ahandle;  //app handle set by client, for debug purpose
 } SRpcMsg;
 
 typedef struct {

src/inc/tsdb.h

@@ -72,13 +72,13 @@ typedef void TsdbRepoT;  // use void to hide implementation details from outside
 int        tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
 int32_t    tsdbDropRepo(TsdbRepoT *repo);
 TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH);
-int32_t    tsdbCloseRepo(TsdbRepoT *repo);
+int32_t    tsdbCloseRepo(TsdbRepoT *repo, int toCommit);
 int32_t    tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg);
 
 // --------- TSDB TABLE DEFINITION
 typedef struct {
-  int64_t uid;  // the unique table ID
+  uint64_t uid;  // the unique table ID
-  int32_t tid;  // the table ID in the repository.
+  int32_t  tid;  // the table ID in the repository.
 } STableId;
 
 // --------- TSDB TABLE configuration
@@ -88,14 +88,14 @@ typedef struct {
   STableId   tableId;
   int32_t    sversion;
   char *     sname;  // super table name
-  int64_t    superUid;
+  uint64_t   superUid;
   STSchema * schema;
   STSchema * tagSchema;
   SDataRow   tagValues;
 } STableCfg;
 
-int  tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t tid);
+int  tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid);
-int  tsdbTableSetSuperUid(STableCfg *config, int64_t uid);
+int  tsdbTableSetSuperUid(STableCfg *config, uint64_t uid);
 int  tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup);
 int  tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup);
 int  tsdbTableSetTagValue(STableCfg *config, SDataRow row, bool dup);
@@ -109,7 +109,7 @@ char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes);
 int   tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
 int   tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
 int   tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
-TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid);
+TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid);
 
 uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size);
 
@@ -140,7 +140,7 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid);
  *
  * @return the number of points inserted, -1 for failure and the error number is set
  */
-int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg);
+int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ;
 
 // -- FOR QUERY TIME SERIES DATA
 
@@ -272,9 +272,9 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
  * @param stableid. super table sid
  * @param pTagCond. tag query condition
  */
-int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, uint64_t uid, const char *pTagCond, size_t len,
-  int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols);
+                                 int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
-
+                                 SColIndex *pColIndex, int32_t numOfCols);
 
 /**
  * create the table group result including only one table, used to handle the normal table query
@@ -284,7 +284,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC
  * @param pGroupInfo  the generated result
  * @return
  */
-int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo);
+int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo);
 
 /**
  * clean up the query handle

src/inc/tsync.h

@@ -57,7 +57,7 @@ typedef struct {
 // if name is empty(name[0] is zero), get the file from index or after, used by master
 // if name is provided(name[0] is not zero), get the named file at the specified index, used by unsynced node
 // it returns the file magic number and size, if file not there, magic shall be 0.
-typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); 
+typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion); 
 
 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
@@ -73,7 +73,7 @@ typedef void     (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
 typedef void     (*FNotifyRole)(void *ahandle, int8_t role);
 
 // when data file is synced successfully, notity app
-typedef void     (*FNotifyFileSynced)(void *ahandle);
+typedef void     (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
 
 typedef struct {
   int32_t    vgId;      // vgroup ID

src/kit/shell/inc/shell.h

@@ -41,14 +41,13 @@
 // dynamic config timestamp width according to maximum time precision
 extern int32_t TIMESTAMP_OUTPUT_LENGTH;
 
-typedef struct History History;
+typedef struct SShellHistory {
-struct History {
   char* hist[MAX_HISTORY_SIZE];
   int   hstart;
   int   hend;
-};
+} SShellHistory;
 
-struct arguments {
+typedef struct SShellArguments {
   char* host;
   char* password;
   char* user;
@@ -62,11 +61,11 @@ struct arguments {
   char* commands;
   int   abort;
   int   port;
-};
+} SShellArguments;
 
 /**************** Function declarations ****************/
-extern void shellParseArgument(int argc, char* argv[], struct arguments* arguments);
+extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
-extern TAOS* shellInit(struct arguments* args);
+extern TAOS* shellInit(SShellArguments* args);
 extern void* shellLoopQuery(void* arg);
 extern void taos_error(TAOS* con);
 extern int regex_match(const char* s, const char* reg, int cflags);
@@ -76,7 +75,7 @@ void shellRunCommandOnServer(TAOS* con, char command[]);
 void read_history();
 void write_history();
 void source_file(TAOS* con, char* fptr);
-void source_dir(TAOS* con, struct arguments* args);
+void source_dir(TAOS* con, SShellArguments* args);
 void get_history_path(char* history);
 void cleanup_handler(void* arg);
 void exitShell();
@@ -89,12 +88,12 @@ int isCommentLine(char *line);
 extern char           PROMPT_HEADER[];
 extern char           CONTINUE_PROMPT[];
 extern int            prompt_size;
-extern History        history;
+extern SShellHistory  history;
 extern struct termios oldtio;
 extern void           set_terminal_mode();
 extern int get_old_terminal_mode(struct termios* tio);
-extern void             reset_terminal_mode();
+extern void            reset_terminal_mode();
-extern struct arguments args;
+extern SShellArguments args;
-extern TAOS_RES*        result;
+extern TAOS_RES*       result;
 
 #endif

src/kit/shell/src/shellEngine.c

@@ -33,12 +33,12 @@ char      PROMPT_HEADER[] = "taos> ";
 char      CONTINUE_PROMPT[] = "   -> ";
 int       prompt_size = 6;
 TAOS_RES *result = NULL;
-History   history;
+SShellHistory   history;
 
 /*
  * FUNCTION: Initialize the shell.
  */
-TAOS *shellInit(struct arguments *args) {
+TAOS *shellInit(SShellArguments *args) {
   printf("\n");
   printf(CLIENT_VERSION, tsOsName, taos_get_client_info());
   fflush(stdout);

src/kit/shell/src/shellImport.c

@@ -221,7 +221,7 @@ void* shellImportThreadFp(void *arg)
   return NULL;
 }
 
-static void shellRunImportThreads(struct arguments* args)
+static void shellRunImportThreads(SShellArguments* args)
 {
   pthread_attr_t thattr;
   ShellThreadObj *threadObj = (ShellThreadObj *)calloc(args->threadNum, sizeof(ShellThreadObj));
@@ -254,7 +254,7 @@ static void shellRunImportThreads(struct arguments* args)
   free(threadObj);
 }
 
-void source_dir(TAOS* con, struct arguments* args) {
+void source_dir(TAOS* con, SShellArguments* args) {
   shellGetDirectoryFileList(args->dir);
   int64_t start = taosGetTimestampMs();
 

src/kit/shell/src/shellLinux.c

@@ -50,7 +50,7 @@ static struct argp_option options[] = {
 static error_t parse_opt(int key, char *arg, struct argp_state *state) {
   /* Get the input argument from argp_parse, which we
   know is a pointer to our arguments structure. */
-  struct arguments *arguments = state->input;
+  SShellArguments *arguments = state->input;
   wordexp_t full_path;
 
   switch (key) {
@@ -129,7 +129,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 /* Our argp parser. */
 static struct argp argp = {options, parse_opt, args_doc, doc};
 
-void shellParseArgument(int argc, char *argv[], struct arguments *arguments) {
+void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
   static char verType[32] = {0};
   sprintf(verType, "version: %s\n", version);
 

src/kit/shell/src/shellMain.c

@@ -62,7 +62,7 @@ int checkVersion() {
 }
 
 // Global configurations
-struct arguments args = {
+SShellArguments args = {
   .host = NULL,
   .password = NULL,
   .user = NULL,

src/kit/taosdemo/taosdemo.c

@@ -67,7 +67,7 @@ static struct argp_option options[] = {
   {0}};
 
 /* Used by main to communicate with parse_opt. */
-struct arguments {
+typedef struct DemoArguments {
   char  *host;
   uint16_t    port;
   char  *user;
@@ -87,13 +87,13 @@ struct arguments {
   int    num_of_DPT;
   int    abort;
   char **arg_list;
-};
+} SDemoArguments;
 
 /* Parse a single option. */
 static error_t parse_opt(int key, char *arg, struct argp_state *state) {
   /* Get the input argument from argp_parse, which we
      know is a pointer to our arguments structure. */
-  struct arguments *arguments = state->input;
+  SDemoArguments *arguments = state->input;
   wordexp_t full_path;
   char **sptr;
   switch (key) {
@@ -269,7 +269,7 @@ double getCurrentTime();
 void callBack(void *param, TAOS_RES *res, int code);
 
 int main(int argc, char *argv[]) {
-  struct arguments arguments = {NULL,            // host
+  SDemoArguments arguments = {NULL,            // host
                                 0,               // port
                                 "root",          // user
                                 "taosdata",      // password

src/kit/taosdump/taosdump.c

@@ -168,7 +168,7 @@ static struct argp_option options[] = {
   {0}};
 
 /* Used by main to communicate with parse_opt. */
-struct arguments {
+typedef struct SDumpArguments {
   // connection option
   char *host;
   char *user;
@@ -193,13 +193,13 @@ struct arguments {
   char **arg_list;
   int arg_list_len;
   bool isDumpIn;
-};
+} SDumpArguments;
 
 /* Parse a single option. */
 static error_t parse_opt(int key, char *arg, struct argp_state *state) {
   /* Get the input argument from argp_parse, which we
      know is a pointer to our arguments structure. */
-  struct arguments *arguments = state->input;
+  SDumpArguments *arguments = state->input;
   wordexp_t full_path;
 
   switch (key) {
@@ -296,31 +296,31 @@ char *command = NULL;
 char *lcommand = NULL;
 char *buffer = NULL;
 
-int taosDumpOut(struct arguments *arguments);
+int taosDumpOut(SDumpArguments *arguments);
 
-int taosDumpIn(struct arguments *arguments);
+int taosDumpIn(SDumpArguments *arguments);
 
 void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp);
 
-int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp);
+int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp);
 
-void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, struct arguments *arguments, FILE *fp);
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp);
 
-void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, struct arguments *arguments,
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, SDumpArguments *arguments,
                                    FILE *fp);
 
-int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp);
+int32_t taosDumpTable(char *table, char *metric, SDumpArguments *arguments, FILE *fp);
 
-int32_t taosDumpMetric(char *metric, struct arguments *arguments, FILE *fp);
+int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp);
 
-int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments);
+int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments);
 
-int taosCheckParam(struct arguments *arguments);
+int taosCheckParam(SDumpArguments *arguments);
 
 void taosFreeDbInfos();
 
 int main(int argc, char *argv[]) {
-  struct arguments arguments = {
+  SDumpArguments arguments = {
     // connection option
     NULL, "root", "taosdata", 0,
     // output file
@@ -424,7 +424,7 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) {
   return -1;
 }
 
-int taosDumpOut(struct arguments *arguments) {
+int taosDumpOut(SDumpArguments *arguments) {
   TAOS_ROW row;
   char *temp = NULL;
   FILE *fp = NULL;
@@ -602,7 +602,7 @@ void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) {
   fprintf(fp, "%s\n\n", buffer);
 }
 
-int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp) {
+int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
   TAOS_ROW row;
   int fd = -1;
   STableRecord tableRecord;
@@ -660,7 +660,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp) {
   return 0;
 }
 
-void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, struct arguments *arguments, FILE *fp) {
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) {
   char *pstr = NULL;
   pstr = buffer;
   int counter = 0;
@@ -703,7 +703,7 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, struct argume
   fprintf(fp, "%s\n\n", buffer);
 }
 
-void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, struct arguments *arguments,
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, SDumpArguments *arguments,
                                 FILE *fp) {
   char *pstr = NULL;
   pstr = buffer;
@@ -786,7 +786,7 @@ int taosGetTableDes(char *table, STableDef *tableDes) {
   return count;
 }
 
-int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp) {
+int32_t taosDumpTable(char *table, char *metric, SDumpArguments *arguments, FILE *fp) {
   int count = 0;
 
   STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
@@ -828,7 +828,7 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI
   return taosDumpTableData(fp, table, arguments);
 }
 
-int32_t taosDumpMetric(char *metric, struct arguments *arguments, FILE *fp) {
+int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
   TAOS_ROW row = NULL;
   int fd = -1;
   STableRecord tableRecord;
@@ -877,7 +877,7 @@ int32_t taosDumpMetric(char *metric, struct arguments *arguments, FILE *fp) {
   return 0;
 }
 
-int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) {
+int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
   /* char       temp[MAX_COMMAND_SIZE] = "\0"; */
   int count = 0;
   char *pstr = NULL;
@@ -987,7 +987,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) {
   return 0;
 }
 
-int taosCheckParam(struct arguments *arguments) {
+int taosCheckParam(SDumpArguments *arguments) {
   if (arguments->all_databases && arguments->databases) {
     fprintf(stderr, "conflict option --all-databases and --databases\n");
     return -1;
@@ -1072,7 +1072,7 @@ void taosReplaceCtrlChar(char *str) {
   *pstr = '\0';
 }
 
-int taosDumpIn(struct arguments *arguments) {
+int taosDumpIn(SDumpArguments *arguments) {
   assert(arguments->isDumpIn);
 
   int     tsize = 0;

src/mnode/inc/mgmtAcct.h

@@ -25,7 +25,7 @@ extern "C" {
 int32_t mgmtInitAccts();
 void    mgmtCleanUpAccts();
 void *  mgmtGetAcct(char *acctName);
-void *  mgmtGetNextAcct(void *pNode, SAcctObj **pAcct);
+void *  mgmtGetNextAcct(void *pIter, SAcctObj **pAcct);
 void    mgmtIncAcctRef(SAcctObj *pAcct);
 void    mgmtDecAcctRef(SAcctObj *pAcct);
 void    mgmtAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);

src/mnode/inc/mgmtDb.h

@@ -32,7 +32,7 @@ int32_t mgmtInitDbs();
 void    mgmtCleanUpDbs();
 SDbObj *mgmtGetDb(char *db);
 SDbObj *mgmtGetDbByTableId(char *db);
-void *  mgmtGetNextDb(void *pNode, SDbObj **pDb);
+void *  mgmtGetNextDb(void *pIter, SDbObj **pDb);
 void    mgmtIncDbRef(SDbObj *pDb);
 void    mgmtDecDbRef(SDbObj *pDb);
 bool    mgmtCheckIsMonitorDB(char *db, char *monitordb);

src/mnode/inc/mgmtDef.h

@@ -33,7 +33,7 @@ typedef struct SDnodeObj {
   int32_t    dnodeId;
   uint16_t   dnodePort;
   char       dnodeFqdn[TSDB_FQDN_LEN + 1];
-  char       dnodeEp[TSDB_FQDN_LEN + 1];
+  char       dnodeEp[TSDB_EP_LEN + 1];
   int64_t    createdTime;
   uint32_t   lastAccess;
   int32_t    openVnodes;
@@ -68,7 +68,7 @@ typedef struct SMnodeObj {
 
 // todo use dynamic length string
 typedef struct {
-  char   tableId[TSDB_TABLE_ID_LEN + 1];
+  char  *tableId;
   int8_t type;
 } STableObj;
 
@@ -77,6 +77,7 @@ typedef struct SSuperTableObj {
   uint64_t   uid;
   int64_t    createdTime;
   int32_t    sversion;
+  int32_t    tversion;
   int32_t    numOfColumns;
   int32_t    numOfTags;
   int8_t     reserved[15];
@@ -122,7 +123,6 @@ typedef struct SVgObj {
   int32_t        numOfVnodes;
   int32_t        lbDnodeId;
   int32_t        lbTime;
-  int8_t         status;
   int8_t         inUse;
   int8_t         reserved[13];
   int8_t         updateEnd[1];
@@ -223,7 +223,7 @@ typedef struct SAcctObj {
 typedef struct {
   int8_t   type;
   char     db[TSDB_DB_NAME_LEN + 1];
-  void *   pNode;
+  void *   pIter;
   int16_t  numOfColumns;
   int32_t  rowSize;
   int32_t  numOfRows;

src/mnode/inc/mgmtDnode.h

@@ -34,7 +34,7 @@ char*   mgmtGetDnodeStatusStr(int32_t dnodeStatus);
 void    mgmtMonitorDnodeModule();
 
 int32_t mgmtGetDnodesNum();
-void *  mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode);
+void *  mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode);
 void    mgmtIncDnodeRef(SDnodeObj *pDnode);
 void    mgmtDecDnodeRef(SDnodeObj *pDnode);
 void *  mgmtGetDnode(int32_t dnodeId);

src/mnode/inc/mgmtMnode.h

@@ -37,7 +37,7 @@ void    mgmtDropMnodeLocal(int32_t dnodeId);
 
 void *  mgmtGetMnode(int32_t mnodeId);
 int32_t mgmtGetMnodesNum();
-void *  mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
+void *  mgmtGetNextMnode(void *pIter, struct SMnodeObj **pMnode);
 void    mgmtIncMnodeRef(struct SMnodeObj *pMnode);
 void    mgmtDecMnodeRef(struct SMnodeObj *pMnode);
 

src/mnode/inc/mgmtSdb.h

@@ -35,7 +35,8 @@ typedef enum {
 typedef enum {
   SDB_KEY_STRING, 
   SDB_KEY_INT,
-  SDB_KEY_AUTO
+  SDB_KEY_AUTO,
+  SDB_KEY_VAR_STRING,
 } ESdbKey;
 
 typedef enum {
@@ -80,7 +81,8 @@ int32_t sdbDeleteRow(SSdbOper *pOper);
 int32_t sdbUpdateRow(SSdbOper *pOper);
 
 void    *sdbGetRow(void *handle, void *key);
-void    *sdbFetchRow(void *handle, void *pNode, void **ppRow);
+void    *sdbFetchRow(void *handle, void *pIter, void **ppRow);
+void     sdbFreeIter(void *pIter);
 void     sdbIncRef(void *thandle, void *pRow);
 void     sdbDecRef(void *thandle, void *pRow);
 int64_t  sdbGetNumOfRows(void *handle);

src/mnode/inc/mgmtTable.h

@@ -27,8 +27,8 @@ void    mgmtCleanUpTables();
 void *  mgmtGetTable(char *tableId);
 void    mgmtIncTableRef(void *pTable);
 void    mgmtDecTableRef(void *pTable);
-void *  mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable);
+void *  mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable);
-void *  mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable);
+void *  mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
 void    mgmtDropAllChildTables(SDbObj *pDropDb);
 void    mgmtDropAllSuperTables(SDbObj *pDropDb);
 

src/mnode/inc/mgmtUser.h

@@ -24,7 +24,7 @@ extern "C" {
 int32_t   mgmtInitUsers();
 void      mgmtCleanUpUsers();
 SUserObj *mgmtGetUser(char *name);
-void *    mgmtGetNextUser(void *pNode, SUserObj **pUser);
+void *    mgmtGetNextUser(void *pIter, SUserObj **pUser);
 void      mgmtIncUserRef(SUserObj *pUser);
 void      mgmtDecUserRef(SUserObj *pUser);
 SUserObj *mgmtGetUserFromConn(void *pConn);

src/mnode/inc/mgmtVgroup.h

@@ -22,11 +22,6 @@ extern "C" {
 
 #include "mgmtDef.h"
 
-enum _TSDB_VG_STATUS {
-  TSDB_VG_STATUS_READY,
-  TSDB_VG_STATUS_UPDATE
-};
-
 int32_t mgmtInitVgroups();
 void    mgmtCleanUpVgroups();
 SVgObj *mgmtGetVgroup(int32_t vgId);
@@ -35,7 +30,7 @@ void    mgmtDecVgroupRef(SVgObj *pVgroup);
 void    mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
 void    mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
 
-void *  mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup);
+void *  mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup);
 void    mgmtUpdateVgroup(SVgObj *pVgroup);
 void    mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *dnodeId, SVnodeLoad *pVload);
 

src/mnode/src/mgmtAcct.c

@@ -126,8 +126,8 @@ void *mgmtGetAcct(char *name) {
   return sdbGetRow(tsAcctSdb, name);
 }
 
-void *mgmtGetNextAcct(void *pNode, SAcctObj **pAcct) {
+void *mgmtGetNextAcct(void *pIter, SAcctObj **pAcct) {
-  return sdbFetchRow(tsAcctSdb, pNode, (void **)pAcct); 
+  return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); 
 }
 
 void mgmtIncAcctRef(SAcctObj *pAcct) {

src/mnode/src/mgmtBalance.c

@@ -22,6 +22,7 @@
 #include "mgmtInt.h"
 #include "mgmtMnode.h"
 #include "mgmtDnode.h"
+#include "mgmtSdb.h"
 #include "mgmtVgroup.h"
 
 #ifndef _SYNC
@@ -33,13 +34,13 @@ void    balanceUpdateMgmt() {}
 void    balanceReset() {}
 
 int32_t balanceAllocVnodes(SVgObj *pVgroup) {
-  void *     pNode = NULL;
+  void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
   float      vnodeUsage = 1000.0;
 
   while (1) {
-    pNode = mgmtGetNextDnode(pNode, &pDnode);
+    pIter = mgmtGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
 
     if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) {
@@ -55,6 +56,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
     mgmtDecDnodeRef(pDnode);
   }
 
+  sdbFreeIter(pIter);
+
   if (pSelDnode == NULL) {
     mError("failed to alloc vnode to vgroup");
     return TSDB_CODE_NO_ENOUGH_DNODES;

src/mnode/src/mgmtDb.c

@@ -156,8 +156,8 @@ int32_t mgmtInitDbs() {
   return 0;
 }
 
-void *mgmtGetNextDb(void *pNode, SDbObj **pDb) {
+void *mgmtGetNextDb(void *pIter, SDbObj **pDb) {
-  return sdbFetchRow(tsDbSdb, pNode, (void **)pDb);
+  return sdbFetchRow(tsDbSdb, pIter, (void **)pDb);
 }
 
 SDbObj *mgmtGetDb(char *db) {
@@ -583,7 +583,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
   if (pUser == NULL) return 0;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextDb(pShow->pNode, &pDb);
+    pShow->pIter = mgmtGetNextDb(pShow->pIter, &pDb);
     if (pDb == NULL) break;
 
     cols = 0;
@@ -865,14 +865,15 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     }
   }
 
-  void *pNode = NULL;
+  void *pIter = NULL;
   while (1) {
     SVgObj *pVgroup = NULL;
-    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;   
     mgmtSendCreateVgroupMsg(pVgroup, NULL);
     mgmtDecVgroupRef(pVgroup);
   }
+  sdbFreeIter(pIter);
 
   if (oldReplica != pDb->cfg.replications) {
     balanceNotify();
@@ -983,12 +984,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
 void  mgmtDropAllDbs(SAcctObj *pAcct)  {
   int32_t numOfDbs = 0;
   SDbObj *pDb = NULL;
-  void *  pNode = NULL;
+  void *  pIter = NULL;
 
   mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user);
 
   while (1) {
-    pNode = mgmtGetNextDb(pNode, &pDb);
+    pIter = mgmtGetNextDb(pIter, &pDb);
     if (pDb == NULL) break;
 
     if (pDb->pAcct == pAcct) {
@@ -1005,5 +1006,7 @@ void  mgmtDropAllDbs(SAcctObj *pAcct)  {
     mgmtDecDbRef(pDb);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs);
 }

src/mnode/src/mgmtDnode.c

@@ -74,6 +74,7 @@ static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
  
 #ifndef _SYNC 
+  //TODO: drop dnode local
   mgmtDropAllDnodeVgroups(pDnode);
 #endif  
   mgmtDropMnodeLocal(pDnode->dnodeId);
@@ -170,8 +171,8 @@ void mgmtCleanupDnodes() {
   sdbCloseTable(tsDnodeSdb);
 }
 
-void *mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) { 
+void *mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode) { 
-  return sdbFetchRow(tsDnodeSdb, pNode, (void **)pDnode); 
+  return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); 
 }
 
 int32_t mgmtGetDnodesNum() {
@@ -184,17 +185,20 @@ void *mgmtGetDnode(int32_t dnodeId) {
 
 void *mgmtGetDnodeByEp(char *ep) {
   SDnodeObj *pDnode = NULL;
-  void *     pNode = NULL;
+  void *     pIter = NULL;
 
   while (1) {
-    pNode = mgmtGetNextDnode(pNode, &pDnode);
+    pIter = mgmtGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
     if (strcmp(ep, pDnode->dnodeEp) == 0) {
+      sdbFreeIter(pIter);
       return pDnode;
     }
     mgmtDecDnodeRef(pDnode);
   }
 
+  sdbFreeIter(pIter);
+
   return NULL;
 }
 
@@ -530,7 +534,7 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
 
   pShow->numOfRows = mgmtGetDnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
 
   mgmtDecUserRef(pUser);
 
@@ -544,7 +548,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
   char      *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextDnode(pShow->pNode, &pDnode);
+    pShow->pIter = mgmtGetNextDnode(pShow->pIter, &pDnode);
     if (pDnode == NULL) break;
 
     cols = 0;
@@ -636,7 +640,7 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 
   pShow->numOfRows = mgmtGetDnodesNum() * TSDB_MOD_MAX;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -648,7 +652,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo
 
   while (numOfRows < rows) {
     SDnodeObj *pDnode = NULL;
-    pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
+    pShow->pIter = mgmtGetNextDnode(pShow->pIter, (SDnodeObj **)&pDnode);
     if (pDnode == NULL) break;
 
     for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
@@ -738,7 +742,7 @@ static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   }
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -821,7 +825,8 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   if (pShow->payloadLen > 0 ) {
     pDnode = mgmtGetDnodeByEp(pShow->payload);
   } else {
-    mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode);
+    void *pIter = mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode);
+    sdbFreeIter(pIter);
   }
 
   if (pDnode != NULL) {
@@ -830,7 +835,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   }
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = pDnode;
+  pShow->pIter = pDnode;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -844,12 +849,12 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
 
   if (0 == rows) return 0;
 
-  pDnode = (SDnodeObj *)(pShow->pNode);
+  pDnode = (SDnodeObj *)(pShow->pIter);
   if (pDnode != NULL) {
-    void *pNode = NULL;
+    void *pIter = NULL;
     SVgObj *pVgroup;
     while (1) {
-      pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+      pIter = mgmtGetNextVgroup(pIter, &pVgroup);
       if (pVgroup == NULL) break;
 
       for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
@@ -869,6 +874,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
 
       mgmtDecVgroupRef(pVgroup);
     }
+    sdbFreeIter(pIter);
   } else {
     numOfRows = 0;
   }

src/mnode/src/mgmtMnode.c

@@ -95,11 +95,12 @@ static int32_t mgmtMnodeActionDecode(SSdbOper *pOper) {
 static int32_t mgmtMnodeActionRestored() {
   if (mgmtGetMnodesNum() == 1) {
     SMnodeObj *pMnode = NULL;
-    mgmtGetNextMnode(NULL, &pMnode);
+    void *pIter = mgmtGetNextMnode(NULL, &pMnode);
     if (pMnode != NULL) {
       pMnode->role = TAOS_SYNC_ROLE_MASTER;
       mgmtDecMnodeRef(pMnode);
     }
+    sdbFreeIter(pIter);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -157,8 +158,8 @@ void mgmtDecMnodeRef(SMnodeObj *pMnode) {
   sdbDecRef(tsMnodeSdb, pMnode);
 }
 
-void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { 
+void *mgmtGetNextMnode(void *pIter, SMnodeObj **pMnode) { 
-  return sdbFetchRow(tsMnodeSdb, pNode, (void **)pMnode); 
+  return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); 
 }
 
 char *mgmtGetMnodeRoleStr(int32_t role) {
@@ -177,10 +178,10 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
 }
 
 void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
-  void *pNode = NULL;
+  void *pIter = NULL;
   while (1) {
     SMnodeObj *pMnode = NULL;
-    pNode = mgmtGetNextMnode(pNode, &pMnode);
+    pIter = mgmtGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
     strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
@@ -194,6 +195,7 @@ void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
     
     mgmtDecMnodeRef(pMnode);
   }
+  sdbFreeIter(pIter);
 }
 
 void mgmtGetMnodeInfos(void *param) {
@@ -201,10 +203,10 @@ void mgmtGetMnodeInfos(void *param) {
   mnodes->inUse = 0;
   
   int32_t index = 0;
-  void *pNode = NULL;
+  void *pIter = NULL;
   while (1) {
     SMnodeObj *pMnode = NULL;
-    pNode = mgmtGetNextMnode(pNode, &pMnode);
+    pIter = mgmtGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
     mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
@@ -216,6 +218,7 @@ void mgmtGetMnodeInfos(void *param) {
     index++;
     mgmtDecMnodeRef(pMnode);
   }
+  sdbFreeIter(pIter);
 
   mnodes->nodeNum = index;
 }
@@ -317,7 +320,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
 
   pShow->numOfRows = mgmtGetMnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -330,7 +333,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
   char      *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextMnode(pShow->pNode, &pMnode);
+    pShow->pIter = mgmtGetNextMnode(pShow->pIter, &pMnode);
     if (pMnode == NULL) break;
 
     cols = 0;

src/mnode/src/mgmtProfile.c

@@ -140,7 +140,7 @@ int32_t mgmtGetQueries(SShowObj *pShow, void *pConn) {
 //
 //  // sorting based on useconds
 //
-//  pShow->pNode = pQueryShow;
+//  pShow->pIter = pQueryShow;
 
   return 0;
 }
@@ -187,7 +187,7 @@ int32_t mgmtGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
 
   pShow->numOfRows = 1000000;
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
   mgmtGetQueries(pShow, pConn);
@@ -252,7 +252,7 @@ int32_t mgmtRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pCo
   char *pWrite;
   int32_t   cols = 0;
 
-  SQueryShow *pQueryShow = (SQueryShow *)pShow->pNode;
+  SQueryShow *pQueryShow = (SQueryShow *)pShow->pIter;
 
   if (rows > pQueryShow->numOfQueries - pQueryShow->index) rows = pQueryShow->numOfQueries - pQueryShow->index;
 
@@ -339,7 +339,7 @@ int32_t mgmtGetStreams(SShowObj *pShow, void *pConn) {
 //
 //  // sorting based on useconds
 //
-//  pShow->pNode = pStreamShow;
+//  pShow->pIter = pStreamShow;
 
   return 0;
 }
@@ -397,7 +397,7 @@ int32_t mgmtGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
 
   pShow->numOfRows = 1000000;
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
   mgmtGetStreams(pShow, pConn);
@@ -409,7 +409,7 @@ int32_t mgmtRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pCo
   char *pWrite;
   int32_t   cols = 0;
 
-  SStreamShow *pStreamShow = (SStreamShow *)pShow->pNode;
+  SStreamShow *pStreamShow = (SStreamShow *)pShow->pIter;
 
   if (rows > pStreamShow->numOfStreams - pStreamShow->index) rows = pStreamShow->numOfStreams - pStreamShow->index;
 
@@ -592,7 +592,7 @@ int mgmtGetConns(SShowObj *pShow, void *pConn) {
   //
   //  // sorting based on useconds
   //
-  //  pShow->pNode = pConnShow;
+  //  pShow->pIter = pConnShow;
 
   return 0;
 }
@@ -627,7 +627,7 @@ int32_t mgmtGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
 
   pShow->numOfRows = 1000000;
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
   mgmtGetConns(pShow, pConn);
@@ -639,7 +639,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
   char      *pWrite;
   int32_t   cols = 0;
 
-  SConnShow *pConnShow = (SConnShow *)pShow->pNode;
+  SConnShow *pConnShow = (SConnShow *)pShow->pIter;
 
   if (rows > pConnShow->numOfConns - pConnShow->index) rows = pConnShow->numOfConns - pConnShow->index;
 

src/mnode/src/mgmtSdb.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
+#include "hash.h"
 #include "trpc.h"
 #include "tutil.h"
 #include "tbalance.h"
@@ -23,8 +24,6 @@
 #include "twal.h"
 #include "tsync.h"
 #include "tglobal.h"
-#include "hashint.h"
-#include "hashstr.h"
 #include "dnode.h"
 #include "mgmtDef.h"
 #include "mgmtInt.h"
@@ -83,12 +82,6 @@ typedef struct {
 } SSdbRow;
 
 static SSdbObject tsSdbObj = {0};
-static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash};
-static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash};
-static void  (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash};
-static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData};
-static void  (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash};
-static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData};
 static int sdbWrite(void *param, void *data, int type);
 
 int32_t sdbGetId(void *handle) {
@@ -111,6 +104,14 @@ bool sdbIsServing() {
   return tsSdbObj.status == SDB_STATUS_SERVING; 
 }
 
+static void *sdbGetObjKey(SSdbTable *pTable, void *key) {
+  if (pTable->keyType == SDB_KEY_VAR_STRING) {
+    return *(char **)key;
+  }
+
+  return key;
+}
+
 static char *sdbGetActionStr(int32_t action) {
   switch (action) {
     case SDB_ACTION_INSERT:
@@ -123,20 +124,25 @@ static char *sdbGetActionStr(int32_t action) {
   return "invalid";
 }
 
-static char *sdbGetkeyStr(SSdbTable *pTable, void *row) {
+static char *sdbGetKeyStr(SSdbTable *pTable, void *key) {
   static char str[16];
   switch (pTable->keyType) {
     case SDB_KEY_STRING:
-      return (char *)row;
+    case SDB_KEY_VAR_STRING:
+      return (char *)key;
     case SDB_KEY_INT:
     case SDB_KEY_AUTO:
-      sprintf(str, "%d", *(int32_t *)row);
+      sprintf(str, "%d", *(int32_t *)key);
       return str;
     default:
       return "invalid";
   }
 }
 
+static char *sdbGetKeyStrFromObj(SSdbTable *pTable, void *key) {
+  return sdbGetKeyStr(pTable, sdbGetObjKey(pTable, key));
+}
+
 static void *sdbGetTableFromId(int32_t tableId) {
   return tsSdbObj.tableList[tableId];
 }
@@ -192,7 +198,7 @@ void sdbUpdateMnodeRoles() {
   }
 }
 
-static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) {
+static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) {
   sdbUpdateMnodeRoles();
   return 0;
 }
@@ -244,10 +250,10 @@ void sdbUpdateSync() {
   }
 
   if (index == 0) {
-    void *pNode = NULL;
+    void *pIter = NULL;
     while (1) {
       SMnodeObj *pMnode = NULL;
-      pNode = mgmtGetNextMnode(pNode, &pMnode);
+      pIter = mgmtGetNextMnode(pIter, &pMnode);
       if (pMnode == NULL) break;
 
       syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
@@ -257,6 +263,7 @@ void sdbUpdateSync() {
 
       mgmtDecMnodeRef(pMnode);
     }
+    sdbFreeIter(pIter);
   }
 
   syncCfg.replica = index;
@@ -338,46 +345,48 @@ void sdbCleanUp() {
   pthread_mutex_destroy(&tsSdbObj.mutex);
 }
 
-void sdbIncRef(void *handle, void *pRow) {
+void sdbIncRef(void *handle, void *pObj) {
-  if (pRow) {
+  if (pObj == NULL) return;
-    SSdbTable *pTable = handle;
+
-    int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
+  SSdbTable *pTable = handle;
-    atomic_add_fetch_32(pRefCount, 1);
+  int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
-    if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
+  atomic_add_fetch_32(pRefCount, 1);
-      sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
+  if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
-               *pRefCount);
+    sdbTrace("table:%s, add ref to record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
-    }
   }
 }
 
-void sdbDecRef(void *handle, void *pRow) {
+void sdbDecRef(void *handle, void *pObj) {
-  if (pRow) {
+  if (pObj == NULL) return;
-    SSdbTable *pTable = handle;
+
-    int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
+  SSdbTable *pTable = handle;
-    int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
+  int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
-    if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
+  int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
-      sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
+  if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
-               *pRefCount);
+    sdbTrace("table:%s, def ref of record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
-    }
+  }
-    int8_t *updateEnd = pRow + pTable->refCountPos - 1;
+
-    if (refCount <= 0 && *updateEnd) {
+  int8_t *updateEnd = pObj + pTable->refCountPos - 1;
-      sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName,
+  if (refCount <= 0 && *updateEnd) {
-               sdbGetkeyStr(pTable, pRow), *pRefCount);
+    sdbTrace("table:%s, record:%s:%d is destroyed", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
-      SSdbOper oper = {.pObj = pRow};
+    SSdbOper oper = {.pObj = pObj};
-      (*pTable->destroyFp)(&oper);
+    (*pTable->destroyFp)(&oper);
-    }
   }
 }
 
-static SSdbRow *sdbGetRowMeta(void *handle, void *key) {
+static SSdbRow *sdbGetRowMeta(SSdbTable *pTable, void *key) {
-  SSdbTable *pTable = (SSdbTable *)handle;
+  if (pTable == NULL) return NULL;
-  SSdbRow * pMeta;
 
-  if (handle == NULL) return NULL;
+  int32_t keySize = sizeof(int32_t);
+  if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
+    keySize = strlen((char *)key);
+  }
   
-  pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
+  return taosHashGet(pTable->iHandle, key, keySize);
+}
 
-  return pMeta;
+static SSdbRow *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) {
+  return sdbGetRowMeta(pTable, sdbGetObjKey(pTable, key));
 }
 
 void *sdbGetRow(void *handle, void *key) {
@@ -387,24 +396,41 @@ void *sdbGetRow(void *handle, void *key) {
   if (handle == NULL) return NULL;
 
   pthread_mutex_lock(&pTable->mutex);
-  pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
+
+  int32_t keySize = sizeof(int32_t);
+  if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
+    keySize = strlen((char *)key);
+  }
+  pMeta = taosHashGet(pTable->iHandle, key, keySize);
+
   if (pMeta) sdbIncRef(pTable, pMeta->row);
   pthread_mutex_unlock(&pTable->mutex);
 
-  if (pMeta == NULL) {
+  if (pMeta == NULL) return NULL;
-    return NULL;
-  }
 
   return pMeta->row;
 }
 
+static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) {
+  return sdbGetRow(pTable, sdbGetObjKey(pTable, key));
+}
+
 static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
   SSdbRow rowMeta;
   rowMeta.rowSize = pOper->rowSize;
   rowMeta.row = pOper->pObj;
 
   pthread_mutex_lock(&pTable->mutex);
-  (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta);
+
+  void *  key = sdbGetObjKey(pTable, pOper->pObj);
+  int32_t keySize = sizeof(int32_t);
+
+  if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
+    keySize = strlen((char *)key);
+  }
+
+  taosHashPut(pTable->iHandle, key, keySize, &rowMeta, sizeof(SSdbRow));
+
   sdbIncRef(pTable, pOper->pObj);
   pTable->numOfRows++;
 
@@ -417,7 +443,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
   pthread_mutex_unlock(&pTable->mutex);
 
   sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
-           sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
+           sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
 
   (*pTable->insertFp)(pOper);
   return TSDB_CODE_SUCCESS;
@@ -427,12 +453,21 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
   (*pTable->deleteFp)(pOper);
   
   pthread_mutex_lock(&pTable->mutex);
-  (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj);
+
+  void *  key = sdbGetObjKey(pTable, pOper->pObj);
+  int32_t keySize = sizeof(int32_t);
+
+  if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
+    keySize = strlen((char *)key);
+  }
+
+  taosHashRemove(pTable->iHandle, key, keySize);
+
   pTable->numOfRows--;
   pthread_mutex_unlock(&pTable->mutex);
 
   sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
-           sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
+           sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
 
   int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
   *updateEnd = 1;
@@ -443,7 +478,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
 
 static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
   sdbTrace("table:%s, update record:%s in hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
-           sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
+           sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
 
   (*pTable->updateFp)(pOper);
   return TSDB_CODE_SUCCESS;
@@ -474,7 +509,7 @@ static int sdbWrite(void *param, void *data, int type) {
     } else if (pHead->version != tsSdbObj.version + 1) {
       pthread_mutex_unlock(&tsSdbObj.mutex);
       sdbError("table:%s, failed to restore %s record:%s from wal, version:%" PRId64 " too large, sdb version:%" PRId64,
-               pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version,
+               pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version,
                tsSdbObj.version);
       return TSDB_CODE_OTHERS;
     } else {
@@ -526,8 +561,8 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
   SSdbTable *pTable = (SSdbTable *)pOper->table;
   if (pTable == NULL) return -1;
 
-  if (sdbGetRow(pTable, pOper->pObj)) {
+  if (sdbGetRowFromObj(pTable, pOper->pObj)) {
-    sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj));
+    sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj));
     sdbDecRef(pTable, pOper->pObj);
     return TSDB_CODE_ALREADY_THERE;
   }
@@ -566,7 +601,7 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
   SSdbTable *pTable = (SSdbTable *)pOper->table;
   if (pTable == NULL) return -1;
 
-  SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
+  SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj);
   if (pMeta == NULL) {
     sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
     return -1;
@@ -576,25 +611,27 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
   assert(pMetaRow != NULL);
 
   if (pOper->type == SDB_OPER_GLOBAL) {
-    int32_t rowSize = 0;
+    void *  key = sdbGetObjKey(pTable, pOper->pObj);
+    int32_t keySize = 0;
     switch (pTable->keyType) {
       case SDB_KEY_STRING:
-        rowSize = strlen((char *)pOper->pObj) + 1;
+      case SDB_KEY_VAR_STRING:
+        keySize = strlen((char *)key) + 1;
         break;
       case SDB_KEY_INT:
       case SDB_KEY_AUTO:
-        rowSize = sizeof(uint64_t);
+        keySize = sizeof(uint32_t);
         break;
       default:
         return -1;
     }
 
-    int32_t   size = sizeof(SWalHead) + rowSize;
+    int32_t   size = sizeof(SWalHead) + keySize;
     SWalHead *pHead = taosAllocateQitem(size);
     pHead->version = 0;
-    pHead->len = rowSize;
+    pHead->len = keySize;
     pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
-    memcpy(pHead->cont, pOper->pObj, rowSize);
+    memcpy(pHead->cont, key, keySize);
 
     int32_t code = sdbWrite(pOper, pHead, pHead->msgType);
     taosFreeQitem(pHead);
@@ -608,7 +645,7 @@ int32_t sdbUpdateRow(SSdbOper *pOper) {
   SSdbTable *pTable = (SSdbTable *)pOper->table;
   if (pTable == NULL) return -1;
 
-  SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
+  SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj);
   if (pMeta == NULL) {
     sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
     return -1;
@@ -637,18 +674,35 @@ int32_t sdbUpdateRow(SSdbOper *pOper) {
 
 void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
   SSdbTable *pTable = (SSdbTable *)handle;
-  SSdbRow * pMeta;
-
   *ppRow = NULL;
   if (pTable == NULL) return NULL;
 
-  pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
+  SHashMutableIterator *pIter = pNode;
-  if (pMeta == NULL) return NULL;
+  if (pIter == NULL) {
+    pIter = taosHashCreateIter(pTable->iHandle);
+  }
+
+  if (!taosHashIterNext(pIter)) {
+    taosHashDestroyIter(pIter);
+    return NULL;
+  }
+
+  SSdbRow *pMeta = taosHashIterGet(pIter);
+  if (pMeta == NULL) {
+    taosHashDestroyIter(pIter);
+    return NULL;
+  }
 
   *ppRow = pMeta->row;
   sdbIncRef(handle, pMeta->row);
 
-  return pNode;
+  return pIter;
+}
+
+void sdbFreeIter(void *pIter) {
+  if (pIter != NULL) {
+    taosHashDestroyIter(pIter);
+  }
 }
 
 void *sdbOpenTable(SSdbTableDesc *pDesc) {
@@ -670,9 +724,11 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
   pTable->destroyFp    = pDesc->destroyFp;
   pTable->restoredFp   = pDesc->restoredFp;
 
-  if (sdbInitIndexFp[pTable->keyType] != NULL) {
+  _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
-    pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SSdbRow));
+  if (pTable->keyType == SDB_KEY_STRING) {
+    hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   }
+  pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true);
 
   pthread_mutex_init(&pTable->mutex, NULL);
 
@@ -688,11 +744,10 @@ void sdbCloseTable(void *handle) {
   tsSdbObj.numOfTables--;
   tsSdbObj.tableList[pTable->tableId] = NULL;
 
-  void *pNode = NULL;
+  SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle);
-  while (1) {
+  while (taosHashIterNext(pIter)) {
-    SSdbRow *pMeta;
+    SSdbRow *pMeta = taosHashIterGet(pIter);
-    pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
+    if (pMeta == NULL) continue;
-    if (pMeta == NULL) break;
 
     SSdbOper oper = {
       .pObj = pMeta->row,
@@ -702,9 +757,8 @@ void sdbCloseTable(void *handle) {
     (*pTable->destroyFp)(&oper);
   }
 
-  if (sdbCleanUpIndexFp[pTable->keyType]) {
+  taosHashDestroyIter(pIter);
-    (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle);
+  taosHashCleanup(pTable->iHandle);
-  }
 
   pthread_mutex_destroy(&pTable->mutex);
   

src/mnode/src/mgmtShell.c

@@ -48,6 +48,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg);
+static void mgmtFreeShowObj(void *data);
 
 void *tsMgmtTmr;
 static void *tsMgmtTranQhandle = NULL;
@@ -65,7 +66,7 @@ int32_t mgmtInitShell() {
   
   tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
   tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT");
-  tsQhandleCache = taosCacheInit(tsMgmtTmr, 10);
+  tsQhandleCache = taosCacheInitWithCb(tsMgmtTmr, 10, mgmtFreeShowObj);
 
   return 0;
 }
@@ -123,6 +124,8 @@ void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) {
 
 void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
 
+  mTrace("%p, msg:%s will be processed", rpcMsg->ahandle, taosMsg[rpcMsg->msgType]);
+
   if (rpcMsg->pCont == NULL) {
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_MSG_LEN);
     return;
@@ -476,10 +479,10 @@ void mgmtSendSimpleResp(void *thandle, int32_t code) {
 bool mgmtCheckQhandle(uint64_t qhandle) {
   void *pSaved = taosCacheAcquireByData(tsQhandleCache, (void *)qhandle);
   if (pSaved == (void *)qhandle) {
-    mTrace("qhandle:%p is retrived", qhandle);
+    mTrace("show:%p, is retrieved", qhandle);
     return true;
   } else {
-    mTrace("qhandle:%p is already freed", qhandle);
+    mTrace("show:%p, is already released", qhandle);
     return false;
   }
 }
@@ -491,15 +494,21 @@ void* mgmtSaveQhandle(void *qhandle, int32_t size) {
     void *newQhandle = taosCachePut(tsQhandleCache, key, qhandle, size, 60);
     free(qhandle);
     
-    mTrace("qhandle:%p is saved", newQhandle);
+    mTrace("show:%p, is saved", newQhandle);
     return newQhandle;
   }
   
   return NULL;
 }
 
+static void mgmtFreeShowObj(void *data) {
+  SShowObj *pShow = data;
+  sdbFreeIter(pShow->pIter);
+  mTrace("show:%p, is destroyed", pShow);
+}
+
 void mgmtFreeQhandle(void *qhandle, bool forceRemove) {
-  mTrace("qhandle:%p is freed", qhandle);
+  mTrace("show:%p, is released, force:%s", qhandle, forceRemove ? "true" : "false");
   taosCacheRelease(tsQhandleCache, &qhandle, forceRemove);
 }
 

src/mnode/src/mgmtTable.c

@@ -84,6 +84,7 @@ static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg);
 static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName);
 
 static void mgmtDestroyChildTable(SChildTableObj *pTable) {
+  tfree(pTable->info.tableId);
   tfree(pTable->schema);
   tfree(pTable->sql);
   tfree(pTable);
@@ -180,6 +181,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) {
   SChildTableObj *pNew = pOper->pObj;
   SChildTableObj *pTable = mgmtGetChildTable(pNew->info.tableId);
   if (pTable != pNew) {
+    void *oldTableId = pTable->info.tableId;
     void *oldSql = pTable->sql;
     void *oldSchema = pTable->schema;
     memcpy(pTable, pNew, pOper->rowSize);
@@ -188,6 +190,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) {
     free(pNew);
     free(oldSql);
     free(oldSchema);
+    free(oldTableId);
   }
   mgmtDecTableRef(pTable);
 
@@ -195,51 +198,66 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) {
 }
 
 static int32_t mgmtChildTableActionEncode(SSdbOper *pOper) {
-  const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16);
   SChildTableObj *pTable = pOper->pObj;
   assert(pTable != NULL && pOper->rowData != NULL);
 
-  if (pTable->info.type == TSDB_CHILD_TABLE) {
+  int32_t len = strlen(pTable->info.tableId);
-    memcpy(pOper->rowData, pTable, tsChildTableUpdateSize);
+  if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID;
-    pOper->rowSize = tsChildTableUpdateSize;
+
-  } else {
+  memcpy(pOper->rowData, pTable->info.tableId, len);
+  memset(pOper->rowData + len, 0, 1);
+  len++;
+
+  memcpy(pOper->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize);
+  len += tsChildTableUpdateSize;
+
+  if (pTable->info.type != TSDB_CHILD_TABLE) {
     int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
-    if (maxRowSize < tsChildTableUpdateSize + schemaSize) {
+    memcpy(pOper->rowData + len, pTable->schema, schemaSize);
-      return TSDB_CODE_INVALID_MSG_LEN;
+    len += schemaSize;
+
+    if (pTable->sqlLen != 0) {
+      memcpy(pOper->rowData + len, pTable->sql, pTable->sqlLen);
+      len += pTable->sqlLen;
     }
-    memcpy(pOper->rowData, pTable, tsChildTableUpdateSize);
-    memcpy(pOper->rowData + tsChildTableUpdateSize, pTable->schema, schemaSize);
-    memcpy(pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen);
-    pOper->rowSize = tsChildTableUpdateSize + schemaSize + pTable->sqlLen;
   }
 
+  pOper->rowSize = len;
+
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) {
   assert(pOper->rowData != NULL);
   SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
-  if (pTable == NULL) {
+  if (pTable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
-    return TSDB_CODE_SERV_OUT_OF_MEMORY;
-  }
 
-  memcpy(pTable, pOper->rowData, tsChildTableUpdateSize);
+  int32_t len = strlen(pOper->rowData);
+  if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID;
+  pTable->info.tableId = strdup(pOper->rowData);
+  len++;
+
+  memcpy((char*)pTable + sizeof(char *), pOper->rowData + len, tsChildTableUpdateSize);
+  len += tsChildTableUpdateSize;
 
   if (pTable->info.type != TSDB_CHILD_TABLE) {
     int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
     pTable->schema = (SSchema *)malloc(schemaSize);
     if (pTable->schema == NULL) {
       mgmtDestroyChildTable(pTable);
-      return TSDB_CODE_SERV_OUT_OF_MEMORY;
+      return TSDB_CODE_INVALID_TABLE_TYPE;
     }
-    memcpy(pTable->schema, pOper->rowData + tsChildTableUpdateSize, schemaSize);
+    memcpy(pTable->schema, pOper->rowData + len, schemaSize);
+    len += schemaSize;
 
-    pTable->sql = (char *)malloc(pTable->sqlLen);
+    if (pTable->sqlLen != 0) {
-    if (pTable->sql == NULL) {
+      pTable->sql = malloc(pTable->sqlLen);
-      mgmtDestroyChildTable(pTable);
+      if (pTable->sql == NULL) {
-      return TSDB_CODE_SERV_OUT_OF_MEMORY;
+        mgmtDestroyChildTable(pTable);
+        return TSDB_CODE_SERV_OUT_OF_MEMORY;
+      }
+      memcpy(pTable->sql, pOper->rowData + len, pTable->sqlLen);
     }
-    memcpy(pTable->sql, pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sqlLen);
   }
 
   pOper->pObj = pTable;
@@ -247,25 +265,19 @@ static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) {
 }
 
 static int32_t mgmtChildTableActionRestored() {
-  void *pNode = NULL;
+  void *pIter = NULL;
-  void *pLastNode = NULL;
   SChildTableObj *pTable = NULL;
 
   while (1) {
-    pLastNode = pNode;
     mgmtDecTableRef(pTable);
-    pNode = mgmtGetNextChildTable(pNode, &pTable);
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
     if (pTable == NULL) break;
 
     SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId);
     if (pDb == NULL) {
       mError("ctable:%s, failed to get db, discard it", pTable->info.tableId);
-      SSdbOper desc = {0};
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
     mgmtDecDbRef(pDb);
@@ -274,12 +286,8 @@ static int32_t mgmtChildTableActionRestored() {
     if (pVgroup == NULL) {
       mError("ctable:%s, failed to get vgId:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
       pTable->vgId = 0;
-      SSdbOper desc = {0};
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
     mgmtDecVgroupRef(pVgroup);
@@ -288,24 +296,16 @@ static int32_t mgmtChildTableActionRestored() {
       mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it",
              pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
       pTable->vgId = 0;
-      SSdbOper desc = {0};
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
 
     if (pVgroup->tableList == NULL) {
       mError("ctable:%s, vgId:%d tableList is null", pTable->info.tableId, pTable->vgId);
       pTable->vgId = 0;
-      SSdbOper desc = {0};
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
 
@@ -314,32 +314,30 @@ static int32_t mgmtChildTableActionRestored() {
       if (pSuperTable == NULL) {
         mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid);
         pTable->vgId = 0;
-        SSdbOper desc = {0};
+        SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
-        desc.type = SDB_OPER_LOCAL;
-        desc.pObj = pTable;
-        desc.table = tsChildTableSdb;
         sdbDeleteRow(&desc);
-        pNode = pLastNode;
         continue;
       }
       mgmtDecTableRef(pSuperTable);
     }
   }
 
+  sdbFreeIter(pIter);
+
   return 0;
 }
 
 static int32_t mgmtInitChildTables() {
   SChildTableObj tObj;
-  tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+  tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
 
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_CTABLE,
     .tableName    = "ctables",
     .hashSessions = tsMaxTables,
-    .maxRowSize   = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16),
+    .maxRowSize   = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
-    .keyType      = SDB_KEY_STRING,
+    .keyType      = SDB_KEY_VAR_STRING,
     .insertFp     = mgmtChildTableActionInsert,
     .deleteFp     = mgmtChildTableActionDelete,
     .updateFp     = mgmtChildTableActionUpdate,
@@ -392,6 +390,7 @@ static void mgmtDestroySuperTable(SSuperTableObj *pStable) {
     taosHashCleanup(pStable->vgHash);
     pStable->vgHash = NULL;
   }
+  tfree(pStable->info.tableId);
   tfree(pStable->schema);
   tfree(pStable);
 }
@@ -428,11 +427,13 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
   SSuperTableObj *pNew = pOper->pObj;
   SSuperTableObj *pTable = mgmtGetSuperTable(pNew->info.tableId);
   if (pTable != pNew) {
+    void *oldTableId = pTable->info.tableId;
     void *oldSchema = pTable->schema;
     memcpy(pTable, pNew, pOper->rowSize);
     pTable->schema = pNew->schema;
     free(pNew->vgHash);
     free(pNew);
+    free(oldTableId);
     free(oldSchema);
   }
   mgmtDecTableRef(pTable);
@@ -440,40 +441,50 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
 }
 
 static int32_t mgmtSuperTableActionEncode(SSdbOper *pOper) {
-  const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16);
-
   SSuperTableObj *pStable = pOper->pObj;
   assert(pOper->pObj != NULL && pOper->rowData != NULL);
 
+  int32_t len = strlen(pStable->info.tableId);
+  if (len > TSDB_TABLE_ID_LEN) len = TSDB_CODE_INVALID_TABLE_ID;
+
+  memcpy(pOper->rowData, pStable->info.tableId, len);
+  memset(pOper->rowData + len, 0, 1);
+  len++;
+
+  memcpy(pOper->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize);
+  len += tsSuperTableUpdateSize;
+
   int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
+  memcpy(pOper->rowData + len, pStable->schema, schemaSize);
+  len += schemaSize;
 
-  if (maxRowSize < tsSuperTableUpdateSize + schemaSize) {
+  pOper->rowSize = len;
-    return TSDB_CODE_INVALID_MSG_LEN;
-  }
-
-  memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize);
-  memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize);
-  pOper->rowSize = tsSuperTableUpdateSize + schemaSize;
 
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mgmtSuperTableActionDecode(SSdbOper *pOper) {
   assert(pOper->rowData != NULL);
-
   SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj));
   if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
 
-  memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize);
+  int32_t len = strlen(pOper->rowData);
+  if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID;
+  pStable->info.tableId = strdup(pOper->rowData);
+  len++;
+
+  memcpy((char*)pStable + sizeof(char *), pOper->rowData + len, tsSuperTableUpdateSize);
+  len += tsSuperTableUpdateSize;
 
   int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
   pStable->schema = malloc(schemaSize);
   if (pStable->schema == NULL) {
     mgmtDestroySuperTable(pStable);
-    return -1;
+    return TSDB_CODE_NOT_SUPER_TABLE;
   }
 
-  memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize);
+  memcpy(pStable->schema, pOper->rowData + len, schemaSize);
+  
   pOper->pObj = pStable;
 
   return TSDB_CODE_SUCCESS;
@@ -485,15 +496,15 @@ static int32_t mgmtSuperTableActionRestored() {
 
 static int32_t mgmtInitSuperTables() {
   SSuperTableObj tObj;
-  tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+  tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
 
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_STABLE,
     .tableName    = "stables",
     .hashSessions = TSDB_MAX_SUPER_TABLES,
-    .maxRowSize   = tsSuperTableUpdateSize + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16),
+    .maxRowSize   = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
-    .keyType      = SDB_KEY_STRING,
+    .keyType      = SDB_KEY_VAR_STRING,
     .insertFp     = mgmtSuperTableActionInsert,
     .deleteFp     = mgmtSuperTableActionDelete,
     .updateFp     = mgmtSuperTableActionUpdate,
@@ -560,17 +571,20 @@ static void *mgmtGetSuperTable(char *tableId) {
 
 static void *mgmtGetSuperTableByUid(uint64_t uid) {
   SSuperTableObj *pStable = NULL;
-  void *          pNode = NULL;
+  void *pIter = NULL;
 
   while (1) {
-    pNode = mgmtGetNextSuperTable(pNode, &pStable);
+    pIter = mgmtGetNextSuperTable(pIter, &pStable);
     if (pStable == NULL) break;
     if (pStable->uid == uid) {
+      sdbFreeIter(pIter);
       return pStable;
     }
     mgmtDecTableRef(pStable);
   }
 
+  sdbFreeIter(pIter);
+
   return NULL;
 }
 
@@ -588,12 +602,12 @@ void *mgmtGetTable(char *tableId) {
   return NULL;
 }
 
-void *mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable) {
+void *mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable) {
-  return sdbFetchRow(tsChildTableSdb, pNode, (void **)pTable);
+  return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable);
 }
 
-void *mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable) {
+void *mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable) {
-  return sdbFetchRow(tsSuperTableSdb, pNode, (void **)pTable);
+  return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable);
 }
 
 void mgmtIncTableRef(void *p1) {
@@ -737,18 +751,19 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
 
 static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
   SCMCreateTableMsg *pCreate = pMsg->pCont;
-  SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj));
+  SSuperTableObj *pStable = calloc(1, sizeof(SSuperTableObj));
   if (pStable == NULL) {
     mError("table:%s, failed to create, no enough memory", pCreate->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
     return;
   }
 
-  strcpy(pStable->info.tableId, pCreate->tableId);
+  pStable->info.tableId = strdup(pCreate->tableId);
   pStable->info.type    = TSDB_SUPER_TABLE;
   pStable->createdTime  = taosGetTimestampMs();
   pStable->uid          = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul));
   pStable->sversion     = 0;
+  pStable->tversion     = 0;
   pStable->numOfColumns = htons(pCreate->numOfColumns);
   pStable->numOfTags    = htons(pCreate->numOfTags);
 
@@ -812,6 +827,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
       dnodeSendMsgToDnode(&ipSet, &rpcMsg);
       mgmtDecVgroupRef(pVgroup);
     }
+    taosHashDestroyIter(pIter);
 
     mgmtDropAllChildTablesInStable(pStable);
   } 
@@ -867,7 +883,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
   }
 
   pStable->numOfTags += ntags;
-  pStable->sversion++;
+  pStable->tversion++;
 
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
@@ -894,7 +910,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
   memmove(pStable->schema + pStable->numOfColumns + col, pStable->schema + pStable->numOfColumns + col + 1,
           sizeof(SSchema) * (pStable->numOfTags - col - 1));
   pStable->numOfTags--;
-  pStable->sversion++;
+  pStable->tversion++;
 
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
@@ -1121,7 +1137,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
 
   while (numOfRows < rows) {    
     mgmtDecTableRef(pTable);
-    pShow->pNode = mgmtGetNextSuperTable(pShow->pNode, &pTable);
+    pShow->pIter = mgmtGetNextSuperTable(pShow->pIter, &pTable);
     if (pTable == NULL) break;
     if (strncmp(pTable->info.tableId, prefix, prefixLen)) {
       continue;
@@ -1171,8 +1187,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
 }
 
 void mgmtDropAllSuperTables(SDbObj *pDropDb) {
-  void *pNode = NULL;
+  void *  pIter= NULL;
-  void *pLastNode = NULL;
   int32_t numOfTables = 0;
   int32_t dbNameLen = strlen(pDropDb->name);
   SSuperTableObj *pTable = NULL;
@@ -1180,8 +1195,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
   mPrint("db:%s, all super tables will be dropped from sdb", pDropDb->name);
 
   while (1) {
-    pLastNode = pNode;
+    pIter = mgmtGetNextSuperTable(pIter, &pTable);
-    pNode = mgmtGetNextSuperTable(pNode, &pTable);
     if (pTable == NULL) break;
 
     if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@@ -1191,13 +1205,14 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
         .pObj = pTable,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfTables ++;
     }
 
     mgmtDecTableRef(pTable);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
@@ -1221,6 +1236,7 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
   STableMetaMsg *pMeta   = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16));
   pMeta->uid          = htobe64(pTable->uid);
   pMeta->sversion     = htons(pTable->sversion);
+  pMeta->tversion     = htons(pTable->tversion);
   pMeta->precision    = pMsg->pDb->cfg.precision;
   pMeta->numOfTags    = (uint8_t)pTable->numOfTags;
   pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns);
@@ -1290,6 +1306,8 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
       mgmtDecVgroupRef(pVgroup);
     }
 
+    taosHashDestroyIter(pIter);
+
     pVgroupInfo->numOfVgroups = htonl(vgSize);
 
     // one table is done, try the next table
@@ -1342,12 +1360,14 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb
     pCreate->numOfColumns  = htons(pTable->superTable->numOfColumns);
     pCreate->numOfTags     = htons(pTable->superTable->numOfTags);
     pCreate->sversion      = htonl(pTable->superTable->sversion);
+    pCreate->tversion      = htonl(pTable->superTable->tversion);
     pCreate->tagDataLen    = htonl(tagDataLen);
     pCreate->superTableUid = htobe64(pTable->superTable->uid);
   } else {
     pCreate->numOfColumns  = htons(pTable->numOfColumns);
     pCreate->numOfTags     = 0;
     pCreate->sversion      = htonl(pTable->sversion);
+    pCreate->tversion      = 0;
     pCreate->tagDataLen    = 0;
     pCreate->superTableUid = 0;
   }
@@ -1373,7 +1393,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb
 }
 
 static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) {
-  SChildTableObj *pTable = (SChildTableObj *) calloc(1, sizeof(SChildTableObj));
+  SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
   if (pTable == NULL) {
     mError("table:%s, failed to alloc memory", pCreate->tableId);
     terrno = TSDB_CODE_SERV_OUT_OF_MEMORY;
@@ -1386,16 +1406,16 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
     pTable->info.type = TSDB_NORMAL_TABLE;
   }
 
-  strcpy(pTable->info.tableId, pCreate->tableId);  
+  pTable->info.tableId = strdup(pCreate->tableId);  
-  pTable->createdTime = taosGetTimestampMs();
+  pTable->createdTime  = taosGetTimestampMs();
-  pTable->sid         = tid;
+  pTable->sid          = tid;
-  pTable->vgId        = pVgroup->vgId;
+  pTable->vgId         = pVgroup->vgId;
     
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     char *pTagData = (char *) pCreate->schema;  // it is a tag key
     SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData);
     if (pSuperTable == NULL) {
-      mError("table:%s, corresponding super table does not exist", pCreate->tableId);
+      mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData);
       free(pTable);
       terrno = TSDB_CODE_INVALID_TABLE;
       return NULL;
@@ -1475,16 +1495,21 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  int32_t sid = taosAllocateId(pVgroup->idPool);
-  if (sid <= 0) {
-    mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId);
-    mgmtCreateVgroup(mgmtCloneQueuedMsg(pMsg), pMsg->pDb);
-    return;
-  }
-
   if (pMsg->retry == 0) {
     if (pMsg->pTable == NULL) {
+      int32_t sid = taosAllocateId(pVgroup->idPool);
+      if (sid <= 0) {
+        mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId);
+        mgmtCreateVgroup(mgmtCloneQueuedMsg(pMsg), pMsg->pDb);
+        return;
+      }
+
       pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid);
+      if (pMsg->pTable == NULL) {
+        mgmtSendSimpleResp(pMsg->thandle, terrno);
+        return;
+      }
+
       mgmtIncTableRef(pMsg->pTable);
     }
   } else {
@@ -1670,15 +1695,17 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
   pMeta->sid       = htonl(pTable->sid);
   pMeta->precision = pDb->cfg.precision;
   pMeta->tableType = pTable->info.type;
-  strncpy(pMeta->tableId, pTable->info.tableId, tListLen(pTable->info.tableId));
+  strncpy(pMeta->tableId, pTable->info.tableId, strlen(pTable->info.tableId));
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     pMeta->sversion     = htons(pTable->superTable->sversion);
+    pMeta->tversion     = htons(pTable->superTable->tversion);
     pMeta->numOfTags    = (int8_t)pTable->superTable->numOfTags;
     pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns);
     pMeta->contLen      = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable);
   } else {
     pMeta->sversion     = htons(pTable->sversion);
+    pMeta->tversion     = 0;
     pMeta->numOfTags    = 0;
     pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns);
     pMeta->contLen      = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); 
@@ -1720,7 +1747,12 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
   pCreateMsg->igExists = 1;
   pCreateMsg->getMeta = 1;
   pCreateMsg->contLen = htonl(contLen);
-  memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData));
+
+  contLen = sizeof(STagData);
+  if (contLen > pMsg->contLen - sizeof(SCMTableInfoMsg)) {
+    contLen = pMsg->contLen - sizeof(SCMTableInfoMsg);
+  }
+  memcpy(pCreateMsg->schema, pInfo->tags, contLen);
 
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
   pMsg->pCont = newMsg->pCont;
@@ -1751,8 +1783,7 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
 }
 
 void mgmtDropAllChildTables(SDbObj *pDropDb) {
-  void *pNode = NULL;
+  void *  pIter = NULL;
-  void *pLastNode = NULL;
   int32_t numOfTables = 0;
   int32_t dbNameLen = strlen(pDropDb->name);
   SChildTableObj *pTable = NULL;
@@ -1760,8 +1791,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
   mPrint("db:%s, all child tables will be dropped from sdb", pDropDb->name);
 
   while (1) {
-    pLastNode = pNode;
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
-    pNode = mgmtGetNextChildTable(pNode, &pTable);
     if (pTable == NULL) break;
 
     if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@@ -1771,26 +1801,25 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
         .pObj = pTable,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfTables++;
     }
     mgmtDecTableRef(pTable);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
 static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
-  void *pNode = NULL;
+  void *  pIter = NULL;
-  void *pLastNode = NULL;
   int32_t numOfTables = 0;
   SChildTableObj *pTable = NULL;
 
   mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables);
 
   while (1) {
-    pLastNode = pNode;
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
-    pNode = mgmtGetNextChildTable(pNode, &pTable);
     if (pTable == NULL) break;
 
     if (pTable->superTable == pStable) {
@@ -1800,13 +1829,14 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
         .pObj = pTable,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfTables++;
     }
 
     mgmtDecTableRef(pTable);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
 }
 
@@ -2077,7 +2107,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
 
   while (numOfRows < rows) {
     mgmtDecTableRef(pTable);
-    pShow->pNode = mgmtGetNextChildTable(pShow->pNode, &pTable);
+    pShow->pIter = mgmtGetNextChildTable(pShow->pIter, &pTable);
     if (pTable == NULL) break;
 
     // not belong to current db

src/mnode/src/mgmtUser.c

@@ -155,8 +155,8 @@ SUserObj *mgmtGetUser(char *name) {
   return (SUserObj *)sdbGetRow(tsUserSdb, name);
 }
 
-void *mgmtGetNextUser(void *pNode, SUserObj **pUser) { 
+void *mgmtGetNextUser(void *pIter, SUserObj **pUser) { 
-  return sdbFetchRow(tsUserSdb, pNode, (void **)pUser); 
+  return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); 
 }
 
 void mgmtIncUserRef(SUserObj *pUser) { 
@@ -300,7 +300,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
   char     *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextUser(pShow->pNode, &pUser);
+    pShow->pIter = mgmtGetNextUser(pShow->pIter, &pUser);
     if (pUser == NULL) break;
     
     cols = 0;
@@ -504,15 +504,13 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
 }
 
 void  mgmtDropAllUsers(SAcctObj *pAcct)  {
-  void *    pNode = NULL;
+  void *    pIter = NULL;
-  void *    pLastNode = NULL;
   int32_t   numOfUsers = 0;
   int32_t   acctNameLen = strlen(pAcct->user);
   SUserObj *pUser = NULL;
 
   while (1) {
-    pLastNode = pNode;
+    pIter = mgmtGetNextUser(pIter, &pUser);
-    pNode = mgmtGetNextUser(pNode, &pUser);
     if (pUser == NULL) break;
 
     if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
@@ -522,13 +520,14 @@ void  mgmtDropAllUsers(SAcctObj *pAcct)  {
         .pObj = pUser,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfUsers++;
     }
 
     mgmtDecUserRef(pUser);
   }
 
+  sdbFreeIter(pIter);
+
   mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);
 }
 

src/mnode/src/mgmtVgroup.c

@@ -288,8 +288,8 @@ SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) {
   return pDb->pHead;
 }
 
-void *mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup) { 
+void *mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup) { 
-  return sdbFetchRow(tsVgroupSdb, pNode, (void **)pVgroup); 
+  return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); 
 }
 
 void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
@@ -371,12 +371,6 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
-  pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "vgroup_status");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
   int32_t maxReplica = 0;
   SVgObj  *pVgroup   = NULL;
   STableObj *pTable = NULL;
@@ -429,10 +423,10 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
 
   if (NULL == pTable) {
     pShow->numOfRows = pDb->numOfVgroups;
-    pShow->pNode = pDb->pHead;
+    pShow->pIter = pDb->pHead;
   } else {
     pShow->numOfRows = 1;
-    pShow->pNode = pVgroup;
+    pShow->pIter = pVgroup;
   }
 
    mgmtDecDbRef(pDb);
@@ -457,9 +451,9 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
   }
 
   while (numOfRows < rows) {
-    pVgroup = (SVgObj *) pShow->pNode;
+    pVgroup = (SVgObj *) pShow->pIter;
     if (pVgroup == NULL) break;
-    pShow->pNode = (void *) pVgroup->next;
+    pShow->pIter = (void *) pVgroup->next;
 
     cols = 0;
 
@@ -471,11 +465,6 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
     *(int32_t *) pWrite = pVgroup->numOfTables;
     cols++;
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    char* status = pVgroup->status? "updating" : "ready";
-    STR_TO_VARSTR(pWrite, status);
-    cols++;
-
     for (int32_t i = 0; i < maxReplica; ++i) {
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId;
@@ -489,8 +478,8 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
         cols++;
 
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-        status = mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role);
+        char *role = mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role);
-        STR_TO_VARSTR(pWrite, status);
+        STR_TO_VARSTR(pWrite, role);
         cols++;
       } else {
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -666,7 +655,6 @@ static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) {
 }
 
 void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
-  mTrace("vgId:%d, send drop vnode msg, ahandle:%p", vgId, ahandle);
   SMDDropVnodeMsg *pDrop = mgmtBuildDropVnodeMsg(vgId);
   SRpcMsg rpcMsg = {
       .handle  = ahandle,
@@ -682,6 +670,7 @@ static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
   mTrace("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
+    mTrace("vgId:%d, send drop vnode msg to dnode:%d, ahandle:%p", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, ahandle);
     mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle);
   }
 }
@@ -749,14 +738,12 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
 }
 
 void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
-  void *  pNode = NULL;
+  void *  pIter = NULL;
-  void *  pLastNode = NULL;
   SVgObj *pVgroup = NULL;
   int32_t numOfVgroups = 0;
 
   while (1) {
-    pLastNode = pNode;
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
-    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
@@ -766,24 +753,23 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
         .pObj = pVgroup,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfVgroups++;
       continue;
     }
     mgmtDecVgroupRef(pVgroup);
   }
+
+  sdbFreeIter(pIter);
 }
 
 void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
-  void *pNode = NULL;
+  void *  pIter = NULL;
-  void *pLastNode = NULL;
   int32_t numOfVgroups = 0;
   SVgObj *pVgroup = NULL;
 
   mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name);
   while (1) {
-    pLastNode = pNode;
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
-    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->pDb == pDropDb) {
@@ -793,7 +779,6 @@ void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
         .pObj = pVgroup,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfVgroups++;
 
       if (sendMsg) {
@@ -804,5 +789,7 @@ void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
     mgmtDecVgroupRef(pVgroup);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
 }

src/os/linux/inc/os.h

@@ -253,6 +253,17 @@ void taosBlockSIGPIPE();
 #define BUILDIN_CLZ(val) __builtin_clz(val)
 #define BUILDIN_CTZ(val) __builtin_ctz(val)
 
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/os/linux/src/linuxPlatform.c

@@ -157,7 +157,7 @@ void *taosProcessAlarmSignal(void *tharg) {
   void (*callback)(int) = tharg;
 
   static timer_t         timerId;
-  struct sigevent sevent = {0};
+  struct sigevent sevent = {{0}};
 
   #ifdef _ALPINE
     sevent.sigev_notify = SIGEV_THREAD;

src/plugins/http/src/httpHandle.c

@@ -19,7 +19,6 @@
 #include "tglobal.h"
 #include "tsocket.h"
 #include "ttimer.h"
-#include "shash.h"
 #include "http.h"
 #include "httpLog.h"
 #include "httpCode.h"

src/plugins/http/src/httpSession.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "shash.h"
+#include "hash.h"
 #include "taos.h"
 #include "ttime.h"
 #include "ttimer.h"
@@ -25,7 +25,6 @@
 #include "httpHandle.h"
 #include "httpResp.h"
 
-
 void httpAccessSession(HttpContext *pContext) {
   HttpServer *server = pContext->pThread->pServer;
   pthread_mutex_lock(&server->serverMutex);
@@ -50,8 +49,11 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
   session.taos = taos;
   session.expire = (int)taosGetTimestampSec() + server->sessionExpire;
   session.access = 1;
-  snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+  int sessionIdLen = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
-  pContext->session = (HttpSession *)taosAddStrHash(server->pSessionHash, session.id, (char *)(&session));
+
+  taosHashPut(server->pSessionHash, session.id, sessionIdLen, (char *)(&session), sizeof(HttpSession));
+  pContext->session = taosHashGet(server->pSessionHash, session.id, sessionIdLen);
+
   if (pContext->session == NULL) {
     httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user,
               httpMsg[HTTP_SESSION_FULL]);
@@ -71,9 +73,9 @@ void httpFetchSessionImp(HttpContext *pContext) {
   pthread_mutex_lock(&server->serverMutex);
 
   char sessionId[HTTP_SESSION_ID_LEN];
-  snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+  int sessonIdLen = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
 
-  pContext->session = (HttpSession *)taosGetStrHashData(server->pSessionHash, sessionId);
+  pContext->session = taosHashGet(server->pSessionHash, sessionId, sessonIdLen);
   if (pContext->session != NULL && pContext->session == pContext->session->signature) {
     pContext->session->access++;
     httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, access:%d, expire:%d",
@@ -120,8 +122,7 @@ void httpRestoreSession(HttpContext *pContext) {
   pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpResetSession(char *session) {
+void httpResetSession(HttpSession *pSession) {
-  HttpSession *pSession = (HttpSession *)session;
   httpTrace("close session:%p:%p", pSession, pSession->taos);
   if (pSession->taos != NULL) {
     taos_close(pSession->taos);
@@ -131,15 +132,20 @@ void httpResetSession(char *session) {
 }
 
 void httpRemoveAllSessions(HttpServer *pServer) {
-  if (pServer->pSessionHash != NULL) {
+  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
-    taosCleanUpStrHashWithFp(pServer->pSessionHash, httpResetSession);
+
-    pServer->pSessionHash = NULL;
+  while (taosHashIterNext(pIter)) {
+    HttpSession *pSession = taosHashIterGet(pIter);
+    if (pSession == NULL) continue;
+    httpResetSession(pSession);
   }
+
+  taosHashDestroyIter(pIter);
 }
 
 bool httpInitAllSessions(HttpServer *pServer) {
   if (pServer->pSessionHash == NULL) {
-    pServer->pSessionHash = taosInitStrHash(100, sizeof(HttpSession), taosHashStringStep1);
+    pServer->pSessionHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true);
   }
   if (pServer->pSessionHash == NULL) {
     httpError("http init session pool failed");
@@ -152,46 +158,41 @@ bool httpInitAllSessions(HttpServer *pServer) {
   return true;
 }
 
-int httpSessionExpired(char *session) {
+bool httpSessionExpired(HttpSession *pSession) {
-  HttpSession *pSession = (HttpSession *)session;
+  time_t cur = taosGetTimestampSec();
-  time_t       cur = taosGetTimestampSec();
 
   if (pSession->taos != NULL) {
     if (pSession->expire > cur) {
-      return 0;  // un-expired, so return false
+      return false;  // un-expired, so return false
     }
     if (pSession->access > 0) {
       httpTrace("session:%p:%p is expired, but still access:%d", pSession, pSession->taos,
                 pSession->access);
-      return 0;  // still used, so return false
+      return false;  // still used, so return false
     }
     httpTrace("need close session:%p:%p for it expired, cur:%d, expire:%d, invertal:%d",
               pSession, pSession->taos, cur, pSession->expire, cur - pSession->expire);
   }
 
-  return 1;
+  return true;
 }
 
 void httpRemoveExpireSessions(HttpServer *pServer) {  
-  int expiredNum = 0;
+  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
-  do {
+
+  while (taosHashIterNext(pIter)) {
+    HttpSession *pSession = taosHashIterGet(pIter);
+    if (pSession == NULL) continue;
+
     pthread_mutex_lock(&pServer->serverMutex);
-
+    if (httpSessionExpired(pSession)) {
-    HttpSession *pSession = (HttpSession *)taosVisitStrHashWithFp(pServer->pSessionHash, httpSessionExpired);
+      httpResetSession(pSession);
-    if (pSession == NULL) {
+      taosHashRemove(pServer->pSessionHash, pSession->id, strlen(pSession->id));
-      pthread_mutex_unlock(&pServer->serverMutex);
-      break;
     }
-
-    httpResetSession((char *)pSession);
-    taosDeleteStrHashNode(pServer->pSessionHash, pSession->id, pSession);
-
     pthread_mutex_unlock(&pServer->serverMutex);
+  }
 
-    if (++expiredNum > 10) {
+  taosHashDestroyIter(pIter);
-      break;
-    }
-  } while (true);
 }
 
 void httpProcessSessionExpire(void *handle, void *tmrId) {

src/plugins/http/src/httpSystem.c

@@ -15,7 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "shash.h"
 #include "taos.h"
 #include "tglobal.h"
 #include "tsocket.h"

src/plugins/http/src/httpUtil.c

@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tmd5.h"
-#include "shash.h"
 #include "taos.h"
 #include "http.h"
 #include "httpLog.h"

src/plugins/http/src/tgHandle.c

@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tglobal.h"
-#include "shash.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tgHandle.h"

src/plugins/monitor/src/monitorMain.c

@@ -68,7 +68,7 @@ typedef enum {
 typedef struct {
   void * conn;
   void * timer;
-  char   ep[TSDB_FQDN_LEN];
+  char   ep[TSDB_EP_LEN];
   int8_t cmdIndex;
   int8_t state;
   char   sql[SQL_LENGTH];
@@ -109,6 +109,11 @@ static void monitorStartSystemRetry() {
 }
 
 static void monitorInitConn(void *para, void *unused) {
+  if (dnodeGetDnodeId() <= 0) {
+    monitorStartSystemRetry();
+    return;
+  }
+  
   monitorPrint("starting to initialize monitor service ..");
   tsMonitorConn.state = MONITOR_STATE_INITIALIZING;
 

src/query/inc/qsqltype.h

@@ -1,96 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_QSQLCMD_H
-#define TDENGINE_QSQLCMD_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-enum _sql_type {
-  TSDB_SQL_SELECT = 1,
-  TSDB_SQL_FETCH,
-  TSDB_SQL_INSERT,
-
-  TSDB_SQL_MGMT,  // the SQL below is for mgmt node
-  TSDB_SQL_CREATE_DB,
-  TSDB_SQL_CREATE_TABLE,
-  TSDB_SQL_DROP_DB,
-  TSDB_SQL_DROP_TABLE,
-  TSDB_SQL_CREATE_ACCT,
-  TSDB_SQL_CREATE_USER,  // 10
-  TSDB_SQL_DROP_ACCT,
-  TSDB_SQL_DROP_USER,
-  TSDB_SQL_ALTER_USER,
-  TSDB_SQL_ALTER_ACCT,
-  TSDB_SQL_ALTER_TABLE,
-  TSDB_SQL_ALTER_DB,
-  TSDB_SQL_CREATE_MNODE,
-  TSDB_SQL_DROP_MNODE,
-  TSDB_SQL_CREATE_DNODE,
-  TSDB_SQL_DROP_DNODE,  // 20
-  TSDB_SQL_CFG_DNODE,
-  TSDB_SQL_CFG_MNODE,
-  TSDB_SQL_SHOW,
-  TSDB_SQL_RETRIEVE,
-  TSDB_SQL_KILL_QUERY,
-  TSDB_SQL_KILL_STREAM,
-  TSDB_SQL_KILL_CONNECTION,
-
-  TSDB_SQL_READ,  // SQL below is for read operation
-  TSDB_SQL_CONNECT,
-  TSDB_SQL_USE_DB,  // 30
-  TSDB_SQL_META,
-  TSDB_SQL_STABLEVGROUP,
-  TSDB_SQL_MULTI_META,
-  TSDB_SQL_HB,
-
-  TSDB_SQL_LOCAL,  // SQL below for client local
-  TSDB_SQL_DESCRIBE_TABLE,
-  TSDB_SQL_RETRIEVE_LOCALMERGE,
-  TSDB_SQL_TABLE_JOIN_RETRIEVE,
-
-  /*
-   * build empty result instead of accessing dnode to fetch result
-   * reset the client cache
-   */
-  TSDB_SQL_RETRIEVE_EMPTY_RESULT,
-
-  TSDB_SQL_RESET_CACHE,    // 40
-  TSDB_SQL_SERV_STATUS,
-  TSDB_SQL_CURRENT_DB,
-  TSDB_SQL_SERV_VERSION,
-  TSDB_SQL_CLI_VERSION,
-  TSDB_SQL_CURRENT_USER,
-  TSDB_SQL_CFG_LOCAL,
-
-  TSDB_SQL_MAX  // 47
-};
-
-
-// create table operation type
-enum TSQL_TYPE {
-  TSQL_CREATE_TABLE = 0x1,
-  TSQL_CREATE_STABLE = 0x2,
-  TSQL_CREATE_TABLE_FROM_STABLE = 0x3,
-  TSQL_CREATE_STREAM = 0x4,
-};
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_QSQLCMD_H

src/query/src/tvariant.c

@@ -17,7 +17,7 @@
 #include "hash.h"
 #include "hashfunc.h"
 #include "os.h"
-#include "shash.h"
+#include "hash.h"
 #include "taos.h"
 #include "taosdef.h"
 #include "tstoken.h"

src/rpc/inc/rpcHead.h

@@ -49,6 +49,7 @@ typedef struct {
   char     encrypt:3; // encrypt algorithm, 0: no encryption
   uint16_t tranId;    // transcation ID
   uint32_t linkUid;   // for unique connection ID assigned by client
+  uint64_t ahandle;   // ahandle assigned by client 
   uint32_t sourceId;  // source ID, an index for connection list  
   uint32_t destId;    // destination ID, an index for connection list
   uint32_t destIp;    // destination IP address, for NAT scenario

src/rpc/src/rpcCache.c

@@ -146,7 +146,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in
   rpcUnlockCache(pCache->lockedBy+hash);
 
   pCache->total++;
-  tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]);
+  // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]);
 
   return;
 }
@@ -200,9 +200,9 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy
   rpcUnlockCache(pCache->lockedBy+hash);
 
   if (pData) {
-    tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]);
+    //tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]);
   } else {
-    tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]);
+    //tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]);
   }
 
   return pData;
@@ -240,8 +240,8 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
     pNext = pNode->next;
     pCache->total--;
     pCache->count[hash]--;
-    tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode,
+    //tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode,
-             pCache->count[hash]);
+    //         pCache->count[hash]);
     taosMemPoolFree(pCache->connHashMemPool, (char *)pNode);
     pNode = pNext;
   }

src/rpc/src/rpcHaship.c

@@ -1,167 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "os.h"
-#include "tmempool.h"
-#include "rpcLog.h"
-
-typedef struct SIpHash {
-  uint32_t           ip;
-  uint16_t           port;
-  int                hash;
-  struct SIpHash    *prev;
-  struct SIpHash    *next;
-  void              *data;
-} SIpHash;
-
-typedef struct {
-  SIpHash **ipHashList;
-  mpool_h   ipHashMemPool;
-  int       maxSessions;
-} SHashObj;
-
-int rpcHashIp(void *handle, uint32_t ip, uint16_t port) {
-  SHashObj *pObj = (SHashObj *)handle;
-  int       hash = 0;
-
-  hash = (int)(ip >> 16);
-  hash += (unsigned short)(ip & 0xFFFF);
-  hash += port;
-
-  hash = hash % pObj->maxSessions;
-
-  return hash;
-}
-
-void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) {
-  int       hash;
-  SIpHash  *pNode;
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = rpcHashIp(pObj, ip, port);
-  pNode = (SIpHash *)taosMemPoolMalloc(pObj->ipHashMemPool);
-  pNode->ip = ip;
-  pNode->port = port;
-  pNode->data = data;
-  pNode->prev = 0;
-  pNode->next = pObj->ipHashList[hash];
-  pNode->hash = hash;
-
-  if (pObj->ipHashList[hash] != 0) (pObj->ipHashList[hash])->prev = pNode;
-  pObj->ipHashList[hash] = pNode;
-
-  return pObj;
-}
-
-void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) {
-  int       hash;
-  SIpHash  *pNode;
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  hash = rpcHashIp(pObj, ip, port);
-
-  pNode = pObj->ipHashList[hash];
-  while (pNode) {
-    if (pNode->ip == ip && pNode->port == port) break;
-
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    if (pNode->prev) {
-      pNode->prev->next = pNode->next;
-    } else {
-      pObj->ipHashList[hash] = pNode->next;
-    }
-
-    if (pNode->next) {
-      pNode->next->prev = pNode->prev;
-    }
-
-    taosMemPoolFree(pObj->ipHashMemPool, (char *)pNode);
-  }
-}
-
-void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port) {
-  int       hash;
-  SIpHash  *pNode;
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = rpcHashIp(pObj, ip, port);
-  pNode = pObj->ipHashList[hash];
-
-  while (pNode) {
-    if (pNode->ip == ip && pNode->port == port) {
-      break;
-    }
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    return pNode->data;
-  }
-  return NULL;
-}
-
-void *rpcOpenIpHash(int maxSessions) {
-  SIpHash **ipHashList;
-  mpool_h   ipHashMemPool;
-  SHashObj *pObj;
-
-  ipHashMemPool = taosMemPoolInit(maxSessions, sizeof(SIpHash));
-  if (ipHashMemPool == 0) return NULL;
-
-  ipHashList = calloc(sizeof(SIpHash *), (size_t)maxSessions);
-  if (ipHashList == 0) {
-    taosMemPoolCleanUp(ipHashMemPool);
-    return NULL;
-  }
-
-  pObj = malloc(sizeof(SHashObj));
-  if (pObj == NULL) {
-    taosMemPoolCleanUp(ipHashMemPool);
-    free(ipHashList);
-    return NULL;
-  }
-
-  pObj->maxSessions = maxSessions;
-  pObj->ipHashMemPool = ipHashMemPool;
-  pObj->ipHashList = ipHashList;
-
-  return pObj;
-}
-
-void rpcCloseIpHash(void *handle) {
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  if (pObj->ipHashMemPool) taosMemPoolCleanUp(pObj->ipHashMemPool);
-
-  if (pObj->ipHashList) free(pObj->ipHashList);
-
-  memset(pObj, 0, sizeof(SHashObj));
-  free(pObj);
-}

src/rpc/src/rpcMain.c

@@ -87,6 +87,7 @@ typedef struct {
 } SRpcReqContext;
 
 typedef struct SRpcConn {
+  char      info[50];// debug info: label + pConn + ahandle
   int       sid;     // session ID
   uint32_t  ownId;   // own link ID
   uint32_t  peerId;  // peer link ID
@@ -275,7 +276,7 @@ void *rpcOpen(const SRpcInit *pInit) {
     return NULL;
   }
 
-  tTrace("%s RPC is openned, numOfThreads:%d", pRpc->label, pRpc->numOfThreads);
+  tTrace("%s rpc is openned, threads:%d sessions:%d", pRpc->label, pRpc->numOfThreads, pInit->sessions);
 
   return pRpc;
 }
@@ -299,7 +300,7 @@ void rpcClose(void *param) {
 
   tfree(pRpc->connList);
   pthread_mutex_destroy(&pRpc->mutex);
-  tTrace("%s RPC is closed", pRpc->label);
+  tTrace("%s rpc is closed", pRpc->label);
   tfree(pRpc);
 }
 
@@ -361,9 +362,10 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
   // connection type is application specific. 
   // for TDengine, all the query, show commands shall have TCP connection
   char type = pMsg->msgType;
-  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH ||
+  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE
-      type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META ||
+    || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP
-      type == TSDB_MSG_TYPE_CM_SHOW )
+    || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META
+    || type == TSDB_MSG_TYPE_CM_SHOW )
     pContext->connType = RPC_CONN_TCPC;
   
   rpcSendReqToServer(pRpc, pContext);
@@ -374,8 +376,6 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
 void rpcSendResponse(const SRpcMsg *pRsp) {
   int        msgLen = 0;
   SRpcConn  *pConn = (SRpcConn *)pRsp->handle;
-  SRpcInfo  *pRpc = pConn->pRpc;
-
   SRpcMsg    rpcMsg = *pRsp;
   SRpcMsg   *pMsg = &rpcMsg;
 
@@ -393,7 +393,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
   rpcLockConn(pConn);
 
   if ( pConn->inType == 0 || pConn->user[0] == 0 ) {
-    tTrace("%s %p, connection is already released, rsp wont be sent", pRpc->label, pConn);
+    tTrace("%s, connection is already released, rsp wont be sent", pConn->info);
     rpcUnlockConn(pConn);
     return;
   }
@@ -409,6 +409,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
   pHead->linkUid = pConn->linkUid;
   pHead->port = htons(pConn->localPort);
   pHead->code = htonl(pMsg->code);
+  pHead->ahandle = (uint64_t) pConn->ahandle;
  
   // set pConn parameters
   pConn->inType = 0;
@@ -491,6 +492,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort,
   uint32_t peerIp = taosGetIpFromFqdn(peerFqdn);
   if (peerIp == -1) {
     tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); 
+    terrno = TSDB_CODE_APP_ERROR; 
     return NULL;
   }
 
@@ -506,11 +508,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort,
     if (taosOpenConn[connType]) {
       void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle;
       pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort);
-      if (pConn->chandle) {
+      if (pConn->chandle == NULL) {
-        tTrace("%s %p, rpc connection is set up, sid:%d id:%s %s:%hu connType:%d", pRpc->label, 
-                pConn, pConn->sid, pRpc->user, peerFqdn, pConn->peerPort, pConn->connType);
-      } else {
-        tError("%s %p, failed to set up connection to %s:%hu", pRpc->label, pConn, peerFqdn, pConn->peerPort);
         terrno = TSDB_CODE_NETWORK_UNAVAIL;
         rpcCloseConn(pConn);
         pConn = NULL;
@@ -557,7 +555,7 @@ static void rpcCloseConn(void *thandle) {
   taosFreeId(pRpc->idPool, pConn->sid);
   pConn->pContext = NULL;
 
-  tTrace("%s %p, rpc connection is closed", pRpc->label, pConn);
+  tTrace("%s, rpc connection is closed", pConn->info);
 
   rpcUnlockConn(pConn);
 }
@@ -619,7 +617,6 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
       }
 
       if (terrno != 0) {
-        tWarn("%s %p, user not there or server not ready", pRpc->label, pConn);
         taosFreeId(pRpc->idPool, sid);   // sid shall be released
         pConn = NULL;
       }
@@ -634,8 +631,6 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
     }
   
     taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
-    tTrace("%s %p, rpc connection is allocated, sid:%d id:%s port:%u", 
-           pRpc->label, pConn, sid, pConn->user, pConn->localPort);
   }
 
   return pConn;
@@ -660,7 +655,6 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
 
   if (pConn) {
     if (pConn->linkUid != pHead->linkUid) {
-      tTrace("%s %p, linkUid:0x%x not matched, received:0x%x", pRpc->label, pConn, pConn->linkUid, pHead->linkUid);
       terrno = TSDB_CODE_MISMATCHED_METER_ID;
       pConn = NULL;
     }
@@ -677,21 +671,25 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
   pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType);
   if ( pConn == NULL || pConn->user[0] == 0) {
     pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType);
+  } 
+
+  if (pConn) {
+    pConn->ahandle = pContext->ahandle;
+    sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle);
   } else {
-    tTrace("%s %p, connection is retrieved from cache", pRpc->label, pConn);
+    tError("%s %p, failed to set up connection(%s)", pRpc->label, pContext->ahandle, tstrerror(terrno));
   }
 
   return pConn;
 }
 
 static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) {
-    SRpcInfo *pRpc= pConn->pRpc;
 
     if (pConn->peerId == 0) {
       pConn->peerId = pHead->sourceId;
     } else {
       if (pConn->peerId != pHead->sourceId) {
-        tTrace("%s %p, source Id is changed, old:0x%08x new:0x%08x", pRpc->label, pConn, 
+        tTrace("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, 
                pConn->peerId, pHead->sourceId);
         return TSDB_CODE_INVALID_VALUE;
       }
@@ -700,17 +698,16 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) {
     if (pConn->inTranId == pHead->tranId) {
       if (pConn->inType == pHead->msgType) {
         if (pHead->code == 0) {
-          tTrace("%s %p, %s is retransmitted", pRpc->label, pConn, taosMsg[pHead->msgType]);
+          tTrace("%s, %s is retransmitted", pConn->info, taosMsg[pHead->msgType]);
           rpcSendQuickRsp(pConn, TSDB_CODE_ACTION_IN_PROGRESS);
         } else {
           // do nothing, it is heart beat from client
         }
       } else if (pConn->inType == 0) {
-        tTrace("%s %p, %s is already processed, tranId:%d", pRpc->label, pConn, 
+        tTrace("%s, %s is already processed, tranId:%d", pConn->info, taosMsg[pHead->msgType], pConn->inTranId);
-                taosMsg[pHead->msgType], pConn->inTranId);
         rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response
       } else {
-        tTrace("%s %p, mismatched message %s and tranId", pRpc->label, pConn, taosMsg[pHead->msgType]);
+        tTrace("%s, mismatched message %s and tranId", pConn->info, taosMsg[pHead->msgType]);
       }
 
       // do not reply any message
@@ -718,7 +715,7 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) {
     }
 
     if (pConn->inType != 0) {
-      tTrace("%s %p, last session is not finished, inTranId:%d tranId:%d", pRpc->label, pConn, 
+      tTrace("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, 
               pConn->inTranId, pHead->tranId);
       return TSDB_CODE_LAST_SESSION_NOT_FINISHED;
     }
@@ -750,7 +747,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) {
 
   if (pHead->code == TSDB_CODE_ACTION_IN_PROGRESS) {
     if (pConn->tretry <= tsRpcMaxRetry) {
-      tTrace("%s %p, peer is still processing the transaction", pRpc->label, pConn);
+      tTrace("%s, peer is still processing the transaction", pConn->info);
       pConn->tretry++;
       rpcSendReqHead(pConn);
       taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer);
@@ -789,7 +786,15 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
   }
 
   pConn = rpcGetConnObj(pRpc, sid, pRecv);
-  if (pConn == NULL) return NULL;
+  if (pConn == NULL) {
+    tError("%s %p, failed to get connection obj(%s)", pRpc->label, pHead->ahandle, tstrerror(terrno)); 
+    return NULL;
+  } else {
+    if (rpcIsReq(pHead->msgType)) {
+      pConn->ahandle = (void *)pHead->ahandle;
+      sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle);
+    }
+  }
 
   rpcLockConn(pConn);
   sid = pConn->sid;
@@ -826,7 +831,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
 static void rpcProcessBrokenLink(SRpcConn *pConn) {
   SRpcInfo *pRpc = pConn->pRpc;
   
-  tTrace("%s %p, link is broken", pRpc->label, pConn);
+  tTrace("%s, link is broken", pConn->info);
   // pConn->chandle = NULL;
 
   if (pConn->outType) {
@@ -837,7 +842,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
 
   if (pConn->inType) {
     // if there are pending request, notify the app
-    tTrace("%s %p, connection is gone, notify the app", pRpc->label, pConn);
+    tTrace("%s, connection is gone, notify the app", pConn->info);
 /*
     SRpcMsg rpcMsg;
     rpcMsg.pCont = NULL;
@@ -872,17 +877,17 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   pConn = rpcProcessMsgHead(pRpc, pRecv);
 
   if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) {
-    tTrace("%s %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
+    tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
-        pRpc->label, pConn, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, 
+        pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, 
         pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
   }
 
   int32_t code = terrno;
   if (code != TSDB_CODE_ALREADY_PROCESSED) {
     if (code != 0) { // parsing error
-      if ( rpcIsReq(pHead->msgType) ) {
+      if (rpcIsReq(pHead->msgType)) {
         rpcSendErrorMsgToPeer(pRecv, code);
-        tTrace("%s %p, %s is sent with error code:%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], code);
+        tTrace("%s %p %p, %s is sent with error code:%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
       } 
     } else { // parsing OK
       rpcProcessIncomingMsg(pConn, pHead);
@@ -898,9 +903,9 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
 
   if (pContext->pRsp) { 
     // for synchronous API
-    tsem_post(pContext->pSem);
     memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet));
     memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg));
+    tsem_post(pContext->pSem);
   } else {
     // for asynchronous API 
     SRpcIpSet *pIpSet = NULL;
@@ -924,6 +929,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
   rpcMsg.pCont = pHead->content;
   rpcMsg.msgType = pHead->msgType;
   rpcMsg.code = pHead->code; 
+  rpcMsg.ahandle = pConn->ahandle;
    
   if ( rpcIsReq(pHead->msgType) ) {
     rpcMsg.handle = pConn;
@@ -948,14 +954,14 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
       pContext->redirect++;
       if (pContext->redirect > TSDB_MAX_REPLICA) {
         pHead->code = TSDB_CODE_NETWORK_UNAVAIL; 
-        tWarn("%s %p, too many redirects, quit", pRpc->label, pConn);
+        tWarn("%s, too many redirects, quit", pConn->info);
       }
     }
 
     if (pHead->code == TSDB_CODE_REDIRECT) {
       pContext->numOfTry = 0;
       memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet));
-      tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps);
+      tTrace("%s, redirect is received, numOfIps:%d", pConn->info, pContext->ipSet.numOfIps);
       for (int i=0; i<pContext->ipSet.numOfIps; ++i) 
         pContext->ipSet.port[i] = htons(pContext->ipSet.port[i]);
       rpcSendReqToServer(pRpc, pContext);
@@ -1061,6 +1067,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
     return;
   }
 
+  pConn->ahandle = pContext->ahandle;
   rpcLockConn(pConn);
 
   // set the message header  
@@ -1074,6 +1081,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
   pHead->destId = pConn->peerId;
   pHead->port = 0;
   pHead->linkUid = pConn->linkUid;
+  pHead->ahandle = (uint64_t)pConn->ahandle;
   if (!pConn->secured) memcpy(pHead->user, pConn->user, tListLen(pHead->user));
 
   // set the connection parameters
@@ -1091,29 +1099,28 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
 
 static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
   int        writtenLen = 0;
-  SRpcInfo  *pRpc = pConn->pRpc;
   SRpcHead  *pHead = (SRpcHead *)msg;
 
   msgLen = rpcAddAuthPart(pConn, msg, msgLen);
 
   if ( rpcIsReq(pHead->msgType)) {
     if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
-      tTrace("%s %p, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
+      tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
-             pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn,
+             pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, 
-             pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
+             msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   } else {
     if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
     if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
-      tTrace( "%s %p, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
+      tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
-          pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, 
+          pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, 
           htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 
+  //tTrace("connection type is: %d", pConn->connType);
   writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle);
 
   if (writtenLen != msgLen) {
-    tError("%s %p, failed to send, dataLen:%d writtenLen:%d, reason:%s", pRpc->label, pConn, 
+    tError("%s, failed to send, msgLen:%d written:%d, reason:%s", pConn->info, msgLen, writtenLen, strerror(errno));
-           msgLen, writtenLen, strerror(errno));
   }
  
   tDump(msg, msgLen);
@@ -1128,7 +1135,7 @@ static void rpcProcessConnError(void *param, void *id) {
     return;
   }
   
-  tTrace("%s connection error happens", pRpc->label);
+  tTrace("%s %p, connection error happens", pRpc->label, pContext->ahandle);
 
   if (pContext->numOfTry >= pContext->ipSet.numOfIps) {
     rpcMsg.msgType = pContext->msgType+1;
@@ -1154,23 +1161,21 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
   rpcLockConn(pConn);
 
   if (pConn->outType && pConn->user[0]) {
-    tTrace("%s %p, expected %s is not received", pRpc->label, pConn, taosMsg[(int)pConn->outType + 1]);
+    tTrace("%s, expected %s is not received", pConn->info, taosMsg[(int)pConn->outType + 1]);
     pConn->pTimer = NULL;
     pConn->retry++;
 
     if (pConn->retry < 4) {
-      tTrace("%s %p, re-send msg:%s to %s:%hu", pRpc->label, pConn, 
+      tTrace("%s, re-send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
-             taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
       rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen);      
       taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer);
     } else {
       // close the connection
-      tTrace("%s %p, failed to send msg:%s to %s:%hu", pRpc->label, pConn,
+      tTrace("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
-              taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
       reportDisc = 1;
     }
   } else {
-    tTrace("%s %p, retry timer not processed", pRpc->label, pConn);
+    tTrace("%s, retry timer not processed", pConn->info);
   }
 
   rpcUnlockConn(pConn);
@@ -1187,10 +1192,10 @@ static void rpcProcessIdleTimer(void *param, void *tmrId) {
   SRpcInfo *pRpc = pConn->pRpc;
 
   if (pConn->user[0]) {
-    tTrace("%s %p, close the connection since no activity", pRpc->label, pConn);
+    tTrace("%s, close the connection since no activity", pConn->info);
     if (pConn->inType && pRpc->cfp) {
       // if there are pending request, notify the app
-      tTrace("%s %p, notify the app, connection is gone", pRpc->label, pConn);
+      tTrace("%s, notify the app, connection is gone", pConn->info);
 /*
       SRpcMsg rpcMsg;
       rpcMsg.pCont = NULL;
@@ -1203,7 +1208,7 @@ static void rpcProcessIdleTimer(void *param, void *tmrId) {
     }
     rpcCloseConn(pConn);
   } else {
-    tTrace("%s %p, idle timer:%p not processed", pRpc->label, pConn, tmrId);
+    tTrace("%s, idle timer:%p not processed", pConn->info, tmrId);
   }
 }
 
@@ -1214,11 +1219,11 @@ static void rpcProcessProgressTimer(void *param, void *tmrId) {
   rpcLockConn(pConn);
 
   if (pConn->inType && pConn->user[0]) {
-    tTrace("%s %p, progress timer expired, send progress", pRpc->label, pConn);
+    tTrace("%s, progress timer expired, send progress", pConn->info);
     rpcSendQuickRsp(pConn, TSDB_CODE_ACTION_IN_PROGRESS);
     taosTmrReset(rpcProcessProgressTimer, tsRpcTimer/2, pConn, pRpc->tmrCtrl, &pConn->pTimer);
   } else {
-    tTrace("%s %p, progress timer:%p not processed", pRpc->label, pConn, tmrId);
+    tTrace("%s, progress timer:%p not processed", pConn->info, tmrId);
   }
 
   rpcUnlockConn(pConn);
@@ -1252,7 +1257,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
     memcpy(pCont + overhead, buf, compLen);
     
     pHead->comp = 1;
-    tTrace("compress rpc msg, before:%d, after:%d", contLen, compLen);
+    //tTrace("compress rpc msg, before:%d, after:%d", contLen, compLen);
     finalLen = compLen + overhead;
   } else {
     finalLen = contLen;
@@ -1286,7 +1291,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
       pNewHead->msgLen = rpcMsgLenFromCont(origLen);
       rpcFreeMsg(pHead); // free the compressed message buffer
       pHead = pNewHead; 
-      tTrace("decompress rpc msg, compLen:%d, after:%d", compLen, contLen);
+      //tTrace("decompress rpc msg, compLen:%d, after:%d", compLen, contLen);
     } else {
       tError("failed to allocate memory to decompress msg, contLen:%d", contLen);
     }
@@ -1343,7 +1348,6 @@ static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen) {
 
 static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) {
   SRpcHead *pHead = (SRpcHead *)msg;
-  SRpcInfo *pRpc = pConn->pRpc;
   int       code = 0;
 
   if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)){
@@ -1371,20 +1375,20 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) {
     delta = (int32_t)htonl(pDigest->timeStamp);
     delta -= (int32_t)taosGetTimestampSec();
     if (abs(delta) > 900) {
-      tWarn("%s %p, time diff:%d is too big, msg discarded", pRpc->label, pConn, delta);
+      tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta);
       code = TSDB_CODE_INVALID_TIME_STAMP;
     } else {
       if (rpcAuthenticateMsg(pHead, msgLen-TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) {
-        tError("%s %p, authentication failed, msg discarded", pRpc->label, pConn);
+        tError("%s, authentication failed, msg discarded", pConn->info);
         code = TSDB_CODE_AUTH_FAILURE;
       } else {
         pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest);
         if ( !rpcIsReq(pHead->msgType) ) pConn->secured = 1;  // link is secured for client
-        tTrace("%s %p, message is authenticated", pRpc->label, pConn);
+        //tTrace("%s, message is authenticated", pConn->info);
       }
     }
   } else {
-    tTrace("%s %p, auth spi:%d not matched with received:%d", pRpc->label, pConn, pConn->spi, pHead->spi);
+    tError("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi);
     code = TSDB_CODE_AUTH_FAILURE;
   }
 

src/rpc/src/rpcUdp.c

@@ -18,9 +18,7 @@
 #include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
-#include "thash.h"
 #include "rpcLog.h"
-#include "rpcHaship.h"
 #include "rpcUdp.h"
 #include "rpcHead.h"
 
@@ -29,8 +27,6 @@
 #define RPC_UDP_BUF_TIME 5  // mseconds
 #define RPC_MAX_UDP_SIZE 65480
 
-int tsUdpDelay = 0;
-
 typedef struct {
   void           *signature;
   int             index;
@@ -39,8 +35,6 @@ typedef struct {
   uint16_t        localPort;  // local port
   char            label[12];  // copy from udpConnSet;
   pthread_t       thread;
-  pthread_mutex_t mutex;
-  void           *tmrCtrl;  // copy from UdpConnSet;
   void           *hash;
   void           *shandle;  // handle passed by upper layer during server initialization
   void           *pSet;
@@ -56,26 +50,11 @@ typedef struct {
   void     *shandle;  // handle passed by upper layer during server initialization
   int       threads;
   char      label[12];
-  void     *tmrCtrl;
   void     *(*fp)(SRecvInfo *pPacket);
   SUdpConn  udpConn[];
 } SUdpConnSet;
 
-typedef struct {
-  void              *signature;
-  uint32_t           ip;    // dest IP
-  uint16_t           port;  // dest Port
-  SUdpConn          *pConn;
-  struct sockaddr_in destAdd;
-  void              *msgHdr;
-  int                totalLen;
-  void              *timer;
-  int                emptyNum;
-} SUdpBuf;
-
 static void *taosRecvUdpData(void *param);
-static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port);
-static void taosProcessUdpBufTimer(void *param, void *tmrId);
 
 void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
   SUdpConn    *pConn;
@@ -95,16 +74,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
   pSet->fp = fp;
   strcpy(pSet->label, label);
 
-  if ( tsUdpDelay ) {
-    char udplabel[12];
-    sprintf(udplabel, "%s.b", label);
-    pSet->tmrCtrl = taosTmrInit(RPC_MAX_UDP_CONNS * threads, 5, 5000, udplabel);
-    if (pSet->tmrCtrl == NULL) {
-      tError("%s failed to initialize tmrCtrl") taosCleanUpUdpConnection(pSet);
-      return NULL;
-    }
-  }
-
   uint16_t ownPort;
   for (int i = 0; i < threads; ++i) {
     pConn = pSet->udpConn + i;
@@ -136,11 +105,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
     pConn->index = i;
     pConn->pSet = pSet;
     pConn->signature = pConn;
-    if (tsUdpDelay) {
-      pConn->hash = rpcOpenIpHash(RPC_MAX_UDP_CONNS);
-      pthread_mutex_init(&pConn->mutex, NULL);
-      pConn->tmrCtrl = pSet->tmrCtrl;
-    }
 
     pthread_attr_t thAttr;
     pthread_attr_init(&thAttr);
@@ -174,10 +138,6 @@ void taosCleanUpUdpConnection(void *handle) {
     free(pConn->buffer);
     pthread_cancel(pConn->thread);
     taosCloseSocket(pConn->fd);
-    if (pConn->hash) {
-      rpcCloseIpHash(pConn->hash);
-      pthread_mutex_destroy(&pConn->mutex);
-    }
   }
 
   for (int i = 0; i < pSet->threads; ++i) {
@@ -186,7 +146,6 @@ void taosCleanUpUdpConnection(void *handle) {
     tTrace("chandle:%p is closed", pConn);
   }
 
-  taosTmrCleanUp(pSet->tmrCtrl);
   tfree(pSet);
 }
 
@@ -206,64 +165,42 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
 static void *taosRecvUdpData(void *param) {
   SUdpConn          *pConn = param;
   struct sockaddr_in sourceAdd;
-  int                dataLen;
+  ssize_t            dataLen;
   unsigned int       addLen;
   uint16_t           port;
-  int                minSize = sizeof(SRpcHead);
   SRecvInfo          recvInfo;
 
   memset(&sourceAdd, 0, sizeof(sourceAdd));
   addLen = sizeof(sourceAdd);
   tTrace("%s UDP thread is created, index:%d", pConn->label, pConn->index);
+  char *msg = pConn->buffer;
 
   while (1) {
     dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
     port = ntohs(sourceAdd.sin_port);
-    tTrace("%s msg is recv from 0x%x:%hu len:%d", pConn->label, sourceAdd.sin_addr.s_addr, port, dataLen);
 
     if (dataLen < sizeof(SRpcHead)) {
       tError("%s recvfrom failed, reason:%s\n", pConn->label, strerror(errno));
       continue;
     }
 
-    int   processedLen = 0, leftLen = 0;
+    char *tmsg = malloc(dataLen + tsRpcOverhead);
-    int   msgLen = 0;
+    if (NULL == tmsg) {
-    int   count = 0;
+      tError("%s failed to allocate memory, size:%d", pConn->label, dataLen);
-    char *msg = pConn->buffer;
+      continue;
-    while (processedLen < dataLen) {
-      leftLen = dataLen - processedLen;
-      SRpcHead *pHead = (SRpcHead *)msg;
-      msgLen = htonl((uint32_t)pHead->msgLen);
-      if (leftLen < minSize || msgLen > leftLen || msgLen < minSize) {
-        tError("%s msg is messed up, dataLen:%d processedLen:%d count:%d msgLen:%d", pConn->label, dataLen,
-               processedLen, count, msgLen);
-        break;
-      }
-
-      char *tmsg = malloc((size_t)msgLen + tsRpcOverhead);
-      if (NULL == tmsg) {
-        tError("%s failed to allocate memory, size:%d", pConn->label, msgLen);
-        break;
-      }
-
-      tmsg += tsRpcOverhead;  // overhead for SRpcReqContext
-      memcpy(tmsg, msg, (size_t)msgLen);
-      recvInfo.msg = tmsg;
-      recvInfo.msgLen = msgLen;
-      recvInfo.ip = sourceAdd.sin_addr.s_addr;
-      recvInfo.port = port;
-      recvInfo.shandle = pConn->shandle;
-      recvInfo.thandle = NULL;
-      recvInfo.chandle = pConn;
-      recvInfo.connType = 0;
-      (*(pConn->processData))(&recvInfo);
-
-      processedLen += msgLen;
-      msg += msgLen;
-      count++;
     }
 
-    // tTrace("%s %d UDP packets are received together", pConn->label, count);
+    tmsg += tsRpcOverhead;  // overhead for SRpcReqContext
+    memcpy(tmsg, msg, dataLen);
+    recvInfo.msg = tmsg;
+    recvInfo.msgLen = dataLen;
+    recvInfo.ip = sourceAdd.sin_addr.s_addr;
+    recvInfo.port = port;
+    recvInfo.shandle = pConn->shandle;
+    recvInfo.thandle = NULL;
+    recvInfo.chandle = pConn;
+    recvInfo.connType = 0;
+    (*(pConn->processData))(&recvInfo);
   }
 
   return NULL;
@@ -271,141 +208,17 @@ static void *taosRecvUdpData(void *param) {
 
 int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) {
   SUdpConn *pConn = (SUdpConn *)chandle;
-  SUdpBuf  *pBuf;
 
   if (pConn == NULL || pConn->signature != pConn) return -1;
 
-  if (pConn->hash == NULL) {
+  struct sockaddr_in destAdd;
-    struct sockaddr_in destAdd;
+  memset(&destAdd, 0, sizeof(destAdd));
-    memset(&destAdd, 0, sizeof(destAdd));
+  destAdd.sin_family = AF_INET;
-    destAdd.sin_family = AF_INET;
+  destAdd.sin_addr.s_addr = ip;
-    destAdd.sin_addr.s_addr = ip;
+  destAdd.sin_port = htons(port);
-    destAdd.sin_port = htons(port);
 
-    //tTrace("%s msg is sent to 0x%x:%hu len:%d ret:%d localPort:%hu chandle:0x%x", pConn->label, destAdd.sin_addr.s_addr,
+  int ret = (int)sendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
-    //       port, dataLen, ret, pConn->localPort, chandle);
-    int ret = (int)sendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
 
-    return ret;
+  return ret;
-  }
-
-  pthread_mutex_lock(&pConn->mutex);
-
-  pBuf = (SUdpBuf *)rpcGetIpHash(pConn->hash, ip, port);
-  if (pBuf == NULL) {
-    pBuf = taosCreateUdpBuf(pConn, ip, port);
-    rpcAddIpHash(pConn->hash, pBuf, ip, port);
-  }
-
-  if ((pBuf->totalLen + dataLen > RPC_MAX_UDP_SIZE) || (taosMsgHdrSize(pBuf->msgHdr) >= RPC_MAX_UDP_PKTS)) {
-    taosTmrReset(taosProcessUdpBufTimer, RPC_UDP_BUF_TIME, pBuf, pConn->tmrCtrl, &pBuf->timer);
-
-    taosSendMsgHdr(pBuf->msgHdr, pConn->fd);
-    pBuf->totalLen = 0;
-  }
-
-  taosSetMsgHdrData(pBuf->msgHdr, data, dataLen);
-
-  pBuf->totalLen += dataLen;
-
-  pthread_mutex_unlock(&pConn->mutex);
-
-  return dataLen;
 }
 
-void taosFreeMsgHdr(void *hdr) {
-  struct msghdr *msgHdr = (struct msghdr *)hdr;
-  free(msgHdr->msg_iov);
-}
-
-int taosMsgHdrSize(void *hdr) {
-  struct msghdr *msgHdr = (struct msghdr *)hdr;
-  return (int)msgHdr->msg_iovlen;
-}
-
-void taosSendMsgHdr(void *hdr, int fd) {
-  struct msghdr *msgHdr = (struct msghdr *)hdr;
-  sendmsg(fd, msgHdr, 0);
-  msgHdr->msg_iovlen = 0;
-}
-
-void taosInitMsgHdr(void **hdr, void *dest, int maxPkts) {
-  struct msghdr *msgHdr = (struct msghdr *)malloc(sizeof(struct msghdr));
-  memset(msgHdr, 0, sizeof(struct msghdr));
-  *hdr = msgHdr;
-  struct sockaddr_in *destAdd = (struct sockaddr_in *)dest;
-
-  msgHdr->msg_name = destAdd;
-  msgHdr->msg_namelen = sizeof(struct sockaddr_in);
-  int size = (int)sizeof(struct iovec) * maxPkts;
-  msgHdr->msg_iov = (struct iovec *)malloc((size_t)size);
-  memset(msgHdr->msg_iov, 0, (size_t)size);
-}
-
-void taosSetMsgHdrData(void *hdr, char *data, int dataLen) {
-  struct msghdr *msgHdr = (struct msghdr *)hdr;
-  msgHdr->msg_iov[msgHdr->msg_iovlen].iov_base = data;
-  msgHdr->msg_iov[msgHdr->msg_iovlen].iov_len = (size_t)dataLen;
-  msgHdr->msg_iovlen++;
-}
-
-void taosRemoveUdpBuf(SUdpBuf *pBuf) {
-  taosTmrStopA(&pBuf->timer);
-  rpcDeleteIpHash(pBuf->pConn->hash, pBuf->ip, pBuf->port);
-
-  // tTrace("%s UDP buffer to:0x%lld:%d is removed", pBuf->pConn->label,
-  // pBuf->ip, pBuf->port);
-
-  pBuf->signature = NULL;
-  taosFreeMsgHdr(pBuf->msgHdr);
-  free(pBuf);
-}
-
-void taosProcessUdpBufTimer(void *param, void *tmrId) {
-  SUdpBuf *pBuf = (SUdpBuf *)param;
-  if (pBuf->signature != param) return;
-  if (pBuf->timer != tmrId) return;
-
-  SUdpConn *pConn = pBuf->pConn;
-
-  pthread_mutex_lock(&pConn->mutex);
-
-  if (taosMsgHdrSize(pBuf->msgHdr) > 0) {
-    taosSendMsgHdr(pBuf->msgHdr, pConn->fd);
-    pBuf->totalLen = 0;
-    pBuf->emptyNum = 0;
-  } else {
-    pBuf->emptyNum++;
-    if (pBuf->emptyNum > 200) {
-      taosRemoveUdpBuf(pBuf);
-      pBuf = NULL;
-    }
-  }
-
-  pthread_mutex_unlock(&pConn->mutex);
-
-  if (pBuf) taosTmrReset(taosProcessUdpBufTimer, RPC_UDP_BUF_TIME, pBuf, pConn->tmrCtrl, &pBuf->timer);
-}
-
-static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port) {
-  SUdpBuf *pBuf = (SUdpBuf *)malloc(sizeof(SUdpBuf));
-  memset(pBuf, 0, sizeof(SUdpBuf));
-
-  pBuf->ip = ip;
-  pBuf->port = port;
-  pBuf->pConn = pConn;
-
-  pBuf->destAdd.sin_family = AF_INET;
-  pBuf->destAdd.sin_addr.s_addr = ip;
-  pBuf->destAdd.sin_port = (uint16_t)htons(port);
-  taosInitMsgHdr(&(pBuf->msgHdr), &(pBuf->destAdd), RPC_MAX_UDP_PKTS);
-  pBuf->signature = pBuf;
-  taosTmrReset(taosProcessUdpBufTimer, RPC_UDP_BUF_TIME, pBuf, pConn->tmrCtrl, &pBuf->timer);
-
-  // tTrace("%s UDP buffer to:0x%lld:%d is created", pBuf->pConn->label,
-  // pBuf->ip, pBuf->port);
-
-  return pBuf;
-}
-
-

src/tsdb/inc/tsdbMain.h

@@ -29,23 +29,23 @@ extern "C" {
 
 extern int tsdbDebugFlag;
 
-#define tsdbError(...)                                       \
+#define tsdbError(...)                                      \
-  if (tsdbDebugFlag & DEBUG_ERROR) {                         \
+  if (tsdbDebugFlag & DEBUG_ERROR) {                        \
-    taosPrintLog("ERROR TSDB ", tsdbDebugFlag, __VA_ARGS__); \
+    taosPrintLog("ERROR TDB ", tsdbDebugFlag, __VA_ARGS__); \
   }
-#define tsdbWarn(...)                                       \
+#define tsdbWarn(...)                                      \
-  if (tsdbDebugFlag & DEBUG_WARN) {                         \
+  if (tsdbDebugFlag & DEBUG_WARN) {                        \
-    taosPrintLog("WARN TSDB ", tsdbDebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN TDB ", tsdbDebugFlag, __VA_ARGS__); \
   }
-#define tsdbTrace(...)                                 \
+#define tsdbTrace(...)                                \
-  if (tsdbDebugFlag & DEBUG_TRACE) {                   \
+  if (tsdbDebugFlag & DEBUG_TRACE) {                  \
-    taosPrintLog("TSDB ", tsdbDebugFlag, __VA_ARGS__); \
+    taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); \
   }
 #define tsdbPrint(...) \
-  { taosPrintLog("TSDB ", 255, __VA_ARGS__); }
+  { taosPrintLog("TDB ", 255, __VA_ARGS__); }
 
 // ------------------------------ TSDB META FILE INTERFACES ------------------------------
-#define TSDB_META_FILE_NAME "META"
+#define TSDB_META_FILE_NAME "meta"
 #define TSDB_META_HASH_FRACTION 1.1
 
 typedef int (*iterFunc)(void *, void *cont, int contLen);
@@ -63,9 +63,9 @@ typedef struct {
 } SMetaFile;
 
 SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, afterFunc aFunc, void *appH);
-int32_t    tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
+int32_t    tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen);
-int32_t    tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid);
+int32_t    tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid);
-int32_t    tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
+int32_t    tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen);
 void       tsdbCloseMetaFile(SMetaFile *mfh);
 
 // ------------------------------ TSDB META INTERFACES ------------------------------
@@ -82,7 +82,7 @@ typedef struct {
 typedef struct STable {
   int8_t         type;
   STableId       tableId;
-  int64_t        superUid;  // Super table UID
+  uint64_t       superUid;  // Super table UID
   int32_t        sversion;
   STSchema *     schema;
   STSchema *     tagSchema;
@@ -153,7 +153,7 @@ STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo);
 
 STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);
 // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable);
-STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid);
+STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid);
 char *getTSTupleKey(const void * data);
 
 typedef struct {
@@ -210,16 +210,21 @@ typedef enum {
 extern const char *tsdbFileSuffix[];
 
 typedef struct {
-  int64_t size;      // total size of the file
+  uint32_t offset;
-  int64_t tombSize;  // unused file size
+  uint32_t len;
-  int32_t totalBlocks;
+  uint64_t size;      // total size of the file
-  int32_t totalSubBlocks;
+  uint64_t tombSize;  // unused file size
-} SFileInfo;
+  uint32_t totalBlocks;
+  uint32_t totalSubBlocks;
+} STsdbFileInfo;
+
+void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo);
+void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo);
 
 typedef struct {
   int       fd;
   char      fname[128];
-  SFileInfo info;
+  STsdbFileInfo info;
 } SFile;
 
 #define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1)
@@ -242,8 +247,7 @@ typedef struct {
 
 STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg);
 void        tsdbCloseFileH(STsdbFileH *pFileH);
-int         tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader,
+int         tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile);
-                           int toClose);
 SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
 int         tsdbOpenFile(SFile *pFile, int oflag);
 int         tsdbCloseFile(SFile *pFile);
@@ -266,15 +270,18 @@ void        tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid);
 SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter);
 
 typedef struct {
-  int32_t len;
+  uint32_t len;
-  int32_t offset;
+  uint32_t offset;
-  int32_t padding; // For padding purpose
+  uint32_t padding;  // For padding purpose
-  int32_t hasLast : 1;
+  uint32_t hasLast : 2;
-  int32_t numOfBlocks : 31;
+  uint32_t numOfBlocks : 30;
-  int64_t uid;
+  uint64_t uid;
-  TSKEY   maxKey;
+  TSKEY    maxKey;
 } SCompIdx; /* sizeof(SCompIdx) = 28 */
 
+void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx);
+void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx);
+
 /**
  * if numOfSubBlocks == 0, then the SCompBlock is a sub-block
  * if numOfSubBlocks >= 1, then the SCompBlock is a super-block
@@ -304,7 +311,7 @@ typedef struct {
 typedef struct {
   int32_t    delimiter;  // For recovery usage
   int32_t    checksum;   // TODO: decide if checksum logic in this file or make it one API
-  int64_t    uid;
+  uint64_t   uid;
   SCompBlock blocks[];
 } SCompInfo;
 
@@ -325,13 +332,20 @@ typedef struct {
   int16_t len;    // Column length // TODO: int16_t is not enough
   int32_t type : 8;
   int32_t offset : 24;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+  char    padding[2];
 } SCompCol;
 
 // TODO: Take recover into account
 typedef struct {
   int32_t  delimiter;  // For recovery usage
   int32_t  numOfCols;  // For recovery usage
-  int64_t  uid;        // For recovery usage
+  uint64_t uid;        // For recovery usage
   SCompCol cols[];
 } SCompData;
 
@@ -343,7 +357,7 @@ SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid);
 void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey);
 
 // TSDB repository definition
-typedef struct _tsdb_repo {
+typedef struct STsdbRepo {
   char *rootDir;
   // TSDB configuration
   STsdbCfg config;
@@ -427,9 +441,9 @@ typedef struct {
 } SHelperFile;
 
 typedef struct {
-  int64_t uid;
+  uint64_t uid;
-  int32_t tid;
+  int32_t  tid;
-  int32_t sversion;
+  int32_t  sversion;
 } SHelperTable;
 
 typedef struct {
@@ -451,7 +465,7 @@ typedef struct {
   SCompData *pCompData;
   SDataCols *pDataCols[2];
 
-  void *blockBuffer;  // Buffer to hold the whole data block
+  void *pBuffer;  // Buffer to hold the whole data block
   void *compBuffer;   // Buffer for temperary compress/decompress purpose
 } SRWHelper;
 
@@ -498,6 +512,7 @@ void    tsdbFitRetention(STsdbRepo *pRepo);
 int     tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
 void    tsdbAdjustCacheBlocks(STsdbCache *pCache);
 int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
+int     tsdbUpdateFileHeader(SFile *pFile, uint32_t version);
 
 #ifdef __cplusplus
 }

src/tsdb/src/tsdbCache.c

@@ -154,7 +154,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
     for (int i = 0; i < blocksToAdd; i++) {
       if (tsdbAddCacheBlockToPool(pCache) < 0) {
         tsdbUnLockRepo((TsdbRepoT *)pRepo);
-        tsdbError("tsdbId %d: failed to add cache block to cache pool", pRepo->config.tsdbId);
+        tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId);
         return -1;
       }
     }
@@ -164,7 +164,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
   }
 
   tsdbUnLockRepo((TsdbRepoT *)pRepo);
-  tsdbTrace("vgId: %d tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);
+  tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);
   return 0;
 }
 

src/tsdb/src/tsdbFile.c

@@ -37,8 +37,6 @@ const char *tsdbFileSuffix[] = {
 
 static int compFGroupKey(const void *key, const void *fgroup);
 static int compFGroup(const void *arg1, const void *arg2);
-static int tsdbWriteFileHead(SFile *pFile);
-static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables);
 static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid);
 
 STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg) {
@@ -84,11 +82,23 @@ void tsdbCloseFileH(STsdbFileH *pFileH) {
 }
 
 static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile) {
+  uint32_t version;
+  char buf[512] = "\0";
+
   tsdbGetFileName(dataDir, fid, suffix, pFile->fname);
   if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1;
   pFile->fd = -1;
-  // TODO: recover the file info
+  if (tsdbOpenFile(pFile, O_RDONLY) < 0) return -1;
-  // pFile->info = {0};
+
+  if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1;
+  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) return -1;
+
+  void *pBuf = buf;
+  pBuf = taosDecodeFixed32(pBuf, &version);
+  pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info));
+
+  tsdbCloseFile(pFile);
+
   return 0;
 }
 
@@ -121,8 +131,7 @@ SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int max
   if (pGroup == NULL) {  // if not exists, create one
     pFGroup->fileId = fid;
     for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
-      if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]),
+      if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], &(pFGroup->files[type])) < 0)
-                         type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0)
         goto _err;
     }
 
@@ -286,41 +295,6 @@ static int compFGroup(const void *arg1, const void *arg2) {
   return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId;
 }
 
-static int tsdbWriteFileHead(SFile *pFile) {
-  char head[TSDB_FILE_HEAD_SIZE] = "\0";
-
-  pFile->info.size += TSDB_FILE_HEAD_SIZE;
-
-  // TODO: write version and File statistic to the head
-  lseek(pFile->fd, 0, SEEK_SET);
-  if (write(pFile->fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1;
-
-  return 0;
-}
-
-static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
-  int   size = sizeof(SCompIdx) * maxTables + sizeof(TSCKSUM);
-  void *buf = calloc(1, size);
-  if (buf == NULL) return -1;
-
-  if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {
-    free(buf);
-    return -1;
-  }
-
-  taosCalcChecksumAppend(0, (uint8_t *)buf, size);
-
-  if (write(pFile->fd, buf, size) < 0) {
-    free(buf);
-    return -1;
-  }
-
-  pFile->info.size += size;
-
-  free(buf);
-  return 0;
-}
-
 int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) {
   if (dataDir == NULL || fname == NULL) return -1;
 
@@ -354,7 +328,7 @@ SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) {
   return pGroup;
 }
 
-int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) {
+int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) {
   memset((void *)pFile, 0, sizeof(SFile));
   pFile->fd = -1;
 
@@ -370,19 +344,14 @@ int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables,
     return -1;
   }
 
-  if (writeHeader) {
+  pFile->info.size = TSDB_FILE_HEAD_SIZE;
-    if (tsdbWriteHeadFileIdx(pFile, maxTables) < 0) {
-      tsdbCloseFile(pFile);
-      return -1;
-    }
-  }
 
-  if (tsdbWriteFileHead(pFile) < 0) {
+  if (tsdbUpdateFileHeader(pFile, 0) < 0) {
     tsdbCloseFile(pFile);
     return -1;
   }
 
-  if (toClose) tsdbCloseFile(pFile);
+  tsdbCloseFile(pFile);
 
   return 0;
 }

src/tsdb/src/tsdbMain.c

@@ -18,7 +18,7 @@ int tsdbDebugFlag = 135;
 #define TSDB_MIN_ID 0
 #define TSDB_MAX_ID INT_MAX
 
-#define TSDB_CFG_FILE_NAME "CONFIG"
+#define TSDB_CFG_FILE_NAME "config"
 #define TSDB_DATA_DIR_NAME "data"
 #define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7
 #define TSDB_MAX_LAST_FILE_SIZE (1024 * 1024 * 10) // 10M
@@ -29,7 +29,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
 static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
 static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
 // static int     tsdbOpenMetaFile(char *tsdbDir);
-static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now);
+static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows);
 static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg);
 static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname);
 static void *  tsdbCommitData(void *arg);
@@ -91,7 +91,7 @@ void tsdbFreeCfg(STsdbCfg *pCfg) {
 int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) {
 
   if (mkdir(rootDir, 0755) != 0) {
-    tsdbError("id %d: failed to create rootDir! rootDir %s, reason %s", pCfg->tsdbId, rootDir, strerror(errno));
+    tsdbError("vgId:%d, failed to create rootDir! rootDir:%s, reason:%s", pCfg->tsdbId, rootDir, strerror(errno));
     if (errno == EACCES) {
       return TSDB_CODE_NO_DISK_PERMISSIONS;
     } else if (errno == ENOSPC) {
@@ -150,7 +150,7 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) {
   free(pRepo->rootDir);
   free(pRepo);
 
-  tsdbTrace("vgId: %d tsdb repository is dropped!", id);
+  tsdbTrace("vgId:%d, tsdb repository is dropped!", id);
 
   return 0;
 }
@@ -161,7 +161,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) {
   SFileGroup *pFGroup = NULL;
 
   SFileGroupIter iter;
-  SRWHelper      rhelper = {0};
+  SRWHelper      rhelper = {{0}};
 
   if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err;
   tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC);
@@ -169,6 +169,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) {
     if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
     for (int i = 1; i < pRepo->config.maxTables; i++) {
       STable *  pTable = pMeta->tables[i];
+      if (pTable == NULL) continue;
       SCompIdx *pIdx = &rhelper.pCompIdx[i];
 
       if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey;
@@ -242,7 +243,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
 
   pRepo->state = TSDB_REPO_STATE_ACTIVE;
 
-  tsdbTrace("vgId: %d open tsdb repository successfully!", pRepo->config.tsdbId);
+  tsdbTrace("vgId:%d, open tsdb repository successfully!", pRepo->config.tsdbId);
   return (TsdbRepoT *)pRepo;
 }
 
@@ -258,7 +259,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
  *
  * @return 0 for success, -1 for failure and the error number is set
  */
-int32_t tsdbCloseRepo(TsdbRepoT *repo) {
+int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   if (pRepo == NULL) return 0;
   int id = pRepo->config.tsdbId;
@@ -285,7 +286,7 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) {
   tsdbUnLockRepo(repo);
 
   if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
-  tsdbCommitData((void *)repo);
+  if (toCommit) tsdbCommitData((void *)repo);
 
   tsdbCloseFileH(pRepo->tsdbFileH);
 
@@ -296,7 +297,7 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) {
   tfree(pRepo->rootDir);
   tfree(pRepo);
 
-  tsdbTrace("vgId: %d repository is closed!", id);
+  tsdbTrace("vgId:%d, repository is closed!", id);
 
   return 0;
 }
@@ -358,7 +359,7 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
   pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo);
-  tsdbTrace("vgId: %d start to commit!", pRepo->config.tsdbId);
+  tsdbTrace("vgId:%d, start to commit!", pRepo->config.tsdbId);
 
   return 0;
 }
@@ -391,7 +392,7 @@ int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
   return 0;
 }
 
-TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) {
+TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid);
@@ -406,29 +407,30 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) {
 }
 
 // TODO: need to return the number of data inserted
-int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg) {
+int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) {
   SSubmitMsgIter msgIter;
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   tsdbInitSubmitMsgIter(pMsg, &msgIter);
   SSubmitBlk *pBlock = NULL;
   int32_t code = TSDB_CODE_SUCCESS;
+  int32_t affectedrows = 0;
 
   TSKEY now = taosGetTimestamp(pRepo->config.precision);
 
   while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) {
-    if ((code = tsdbInsertDataToTable(repo, pBlock, now)) != TSDB_CODE_SUCCESS) {
+    if ((code = tsdbInsertDataToTable(repo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) {
       return code;
     }
   }
-
+  pRsp->affectedRows = htonl(affectedrows);
   return code;
 }
 
 /**
  * Initialize a table configuration
  */
-int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t tid) {
+int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) {
   if (config == NULL) return -1;
   if (type != TSDB_NORMAL_TABLE && type != TSDB_CHILD_TABLE) return -1;
 
@@ -445,7 +447,7 @@ int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t ti
 /**
  * Set the super table UID of the created table
  */
-int tsdbTableSetSuperUid(STableCfg *config, int64_t uid) {
+int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) {
   if (config->type != TSDB_CHILD_TABLE) return -1;
   if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1;
 
@@ -611,7 +613,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
     pCfg->precision = TSDB_DEFAULT_PRECISION;
   } else {
     if (!IS_VALID_PRECISION(pCfg->precision)) {
-      tsdbError("id %d: invalid precision configuration! precision %d", pCfg->tsdbId, pCfg->precision);
+      tsdbError("vgId:%d, invalid precision configuration! precision:%d", pCfg->tsdbId, pCfg->precision);
       return -1;
     }
   }
@@ -621,7 +623,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
     pCfg->compression = TSDB_DEFAULT_COMPRESSION;
   } else {
     if (!IS_VALID_COMPRESSION(pCfg->compression)) {
-      tsdbError("id %d: invalid compression configuration! compression %d", pCfg->tsdbId, pCfg->precision);
+      tsdbError("vgId:%d: invalid compression configuration! compression:%d", pCfg->tsdbId, pCfg->precision);
       return -1;
     }
   }
@@ -634,7 +636,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
     pCfg->maxTables = TSDB_DEFAULT_TABLES;
   } else {
     if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) {
-      tsdbError("id %d: invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d",
+      tsdbError("vgId:%d: invalid maxTables configuration! maxTables:%d TSDB_MIN_TABLES:%d TSDB_MAX_TABLES:%d",
                 pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES);
       return -1;
     }
@@ -646,7 +648,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   } else {
     if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
       tsdbError(
-          "id %d: invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE "
+          "vgId:%d, invalid daysPerFile configuration! daysPerFile:%d TSDB_MIN_DAYS_PER_FILE:%d TSDB_MAX_DAYS_PER_FILE:"
           "%d",
           pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
       return -1;
@@ -659,8 +661,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   } else {
     if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
       tsdbError(
-          "id %d: invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d "
+          "vgId:%d, invalid minRowsPerFileBlock configuration! minRowsPerFileBlock:%d TSDB_MIN_MIN_ROW_FBLOCK:%d "
-          "TSDB_MAX_MIN_ROW_FBLOCK %d",
+          "TSDB_MAX_MIN_ROW_FBLOCK:%d",
           pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
       return -1;
     }
@@ -671,8 +673,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   } else {
     if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
       tsdbError(
-          "id %d: invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d "
+          "vgId:%d, invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock:%d TSDB_MIN_MAX_ROW_FBLOCK:%d "
-          "TSDB_MAX_MAX_ROW_FBLOCK %d",
+          "TSDB_MAX_MAX_ROW_FBLOCK:%d",
           pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
       return -1;
     }
@@ -686,8 +688,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   } else {
     if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) {
       tsdbError(
-          "id %d: invalid keep configuration! keep %d TSDB_MIN_KEEP %d "
+          "vgId:%d, invalid keep configuration! keep:%d TSDB_MIN_KEEP:%d "
-          "TSDB_MAX_KEEP %d",
+          "TSDB_MAX_KEEP:%d",
           pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
       return -1;
     }
@@ -754,13 +756,13 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
   if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1;
 
   if (mkdir(dirName, 0755) < 0) {
-    tsdbError("vgId: %d failed to create repository directory! reason %s", pRepo->config.tsdbId, strerror(errno));
+    tsdbError("vgId:%d, failed to create repository directory! reason:%s", pRepo->config.tsdbId, strerror(errno));
     return -1;
   }
 
   tsdbTrace(
-      "vgId: %d set up tsdb environment succeed! cacheBlockSize %d, totalBlocks %d, maxTables %d, daysPerFile %d, keep "
+      "vgId:%d, set up tsdb environment succeed! cacheBlockSize:%d, totalBlocks:%d, maxTables:%d, daysPerFile:%d, keep:"
-      "%d, minRowsPerFileBlock %d, maxRowsPerFileBlock %d, precision %d, compression%d",
+      "%d, minRowsPerFileBlock:%d, maxRowsPerFileBlock:%d, precision:%d, compression:%d",
       pRepo->config.tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep,
       pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression);
   return 0;
@@ -840,19 +842,19 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   
   pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData);
 
-  tsdbTrace("vgId: %d, tid: %d, uid: " PRId64 "a row is inserted to table! key" PRId64,
+  tsdbTrace("vgId:%d, tid:%d, uid:%" PRId64 ", table:%s a row is inserted to table! key:%" PRId64, pRepo->config.tsdbId,
-            pRepo->config.tsdbId, pTable->tableId.tid, pTable->tableId.uid, dataRowKey(row));
+            pTable->tableId.tid, pTable->tableId.uid, varDataVal(pTable->name), dataRowKey(row));
 
   return 0;
 }
 
-static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now) {
+static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
   STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId);
   if (pTable == NULL) {
-    tsdbError("id %d: failed to get table for insert, uid:%" PRIu64 ", tid:%d", pRepo->config.tsdbId, pBlock->uid,
+    tsdbError("vgId:%d, failed to get table for insert, uid:" PRIu64 ", tid:%d", pRepo->config.tsdbId, pBlock->uid,
               pBlock->tid);
     return TSDB_CODE_INVALID_TABLE_ID;
   }
@@ -866,15 +868,16 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
   tsdbInitSubmitBlkIter(pBlock, &blkIter);
   while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
     if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
-      tsdbError(
+      tsdbError("vgId:%d, table:%s, tid:%d, talbe uid:%ld timestamp is out of range. now:" PRId64 ", maxKey:" PRId64
-          "tsdbId: %d, table tid: %d, talbe uid: %ld timestamp is out of range. now: %ld maxKey: %ld, minKey: %ld",
+                ", minKey:" PRId64,
-          pRepo->config.tsdbId, pTable->tableId.tid, pTable->tableId.uid, now, minKey, maxKey);
+                pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, pTable->tableId.uid, now, minKey, maxKey);
       return TSDB_CODE_TIMESTAMP_OUT_OF_RANGE;
     }
 
     if (tdInsertRowToTable(pRepo, row, pTable) < 0) {
       return -1;
     }
+     (*affectedrows)++;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -948,7 +951,7 @@ static void *tsdbCommitData(void *arg) {
   STsdbCache *pCache = pRepo->tsdbCache;
   STsdbCfg *  pCfg = &(pRepo->config);
   SDataCols * pDataCols = NULL;
-  SRWHelper   whelper = {0};
+  SRWHelper   whelper = {{0}};
   if (pCache->imem == NULL) return NULL;
 
   tsdbPrint("vgId: %d, starting to commit....", pRepo->config.tsdbId);
@@ -1018,10 +1021,16 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
 
   // Create and open files for commit
   tsdbGetDataDirName(pRepo, dataDir);
-  if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) goto _err;
+  if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) {
+    tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid);
+    goto _err;
+  }
 
   // Open files for write/read
-  if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err;
+  if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
+    tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId);
+    goto _err;
+  }
 
   // Loop to commit data in each table
   for (int tid = 1; tid < pCfg->maxTables; tid++) {
@@ -1058,13 +1067,22 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
     ASSERT(pDataCols->numOfPoints == 0);
 
     // Move the last block to the new .l file if neccessary
-    if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) goto _err;
+    if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
+      tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId);
+      goto _err;
+    }
 
     // Write the SCompBlock part
-    if (tsdbWriteCompInfo(pHelper) < 0) goto _err;
+    if (tsdbWriteCompInfo(pHelper) < 0) {
+      tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId);
+      goto _err;
+    }
   }
 
-  if (tsdbWriteCompIdx(pHelper) < 0) goto _err;
+  if (tsdbWriteCompIdx(pHelper) < 0) {
+    tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId);
+    goto _err;
+  }
 
   tsdbCloseHelperFile(pHelper, 0);
   // TODO: make it atomic with some methods
@@ -1109,7 +1127,7 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK
 static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) {
   int8_t oldCompRession = pRepo->config.compression;
   pRepo->config.compression = compression;
-  tsdbTrace("vgId: %d tsdb compression is changed from %d to %d", oldCompRession, compression);
+  tsdbTrace("vgId:%d, tsdb compression is changed from %d to %d", oldCompRession, compression);
 }
 
 static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
@@ -1126,13 +1144,21 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
     }
     pRepo->tsdbFileH->maxFGroups = maxFiles;
   }
-  tsdbTrace("vgId: %d keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep);
+  tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep);
 }
 
 static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
-  // TODO
   int oldMaxTables = pRepo->config.maxTables;
-  tsdbTrace("vgId: %d tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
+  if (oldMaxTables < pRepo->config.maxTables) {
+    // TODO
+  }
+
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+  pMeta->maxTables = maxTables;
+  pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *));
+
+  tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
 }
 
 uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size) {

src/tsdb/src/tsdbMeta.c

@@ -6,7 +6,7 @@
 #include "tsdbMain.h"
 
 #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here
-#define TSDB_META_FILE_NAME "META"
+// #define TSDB_META_FILE_NAME "META"
 
 const int32_t DEFAULT_TAG_INDEX_COLUMN = 0;   // skip list built based on the first column of tags
 
@@ -80,16 +80,16 @@ STable *tsdbDecodeTable(void *cont, int contLen) {
   T_READ_MEMBER(ptr, int8_t, pTable->type);
   int len = *(int *)ptr;
   ptr = (char *)ptr + sizeof(int);
-  pTable->name = calloc(1, len + VARSTR_HEADER_SIZE);
+  pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1);
   if (pTable->name == NULL) return NULL;
   
   varDataSetLen(pTable->name, len);
   memcpy(pTable->name->data, ptr, len);
   
   ptr = (char *)ptr + len;
-  T_READ_MEMBER(ptr, int64_t, pTable->tableId.uid);
+  T_READ_MEMBER(ptr, uint64_t, pTable->tableId.uid);
   T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid);
-  T_READ_MEMBER(ptr, int64_t, pTable->superUid);
+  T_READ_MEMBER(ptr, uint64_t, pTable->superUid);
   T_READ_MEMBER(ptr, int32_t, pTable->sversion);
 
   if (pTable->type == TSDB_SUPER_TABLE) {
@@ -310,7 +310,7 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
       
       // todo refactor extract method
       size_t size = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN);
-      super->name = malloc(size + VARSTR_HEADER_SIZE);
+      super->name = calloc(1, size + VARSTR_HEADER_SIZE + 1);
       STR_WITH_SIZE_TO_VARSTR(super->name, pCfg->sname, size);
 
       // index the first tag column
@@ -339,7 +339,7 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
   table->tableId = pCfg->tableId;
   
   size_t size = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN);
-  table->name = malloc(size + VARSTR_HEADER_SIZE);
+  table->name = calloc(1, size + VARSTR_HEADER_SIZE + 1);
   STR_WITH_SIZE_TO_VARSTR(table->name, pCfg->name, size);
   
   table->lastKey = 0;
@@ -356,12 +356,12 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
   // Register to meta
   if (newSuper) {
     tsdbAddTableToMeta(pMeta, super, true);
-    tsdbTrace("vgId: %d super table is created! uid " PRId64, pRepo->config.tsdbId,
+    tsdbTrace("vgId:%d, super table %s is created! uid:%" PRId64, pRepo->config.tsdbId, varDataVal(super->name),
               super->tableId.uid);
   }
   tsdbAddTableToMeta(pMeta, table, true);
-  tsdbTrace("vgId: %d table is created! tid %d, uid " PRId64, pRepo->config.tsdbId, table->tableId.tid,
+  tsdbTrace("vgId:%d, table %s is created! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(table->name),
-            table->tableId.uid);
+            table->tableId.tid, table->tableId.uid);
 
   // Write to meta file
   int bufLen = 0;
@@ -404,12 +404,13 @@ int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
 
   STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid);
   if (pTable == NULL) {
-    tsdbError("vgId %d: failed to drop table since table not exists! tid %d, uid " PRId64, pRepo->config.tsdbId,
+    tsdbError("vgId:%d, failed to drop table since table not exists! tid:%d, uid:" PRId64, pRepo->config.tsdbId,
               tableId.tid, tableId.uid);
     return -1;
   }
 
-  tsdbTrace("vgId: %d table is dropped! tid %d, uid " PRId64, pRepo->config.tsdbId, tableId.tid, tableId.uid);
+  tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
+            tableId.tid, tableId.uid);
   if (tsdbRemoveTableFromMeta(pMeta, pTable) < 0) return -1;
 
   return 0;
@@ -455,7 +456,7 @@ static int32_t tsdbCheckTableCfg(STableCfg *pCfg) {
   return 0;
 }
 
-STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid) {
+STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) {
   void *ptr = taosHashGet(pMeta->map, (char *)(&uid), sizeof(uid));
 
   if (ptr == NULL) return NULL;
@@ -508,10 +509,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
 
       ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE);
 
-      pMeta->tables[tTable->tableId.tid] = NULL;
+      tsdbRemoveTableFromMeta(pMeta, tTable);
-      taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid));
-      pMeta->nTables--;
-      tsdbFreeTable(tTable);
     }
 
     tSkipListDestroyIter(pIter);
@@ -534,8 +532,8 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
     pMeta->nTables--;
   }
 
-  tsdbFreeTable(pTable);
   taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid));
+  tsdbFreeTable(pTable);
   return 0;
 }
 

src/tsdb/src/tsdbMetaFile.c

@@ -23,9 +23,9 @@
 #define TSDB_META_FILE_HEADER_SIZE 512
 
 typedef struct {
-  int32_t offset;
+  int32_t  offset;
-  int32_t size;
+  int32_t  size;
-  int64_t uid;
+  uint64_t uid;
 } SRecordInfo;
 
 // static int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
@@ -76,7 +76,7 @@ SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, af
   return mfh;
 }
 
-int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
+int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) {
   if (taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) {
     return -1;
   }
@@ -112,7 +112,7 @@ int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t co
   return 0;
 }
 
-int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) {
+int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid) {
   char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
   if (ptr == NULL) return -1;
 
@@ -139,7 +139,7 @@ int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) {
   return 0;
 }
 
-int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
+int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) {
   char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
   if (ptr == NULL) return -1;
 

src/tsdb/src/tsdbRWHelper.c

@@ -18,6 +18,7 @@
 #include "tchecksum.h"
 #include "tscompression.h"
 #include "talgo.h"
+#include "tcoding.h"
 
 // Local function definitions
 // static int  tsdbCheckHelperCfg(SHelperCfg *pCfg);
@@ -131,10 +132,10 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t
   // Init block part
   if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
 
-  pHelper->blockBuffer =
+  pHelper->pBuffer =
       tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols +
               pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM));
-  if (pHelper->blockBuffer == NULL) goto _err;
+  if (pHelper->pBuffer == NULL) goto _err;
 
   return 0;
 
@@ -154,7 +155,7 @@ int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
 
 void tsdbDestroyHelper(SRWHelper *pHelper) {
   if (pHelper) {
-    tzfree(pHelper->blockBuffer);
+    tzfree(pHelper->pBuffer);
     tzfree(pHelper->compBuffer);
     tsdbDestroyHelperFile(pHelper);
     tsdbDestroyHelperTable(pHelper);
@@ -211,13 +212,15 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) {
 
     // Create and open .h
     if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1;
-    size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
+    // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
-    if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, tsize) < tsize) goto _err;
+    if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE)
+      goto _err;
 
     // Create and open .l file if should
     if (tsdbShouldCreateNewLast(pHelper)) {
       if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err;
-      if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) goto _err;
+      if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE)
+        goto _err;
     }
   } else {
     if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err;
@@ -238,6 +241,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
     pHelper->files.headF.fd = -1;
   }
   if (pHelper->files.dataF.fd > 0) {
+    if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.dataF), 0);
     close(pHelper->files.dataF.fd);
     pHelper->files.dataF.fd = -1;
   }
@@ -246,6 +250,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
     pHelper->files.lastF.fd = -1;
   }
   if (pHelper->files.nHeadF.fd > 0) {
+    if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0);
     close(pHelper->files.nHeadF.fd);
     pHelper->files.nHeadF.fd = -1;
     if (hasError) {
@@ -257,6 +262,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
   }
   
   if (pHelper->files.nLastF.fd > 0) {
+    if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0);
     close(pHelper->files.nLastF.fd);
     pHelper->files.nLastF.fd = -1;
     if (hasError) {
@@ -416,7 +422,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
     pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
     pIdx->uid = pHelper->tableInfo.uid;
     if (pIdx->offset < 0) return -1;
-    ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx));
+    ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
     if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1;
   }
@@ -426,13 +432,27 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
 
 int tsdbWriteCompIdx(SRWHelper *pHelper) {
   ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
-  if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
+  off_t offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
+  if (offset < 0) return -1;
 
-  ASSERT(tsizeof(pHelper->pCompIdx) == sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM));
+  SFile *pFile = &(pHelper->files.nHeadF);
-  taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx));
+  pFile->info.offset = offset;
 
-  if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx))
+  // TODO: change the implementation of pHelper->pBuffer
-    return -1;
+  void *buf = pHelper->pBuffer;
+  for (uint32_t i = 0; i < pHelper->config.maxTables; i++) {
+    SCompIdx *pCompIdx = pHelper->pCompIdx + i;
+    if (pCompIdx->offset > 0) {
+      buf = taosEncodeVariant32(buf, i);
+      buf = tsdbEncodeSCompIdx(buf, pCompIdx);
+    }
+  }
+
+  int tsize = (char *)buf - (char *)pHelper->pBuffer + sizeof(TSCKSUM);
+  taosCalcChecksumAppend(0, (uint8_t *)pHelper->pBuffer, tsize);
+
+  if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pBuffer, tsize) < tsize) return -1;
+  pFile->info.len = tsize;
   return 0;
 }
 
@@ -441,14 +461,36 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
 
   if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) {
     // If not load from file, just load it in object
-    int fd = pHelper->files.headF.fd;
+    SFile *pFile = &(pHelper->files.headF);
+    int fd = pFile->fd;
 
-    if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
+    memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx));
-    if (tread(fd, (void *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1;
+    if (pFile->info.offset > 0) {
-    if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx))) {
+      ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE);
-      // TODO: File is broken, try to deal with it
+
-      return -1;
+      if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) return -1;
+      if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len)
+        return -1;
+      if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) {
+        // TODO: File is broken, try to deal with it
+        return -1;
+      }
+
+      // Decode it
+      void *ptr = pHelper->pBuffer;
+      while (((char *)ptr - (char *)pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) {
+        uint32_t tid = 0;
+        if ((ptr = taosDecodeVariant32(ptr, &tid)) == NULL) return -1;
+        ASSERT(tid > 0 && tid < pHelper->config.maxTables);
+
+        if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1;
+
+        ASSERT((char *)ptr - (char *)pHelper->pBuffer <= pFile->info.len - sizeof(TSCKSUM));
+      }
+
+      ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM)));
     }
+
   }
   helperSetState(pHelper, TSDB_HELPER_IDX_LOAD);
 
@@ -590,9 +632,9 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32
 static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
   ASSERT(pCompBlock->numOfSubBlocks <= 1);
 
-  ASSERT(tsizeof(pHelper->blockBuffer) >= pCompBlock->len);
+  ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len);
 
-  SCompData *pCompData = (SCompData *)pHelper->blockBuffer;
+  SCompData *pCompData = (SCompData *)pHelper->pBuffer;
 
   int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
   if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err;
@@ -685,7 +727,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
   ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints &&
          rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
 
-  SCompData *pCompData = (SCompData *)(pHelper->blockBuffer);
+  SCompData *pCompData = (SCompData *)(pHelper->pBuffer);
   int64_t offset = 0;
 
   offset = lseek(pFile->fd, 0, SEEK_END);
@@ -701,8 +743,15 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
       continue;
     }
 
+    memset(pCompCol, 0, sizeof(*pCompCol));
+
     pCompCol->colId = pDataCol->colId;
     pCompCol->type = pDataCol->type;
+    if (tDataTypeDesc[pDataCol->type].getStatisFunc) {
+      (*tDataTypeDesc[pDataCol->type].getStatisFunc)(
+          (TSKEY *)(pDataCols->cols[0].pData), pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max),
+          &(pCompCol->sum), &(pCompCol->minIndex), &(pCompCol->maxIndex), &(pCompCol->numOfNull));
+    }
     nColsNotAllNull++;
   }
 
@@ -733,7 +782,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
       }
 
       pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))(
-          (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->blockBuffer) - lsize,
+          (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize,
           pHelper->config.compress, pHelper->compBuffer, tsizeof(pHelper->compBuffer));
     } else {
       pCompCol->len = tlen;
@@ -1162,3 +1211,72 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey)
 
   return ((TSKEY *)ptr2 - (TSKEY *)ptr1) + 1;
 }
+
+void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) {
+  buf = taosEncodeVariant32(buf, pIdx->len);
+  buf = taosEncodeVariant32(buf, pIdx->offset);
+  buf = taosEncodeFixed8(buf, pIdx->hasLast);
+  buf = taosEncodeVariant32(buf, pIdx->numOfBlocks);
+  buf = taosEncodeFixed64(buf, pIdx->uid);
+  buf = taosEncodeFixed64(buf, pIdx->maxKey);
+
+  return buf;
+}
+
+void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) {
+  uint8_t  hasLast = 0;
+  uint32_t numOfBlocks = 0;
+  uint64_t value = 0;
+
+  if ((buf = taosDecodeVariant32(buf, &(pIdx->len))) == NULL) return NULL;
+  if ((buf = taosDecodeVariant32(buf, &(pIdx->offset))) == NULL) return NULL;
+  if ((buf = taosDecodeFixed8(buf, &(hasLast))) == NULL) return NULL;
+  pIdx->hasLast = hasLast;
+  if ((buf = taosDecodeVariant32(buf, &(numOfBlocks))) == NULL) return NULL;
+  pIdx->numOfBlocks = numOfBlocks;
+  if ((buf = taosDecodeFixed64(buf, &value)) == NULL) return NULL;
+  pIdx->uid = (int64_t)value;
+  if ((buf = taosDecodeFixed64(buf, &value)) == NULL) return NULL;
+  pIdx->maxKey = (TSKEY)value;
+
+  return buf;
+}
+
+int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) {
+  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+
+  void *pBuf = (void *)buf;
+  pBuf = taosEncodeFixed32(pBuf, version);
+  pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info));
+
+  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+
+  if (lseek(pFile->fd, 0, SEEK_SET) < 0) return -1;
+  if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1;
+
+  return 0;
+}
+
+
+
+void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) {
+  buf = taosEncodeFixed32(buf, pInfo->offset);
+  buf = taosEncodeFixed32(buf, pInfo->len);
+  buf = taosEncodeFixed64(buf, pInfo->size);
+  buf = taosEncodeFixed64(buf, pInfo->tombSize);
+  buf = taosEncodeFixed32(buf, pInfo->totalBlocks);
+  buf = taosEncodeFixed32(buf, pInfo->totalSubBlocks);
+
+  return buf;
+}
+
+void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) {
+  buf = taosDecodeFixed32(buf, &(pInfo->offset));
+  buf = taosDecodeFixed32(buf, &(pInfo->len));
+  buf = taosDecodeFixed64(buf, &(pInfo->size));
+  buf = taosDecodeFixed64(buf, &(pInfo->tombSize));
+  buf = taosDecodeFixed32(buf, &(pInfo->totalBlocks));
+  buf = taosDecodeFixed32(buf, &(pInfo->totalSubBlocks));
+
+  return buf;
+}

src/tsdb/src/tsdbRead.c

@@ -1933,8 +1933,9 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, int16_t tagNameRelType,
+int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTagCond, size_t len,
-  const char* tbnameCond, STableGroupInfo *pGroupInfo, SColIndex *pColIndex, int32_t numOfCols) {
+                                 int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
+                                 SColIndex* pColIndex, int32_t numOfCols) {
   STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
   if (pTable == NULL) {
     uError("%p failed to get stable, uid:%" PRIu64, tsdb, uid);
@@ -2006,7 +2007,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC
   return ret;
 }
 
-int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, int64_t uid, STableGroupInfo* pGroupInfo) {
+int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) {
   STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
   if (pTable == NULL) {
     return TSDB_CODE_INVALID_TABLE_ID;

src/tsdb/tests/tsdbTests.cpp

@@ -16,7 +16,7 @@ typedef struct {
   TsdbRepoT *pRepo;
   bool       isAscend;
   int        tid;
-  int64_t    uid;
+  uint64_t   uid;
   int        sversion;
   TSKEY      startTime;
   TSKEY      interval;

src/util/inc/hashint.h

@@ -1,26 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _sdb_int_hash_header_
-#define _sdb_int_hash_header_
-
-void *sdbOpenIntHash(int maxSessions, int dataSize);
-void sdbCloseIntHash(void *handle);
-void *sdbAddIntHash(void *handle, void *key, void *pData);
-void sdbDeleteIntHash(void *handle, void *key);
-void *sdbGetIntHashData(void *handle, void *key);
-void *sdbFetchIntHashData(void *handle, void *ptr, void **ppMeta);
-
-#endif

src/util/inc/ihash.h

@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_IHASH_H
-#define TDENGINE_IHASH_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-void *taosInitIntHash(int32_t maxSessions, int32_t dataSize, int32_t (*fp)(void *, uint64_t));
-
-void taosCleanUpIntHash(void *handle);
-
-char *taosGetIntHashData(void *handle, uint64_t key);
-
-void taosDeleteIntHash(void *handle, uint64_t key);
-
-char *taosAddIntHash(void *handle, uint64_t key, char *pData);
-
-int32_t taosHashInt(void *handle, uint64_t key);
-
-void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *));
-
-void taosVisitIntHashWithFp(void *handle, void (*fp)(char *, void *), void *param);
-
-int32_t taosGetIntHashSize(void *handle);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_IHASH_H

src/util/inc/shash.h

@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TSHASH_H
-#define TDENGINE_TSHASH_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-void *taosInitStrHash(uint32_t maxSessions, uint32_t dataSize, uint32_t (*fp)(void *, char *));
-
-void taosCleanUpStrHash(void *handle);
-
-void *taosGetStrHashData(void *handle, char *string);
-
-void taosDeleteStrHash(void *handle, char *string);
-
-void taosDeleteStrHashNode(void *handle, char *string, void *pDeleteNode);
-
-void *taosAddStrHash(void *handle, char *string, char *pData);
-
-void *taosAddStrHashWithSize(void *handle, char *string, char *pData, int dataSize);
-
-uint32_t taosHashString(void *handle, char *string);
-
-uint32_t taosHashStringStep1(void *handle, char *string);
-
-char *taosVisitStrHashWithFp(void *handle, int (*fp)(char *));
-
-void taosCleanUpStrHashWithFp(void *handle, void (*fp)(char *));
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_TSHASH_H

src/util/inc/tcache.h

@@ -67,6 +67,7 @@ typedef struct {
   void *       pTimer;
   SCacheStatis statistics;
   SHashObj *   pHashTable;
+  _hash_free_fn_t freeFp;
   int          numOfElemsInTrash;  // number of element in trash
   int16_t      deleting;           // set the deleting flag to stop refreshing ASAP.
   T_REF_DECLARE()
@@ -88,6 +89,7 @@ typedef struct {
  * @return
  */
 SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTimeInSeconds);
+SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
 
 /**
  * add data into cache

src/util/inc/tcoding.h

@@ -29,6 +29,11 @@ extern "C" {
 static const int32_t TNUMBER = 1;
 #define IS_LITTLE_ENDIAN() (*(uint8_t *)(&TNUMBER) != 0)
 
+static FORCE_INLINE void *taosEncodeFixed8(void *buf, uint8_t value) {
+  ((uint8_t *)buf)[0] = value;
+  return POINTER_SHIFT(buf, sizeof(value));
+}
+
 static FORCE_INLINE void *taosEncodeFixed16(void *buf, uint16_t value) {
   if (IS_LITTLE_ENDIAN()) {
     memcpy(buf, &value, sizeof(value));
@@ -70,6 +75,11 @@ static FORCE_INLINE void *taosEncodeFixed64(void *buf, uint64_t value) {
   return POINTER_SHIFT(buf, sizeof(value));
 }
 
+static FORCE_INLINE void *taosDecodeFixed8(void *buf, uint8_t *value) {
+  *value = ((uint8_t *)buf)[0];
+  return POINTER_SHIFT(buf, sizeof(*value));
+}
+
 static FORCE_INLINE void *taosDecodeFixed16(void *buf, uint16_t *value) {
   if (IS_LITTLE_ENDIAN()) {
     memcpy(value, buf, sizeof(*value));

src/util/inc/thash.h

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_THASH_H
-#define TDENGINE_THASH_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void *taosOpenHash(int maxSessions, int (*fp)(void *, uint64_t));
-
-void taosCloseHash(void *handle);
-
-int taosAddHash(void *handle, uint64_t, uint32_t id);
-
-void taosDeleteHash(void *handle, uint64_t);
-
-int32_t taosGetIdFromHash(void *handle, uint64_t);
-
-int taosHashLong(void *, uint64_t ip);
-
-uint64_t taosHashUInt64(uint64_t handle);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_THASH_H

src/util/inc/tskiplist.h

@@ -174,7 +174,7 @@ void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSiz
 SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
 
 /**
- * get only *one* node of which key is equalled to pKey, even there are more than one nodes are of the same key
+ * get *all* nodes which key are equivalent to pKey
  *
  * @param pSkipList
  * @param pKey
@@ -234,14 +234,13 @@ SSkipListNode *tSkipListIterGet(SSkipListIterator *iter);
 void *tSkipListDestroyIter(SSkipListIterator *iter);
 
 /*
- * remove only one node of the pKey value.
+ * remove nodes of the pKey value.
- * If more than one node has the same value, any one will be removed
+ * If more than one node has the same value, all will be removed
  *
  * @Return
- * true: one node has been removed
+ * the count of removed nodes
- * false: no node has been removed
  */
-bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
+uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
 
 /*
  * remove the specified node in parameters

src/util/src/exception.c

@@ -1,7 +1,8 @@
+#include "os.h"
 #include "exception.h"
 
 
-static _Thread_local SExceptionNode* expList;
+static threadlocal SExceptionNode* expList;
 
 void exceptionPushNode( SExceptionNode* node ) {
     node->prev = expList;