Merge remote-tracking branch 'origin/develop' into feature/crash_gen

.travis.yml

@@ -135,92 +135,6 @@ matrix:
         #   https://scan.coverity.com/faq#frequency
         branch_pattern: coverity_scan
 
-  - os: linux
-    dist: bionic
-    language: c
-    compiler: gcc
-    env: ENV_COVER=true
-
-    git:
-      - depth: 1
-
-    addons:
-      apt:
-        packages:
-          - build-essential
-          - cmake
-          - net-tools
-          - python-pip
-          - python-setuptools
-          - python3-pip
-          - python3-setuptools
-          - lcov
-          - psmisc
-
-    before_script:
-      - cd ${TRAVIS_BUILD_DIR}
-      - mkdir debug
-      - cd debug
-
-    script:
-      - cmake -DCOVER=true .. > /dev/null
-      - make > /dev/null
-
-    after_success:
-      - |-
-        case $TRAVIS_OS_NAME in
-          linux)
-          cd ${TRAVIS_BUILD_DIR}/debug
-          make install > /dev/null || travis_terminate $?
-
-          pip install numpy
-          pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
-          pip3 install numpy
-          pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
-
-          cd ${TRAVIS_BUILD_DIR}/tests
-
-          ./test-all.sh smoke COVER
-
-          TEST_RESULT=$?
-
-          pkill taosd
-          sleep 1
-
-          cd ${TRAVIS_BUILD_DIR}
-          lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info
-          lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' -o coverage.info
-          lcov -l --rc lcov_branch_coverage=1 coverage.info || travis_terminate $?
-
-          gem install coveralls-lcov
-
-          # Color setting
-          RED='\033[0;31m'
-          GREEN='\033[1;32m'
-          GREEN_DARK='\033[0;32m'
-          GREEN_UNDERLINE='\033[4;32m'
-          NC='\033[0m'
-
-          coveralls-lcov coverage.info
-          if [ "$?" -eq "0" ]; then
-            echo -e "${GREEN} ## Uploaded to Coveralls.io! ## ${NC}"
-          else
-            echo -e "${RED} ## Coveralls.io not collect coverage report! ## ${NC} "
-          fi
-
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info
-          if [ "$?" -eq "0" ]; then
-            echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} "
-          else
-            echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} "
-          fi
-
-          if [ "$TEST_RESULT" -ne "0" ]; then
-            travis_terminate $?
-          fi
-          ;;
-        esac
-
   - os: linux
     dist: trusty
     language: c

README.md

@@ -1,5 +1,6 @@
 [![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine)
 [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
+[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
 
 [![TDengine](TDenginelogo.png)](https://www.taosdata.com)
 

src/client/inc/tscUtil.h

@@ -89,27 +89,21 @@ typedef struct SVgroupTableInfo {
 
 int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name,
                            STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
-void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks);
 void tscDestroyDataBlock(STableDataBlocks* pDataBlock);
 void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf);
 
 SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes,
                                    uint32_t offset);
 
-SDataBlockList* tscCreateBlockArrayList();
+void*   tscDestroyBlockArrayList(SArray* pDataBlockList);
-
-void*   tscDestroyBlockArrayList(SDataBlockList* pList);
 int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
-void    tscFreeUnusedDataBlocks(SDataBlockList* pList);
+void    tscFreeUnusedDataBlocks(SArray* pDataBlockList);
-int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pDataList);
+int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pDataList);
-int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size,
+int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size,
                                 int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
                                 STableDataBlocks** dataBlocks);
 
-//UNUSED_FUNC STableIdInfo*  tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx);
-
 /**
- *
  * for the projection query on metric or point interpolation query on metric,
  * we iterate all the meters, instead of invoke query on all qualified meters simultaneously.
  *

src/client/inc/tsclient.h

@@ -171,11 +171,7 @@ typedef struct STableDataBlocks {
    * to avoid it to be removed from cache
    */
   STableMeta *pTableMeta;
-
-  union {
-    char *filename;
   char       *pData;
-  };
 
   // for parameter ('?') binding
   uint32_t    numOfAllocedParams;
@@ -183,11 +179,11 @@ typedef struct STableDataBlocks {
   SParamInfo *params;
 } STableDataBlocks;
 
-typedef struct SDataBlockList {  // todo remove
+//typedef struct SDataBlockList {  // todo remove
-  uint32_t           nSize;
+//  uint32_t           nSize;
-  uint32_t           nAlloc;
+//  uint32_t           nAlloc;
-  STableDataBlocks **pData;
+//  STableDataBlocks **pData;
-} SDataBlockList;
+//} SDataBlockList;
 
 typedef struct SQueryInfo {
   int16_t          command;       // the command may be different for each subclause, so keep it seperately.
@@ -238,8 +234,7 @@ typedef struct {
   void *       pTableList;   // referred table involved in sql
   int32_t      batchSize;    // for parameter ('?') binding and batch processing
   int32_t      numOfParams;
-
+  SArray      *pDataBlocks;  // SArray<STableDataBlocks*> submit data blocks after parsing sql
-  SDataBlockList *pDataBlocks;  // submit data blocks after parsing sql
 } SSqlCmd;
 
 typedef struct SResRec {
@@ -399,7 +394,7 @@ void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ;
 int doAsyncParseSql(SSqlObj* pSql);
 void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen);
 
-void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql);
+void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql);
 void tscKillSTableQuery(SSqlObj *pSql);
 void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
 bool tscIsUpdateQuery(SSqlObj* pSql);

src/client/src/TSDBJNIConnector.c

@@ -565,6 +565,11 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
     sql = (char *)(*env)->GetStringUTFChars(env, jsql, NULL);
   }
 
+  if (topic == NULL || sql == NULL) {
+    jniTrace("jobj:%p, invalid argument: topic or sql is NULL", jobj);
+    return sub;
+  }
+
   TAOS_SUB *tsub = taos_subscribe(taos, (int)restart, topic, sql, NULL, NULL, jinterval);
   sub = (jlong)tsub;
 
@@ -574,8 +579,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
     jniTrace("jobj:%p, successfully subscribe: topic: %s", jobj, topic);
   }
 
-  if (topic != NULL) (*env)->ReleaseStringUTFChars(env, jtopic, topic);
+  (*env)->ReleaseStringUTFChars(env, jtopic, topic);
-  if (sql != NULL) (*env)->ReleaseStringUTFChars(env, jsql, sql);
+  (*env)->ReleaseStringUTFChars(env, jsql, sql);
 
   return sub;
 }

src/client/src/tscLocalMerge.c

@@ -123,7 +123,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
     }
   }
 
-  if (n == 0) {
+  if (n == 0 || pCtx == NULL) {
     free(pTagCtx);
   } else {
     pCtx->tagInfo.pTagCtxList = pTagCtx;

src/client/src/tscParseInsert.c

@@ -46,18 +46,28 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
     return TK_ILLEGAL;
   }
   
-  int32_t radix = 10;
 
-  int32_t radixList[3] = {16, 8, 2}; // the integer number with different radix: hex, oct, bin
+  int32_t radix = 10;
-  if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) {
+  if (pToken->type == TK_HEX) {
-    radix = radixList[pToken->type - TK_HEX];
+    radix = 16;
+  } else if (pToken->type == TK_BIN) {
+    radix = 2;
   }
   
   errno = 0;
   *value = strtoll(pToken->z, endPtr, radix);
+  if (**endPtr == 'e' || **endPtr == 'E' || **endPtr == '.') {
+    errno = 0;
+    double v = round(strtod(pToken->z, endPtr));
+    if (v > INT64_MAX || v <= INT64_MIN) {
+      errno = ERANGE;
+    } else {
+      *value = v;
+    }
+  }
   
   // not a valid integer number, return error
-  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+  if (*endPtr - pToken->z != pToken->n) {
     return TK_ILLEGAL;
   }
 
@@ -73,11 +83,11 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
   *value = strtod(pToken->z, endPtr);
   
   // not a valid integer number, return error
-  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+  if ((*endPtr - pToken->z) != pToken->n) {
     return TK_ILLEGAL;
-  } else {
-    return pToken->type;
   }
+
+  return pToken->type;
 }
 
 int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@@ -781,12 +791,12 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     sToken = tStrGetToken(sql, &index, false, 0, NULL);
     sql += index;
 
-    STagData *pTag = (STagData *)pCmd->payload;
+    tscAllocPayload(pCmd, sizeof(STagData));
+    STagData *pTag = (STagData *) pCmd->payload;
+
     memset(pTag, 0, sizeof(STagData));
     
-    /*
+    //the source super table is moved to the secondary position of the pTableMetaInfo list
-     * the source super table is moved to the secondary position of the pTableMetaInfo list
-     */
     if (pQueryInfo->numOfTables < 2) {
       tscAddEmptyMetaInfo(pQueryInfo);
     }
@@ -887,9 +897,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       index = 0;
       sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes);
       sql += index;
-      if (sToken.n == 0) {
+
-        break;
+      if (sToken.n == 0 || sToken.type == TK_RP) {
-      } else if (sToken.type == TK_RP) {
         break;
       }
 
@@ -903,11 +912,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
-
-      if ((pTagSchema[colIndex].type == TSDB_DATA_TYPE_BINARY || pTagSchema[colIndex].type == TSDB_DATA_TYPE_NCHAR) &&
-          sToken.n > pTagSchema[colIndex].bytes) {
-        return tscInvalidSQLErrMsg(pCmd->payload, "string too long", sToken.z);
-      }
     }
 
     index = 0;
@@ -986,14 +990,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
   return code;
 }
 
-int validateTableName(char *tblName, int len) {
+int validateTableName(char *tblName, int len, SSQLToken* psTblToken) {
-  char buf[TSDB_TABLE_ID_LEN] = {0};
+  tstrncpy(psTblToken->z, tblName, TSDB_TABLE_ID_LEN);
-  tstrncpy(buf, tblName, sizeof(buf));
 
-  SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
+  psTblToken->n    = len;
-  tSQLGetToken(buf, &token.type);
+  psTblToken->type = TK_ID;
+  tSQLGetToken(psTblToken->z, &psTblToken->type);
 
-  return tscValidateName(&token);
+  return tscValidateName(psTblToken);
 }
 
 static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) {
@@ -1031,15 +1035,13 @@ int tsParseInsertSql(SSqlObj *pSql) {
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   }
 
-  // TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536, but TSDB_PAYLOAD_SIZE is 65380
+  if ((code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) {
-  if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
     return code;
   }
 
   if (NULL == pCmd->pTableList) {
     pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
-
+    pCmd->pDataBlocks = taosArrayInit(4, POINTER_BYTES);
-    pSql->cmd.pDataBlocks = tscCreateBlockArrayList();
     if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) {
       code = TSDB_CODE_TSC_OUT_OF_MEMORY;
       goto _error;
@@ -1077,14 +1079,16 @@ int tsParseInsertSql(SSqlObj *pSql) {
     }
 
     pCmd->curSql = sToken.z;
-
+    char buf[TSDB_TABLE_ID_LEN];
+    SSQLToken sTblToken;
+    sTblToken.z = buf;
     // Check if the table name available or not
-    if (validateTableName(sToken.z, sToken.n) != TSDB_CODE_SUCCESS) {
+    if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) {
       code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z);
       goto _error;
     }
 
-    if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
+    if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) {
       goto _error;
     }
 
@@ -1149,29 +1153,19 @@ int tsParseInsertSql(SSqlObj *pSql) {
         goto _error;
       }
 
-      char fname[PATH_MAX] = {0};
+      strncpy(pCmd->payload, sToken.z, sToken.n);
-      strncpy(fname, sToken.z, sToken.n);
+      strdequote(pCmd->payload);
-      strdequote(fname);
 
+      // todo refactor extract method
       wordexp_t full_path;
-      if (wordexp(fname, &full_path, 0) != 0) {
+      if (wordexp(pCmd->payload, &full_path, 0) != 0) {
         code = tscInvalidSQLErrMsg(pCmd->payload, "invalid filename", sToken.z);
         goto _error;
       }
-      strcpy(fname, full_path.we_wordv[0]);
+
+      tstrncpy(pCmd->payload, full_path.we_wordv[0], pCmd->allocSize);
       wordfree(&full_path);
 
-      STableDataBlocks *pDataBlock = NULL;
-      STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
-      
-      int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name,
-                                       pTableMeta, &pDataBlock);
-      if (ret != TSDB_CODE_SUCCESS) {
-        goto _error;
-      }
-
-      tscAppendDataBlock(pCmd->pDataBlocks, pDataBlock);
-      strcpy(pDataBlock->filename, fname);
     } else if (sToken.type == TK_LP) {
       /* insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); */
       STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta;
@@ -1258,7 +1252,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
     goto _clean;
   }
 
-  if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId
+  if (taosArrayGetSize(pCmd->pDataBlocks) > 0) { // merge according to vgId
     if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -1368,177 +1362,146 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
     return code;
   }
 
-  // the pDataBlock is different from the pTableDataBlocks
+  STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0);
-  STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0];
   if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) {
     return code;
   }
 
-  if ((code = tscProcessSql(pSql)) != TSDB_CODE_SUCCESS) {
+  return tscProcessSql(pSql);
-    return code;
-  }
-
-  return TSDB_CODE_SUCCESS;
 }
 
-static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
+typedef struct SImportFileSupport {
-  size_t          readLen = 0;
+  SSqlObj *pSql;
-  char *          line = NULL;
+  FILE    *fp;
-  size_t          n = 0;
+} SImportFileSupport;
-  int             len = 0;
+
-  int32_t         maxRows = 0;
+static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
-  SSqlCmd *       pCmd = &pSql->cmd;
+  assert(param != NULL && tres != NULL);
-  int             numOfRows = 0;
+
-  int32_t         code = 0;
+  SSqlObj *pSql = tres;
-  int             nrows = 0;
+  SSqlCmd *pCmd = &pSql->cmd;
+
+  SImportFileSupport *pSupporter = (SImportFileSupport *) param;
+
+  SSqlObj *pParentSql = pSupporter->pSql;
+  FILE    *fp = pSupporter->fp;
+
+  if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {  // handle error
+    assert(taos_errno(pSql) == code);
+
+    taos_free_result(pSql);
+    tfree(pSupporter);
+    fclose(fp);
+
+    pParentSql->res.code = code;
+    return;
+  }
+
+  // accumulate the total submit records
+  pParentSql->res.numOfRows += pSql->res.numOfRows;
 
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   STableMeta *    pTableMeta = pTableMetaInfo->pTableMeta;
+  SSchema *       pSchema = tscGetTableSchema(pTableMeta);
   STableComInfo   tinfo = tscGetTableInfo(pTableMeta);
 
-  assert(pCmd->numOfClause == 1);
-  
-  int32_t rowSize = tinfo.rowSize;
-
-  pCmd->pDataBlocks = tscCreateBlockArrayList();
-  STableDataBlocks *pTableDataBlock = NULL;
-  int32_t           ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SSubmitBlk),
-                                   pTableMetaInfo->name, pTableMeta, &pTableDataBlock);
-  if (ret != TSDB_CODE_SUCCESS) {
-    return -1;
-  }
-
-  tscAppendDataBlock(pCmd->pDataBlocks, pTableDataBlock);
-
-  code = tscAllocateMemIfNeed(pTableDataBlock, rowSize, &maxRows);
-  if (TSDB_CODE_SUCCESS != code) return -1;
-
-  int                count = 0;
   SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns};
-  SSchema *          pSchema = tscGetTableSchema(pTableMeta);
-
   tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns);
 
+  size_t  n = 0;
+  ssize_t readLen = 0;
+  char *  line = NULL;
+  int32_t count = 0;
+  int32_t maxRows = 0;
+
+  tscDestroyBlockArrayList(pSql->cmd.pDataBlocks);
+  pCmd->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
+
+  STableDataBlocks *pTableDataBlock = NULL;
+  int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock);
+  if (ret != TSDB_CODE_SUCCESS) {
+//    return ret;
+  }
+
+  taosArrayPush(pCmd->pDataBlocks, &pTableDataBlock);
+  tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows);
+
+  char *tokenBuf = calloc(1, 4096);
+
   while ((readLen = getline(&line, &n, fp)) != -1) {
-    // line[--readLen] = '\0';
+    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
-    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) line[--readLen] = 0;
+      line[--readLen] = 0;
-    if (readLen == 0) continue;  // fang, <= to ==
+    }
+
+    if (readLen == 0) {
+      continue;
+    }
 
     char *lineptr = line;
     strtolower(line, line);
 
-    len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, tinfo.precision, &code, tmpTokenBuf);
+    int32_t len =
+        tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, tinfo.precision, &code, tokenBuf);
     if (len <= 0 || pTableDataBlock->numOfParams > 0) {
       pSql->res.code = code;
-      return (-code);
+      break;
     }
 
     pTableDataBlock->size += len;
 
-    count++;
+    if (++count >= maxRows) {
-    nrows++;
+      break;
-    if (count >= maxRows) {
+    }
-      if ((code = doPackSendDataBlock(pSql, count, pTableDataBlock)) != TSDB_CODE_SUCCESS) {
-        return -code;
   }
 
-      pTableDataBlock = pCmd->pDataBlocks->pData[0];
+  tfree(tokenBuf);
-      pTableDataBlock->size = sizeof(SSubmitBlk);
+  free(line);
-      pTableDataBlock->rowSize = tinfo.rowSize;
-
-      numOfRows += pSql->res.numOfRows;
-      pSql->res.numOfRows = 0;
-      count = 0;
-    }
-  }
 
   if (count > 0) {
     if ((code = doPackSendDataBlock(pSql, count, pTableDataBlock)) != TSDB_CODE_SUCCESS) {
-      return -code;
+      pParentSql->res.code = code;
     }
 
-    numOfRows += pSql->res.numOfRows;
+  } else {
-    pSql->res.numOfRows = 0;
+    taos_free_result(pSql);
+    tfree(pSupporter);
+    fclose(fp);
+
+    pParentSql->fp = pParentSql->fetchFp;
+
+    // all data has been sent to vnode, call user function
+    int32_t v = (pParentSql->res.code != TSDB_CODE_SUCCESS) ? pParentSql->res.code : pParentSql->res.numOfRows;
+    (*pParentSql->fp)(pParentSql->param, pParentSql, v);
   }
-
-  if (line) tfree(line);
-
-  return numOfRows;
 }
 
-void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
+void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
   SSqlCmd *pCmd = &pSql->cmd;
   if (pCmd->command != TSDB_SQL_INSERT) {
     return;
   }
 
-  SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+  assert(pCmd->dataSourceType == DATA_FROM_DATA_FILE  && strlen(pCmd->payload) != 0);
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  STableDataBlocks *pDataBlock = NULL;
+  SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport));
-  int32_t           affected_rows = 0;
+  SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL);
 
-  assert(pCmd->dataSourceType == DATA_FROM_DATA_FILE && pCmd->pDataBlocks != NULL);
+  pNew->cmd.pDataBlocks = taosArrayInit(4, POINTER_BYTES);
-  SDataBlockList *pDataBlockList = pCmd->pDataBlocks;
-  pCmd->pDataBlocks = NULL;
-
-  char path[PATH_MAX] = {0};
-
-  for (int32_t i = 0; i < pDataBlockList->nSize; ++i) {
-    pDataBlock = pDataBlockList->pData[i];
-    if (pDataBlock == NULL) {
-      continue;
-    }
-
-    if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) {
-      tscError("%p failed to malloc when insert file", pSql);
-      continue;
-    }
   pCmd->count = 1;
 
-    tstrncpy(path, pDataBlock->filename, sizeof(path));
+  FILE *fp = fopen(pCmd->payload, "r");
-
-    FILE *fp = fopen(path, "r");
   if (fp == NULL) {
-      tscError("%p failed to open file %s to load data from file, reason:%s", pSql, path, strerror(errno));
+    pSql->res.code = TAOS_SYSTEM_ERROR(errno);
-      continue;
+    tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));
+
+    tfree(pSupporter)
+    tscQueueAsyncRes(pSql);
+
+    return;
   }
 
-    tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
+  pSupporter->pSql = pSql;
-    memset(pDataBlock->pData, 0, pDataBlock->nAllocSize);
+  pSupporter->fp = fp;
 
-    int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo);
+  parseFileSendDataBlock(pSupporter, pNew, 0);
-    if (ret != TSDB_CODE_SUCCESS) {
-      tscError("%p get meter meta failed, abort", pSql);
-      continue;
-    }
-
-    char *tmpTokenBuf = calloc(1, 4096);  // used for deleting Escape character: \\, \', \"
-    if (NULL == tmpTokenBuf) {
-      tscError("%p calloc failed", pSql);
-      continue;
-    }
-
-    int nrows = tscInsertDataFromFile(pSql, fp, tmpTokenBuf);
-    free(tmpTokenBuf);
-
-    pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
-
-    if (nrows < 0) {
-      fclose(fp);
-      tscTrace("%p no records(%d) in file %s", pSql, nrows, path);
-      continue;
-    }
-
-    fclose(fp);
-    affected_rows += nrows;
-
-    tscTrace("%p Insert data %d records from file %s", pSql, nrows, path);
-  }
-
-  pSql->res.numOfRows = affected_rows;
-
-  // all data have been submit to vnode, release data blocks
-  pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
-  tscDestroyBlockArrayList(pDataBlockList);
 }

src/client/src/tscPrepare.c

@@ -331,8 +331,9 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
     binded = pCmd->batchSize / 2;
   }
 
-  for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
+  size_t size = taosArrayGetSize(pCmd->pDataBlocks);
-    STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
+  for (int32_t i = 0; i < size; ++i) {
+    STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i);
     uint32_t          totalDataSize = pBlock->size - sizeof(SSubmitBlk);
     uint32_t          dataSize = totalDataSize / alloced;
     assert(dataSize * alloced == totalDataSize);
@@ -370,8 +371,9 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
     return TSDB_CODE_SUCCESS;
   }
 
-  for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
+  size_t total = taosArrayGetSize(pCmd->pDataBlocks);
-    STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
+  for (int32_t i = 0; i < total; ++i) {
+    STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i);
 
     uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
     pBlock->size += totalDataSize / alloced;
@@ -395,8 +397,10 @@ static int insertStmtReset(STscStmt* pStmt) {
   SSqlCmd* pCmd = &pStmt->pSql->cmd;
   if (pCmd->batchSize > 2) {
     int32_t alloced = (pCmd->batchSize + 1) / 2;
-    for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) {
+
-      STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i];
+    size_t size = taosArrayGetSize(pCmd->pDataBlocks);
+    for (int32_t i = 0; i < size; ++i) {
+      STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i);
 
       uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk);
       pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced;
@@ -424,14 +428,14 @@ static int insertStmtExecute(STscStmt* stmt) {
   STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   assert(pCmd->numOfClause == 1);
 
-  if (pCmd->pDataBlocks->nSize > 0) {
+  if (taosArrayGetSize(pCmd->pDataBlocks) > 0) {
     // merge according to vgid
     int code = tscMergeTableDataBlocks(stmt->pSql, pCmd->pDataBlocks);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
 
-    STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0];
+    STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0);
     code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock);
     if (code != TSDB_CODE_SUCCESS) {
       return code;

src/client/src/tscSQLParser.c

@@ -1483,6 +1483,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
   const char* msg6 = "function applied to tags not allowed";
   const char* msg7 = "normal table can not apply this function";
   const char* msg8 = "multi-columns selection does not support alias column name";
+  const char* msg9 = "invalid function";
 
   switch (optr) {
     case TK_COUNT: {
@@ -1683,7 +1684,9 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
       bool requireAllFields = (pItem->pNode->pParam == NULL);
 
       int16_t functionID = 0;
-      changeFunctionID(optr, &functionID);
+      if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg9);
+      }
 
       if (!requireAllFields) {
         if (pItem->pNode->pParam->nExpr < 1) {
@@ -3183,10 +3186,22 @@ static bool isValidExpr(tSQLExpr* pLeft, tSQLExpr* pRight, int32_t optr) {
    *
    * However, columnA < 4+12 is valid
    */
-  if ((pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_AVG_IRATE) ||
+  if (pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_AVG_IRATE) {
-      (pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_AVG_IRATE) ||
+    return false;
-      (pLeft->nSQLOptr >= TK_BOOL && pLeft->nSQLOptr <= TK_BINARY && pRight->nSQLOptr >= TK_BOOL &&
+  }
-       pRight->nSQLOptr <= TK_BINARY)) {
+
+  if (pRight == NULL) {
+    return true;
+  }
+
+  if (pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_AVG_IRATE) {
+    return false;
+  }
+  
+  if (pLeft->nSQLOptr >= TK_BOOL
+    && pLeft->nSQLOptr <= TK_BINARY
+    && pRight->nSQLOptr >= TK_BOOL
+    && pRight->nSQLOptr <= TK_BINARY) {
     return false;
   }
 
@@ -3759,13 +3774,17 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p
   if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     SColumnIndex index = {0};
 
-    getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index);
+    if (getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
+      tscError("%p: invalid column name (left)", pQueryInfo);
+    }
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   
     index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
     tscColumnListInsert(pTableMetaInfo->tagColList, &index);
   
-    getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index);
+    if (getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
+      tscError("%p: invalid column name (right)", pQueryInfo);
+    }
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   
     index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
@@ -4463,7 +4482,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
 
-    SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) (pCmd->payload + tsRpcHeadSize);
+    SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
     pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId);
     pUpdateMsg->tid = htonl(pTableMeta->sid);
     pUpdateMsg->uid = htobe64(pTableMeta->uid);

src/client/src/tscServer.c

@@ -176,18 +176,16 @@ int tscSendMsgToServer(SSqlObj *pSql) {
   
   char *pMsg = rpcMallocCont(pCmd->payloadLen);
   if (NULL == pMsg) {
-    tscError("%p msg:%s malloc fail", pSql, taosMsg[pSql->cmd.msgType]);
+    tscError("%p msg:%s malloc failed", pSql, taosMsg[pSql->cmd.msgType]);
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  if (pSql->cmd.command < TSDB_SQL_MGMT) {
+  // set the mgmt ip list
-    memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen);
+  if (pSql->cmd.command >= TSDB_SQL_MGMT) {
-  } else {
     pSql->ipList = tscMgmtIpSet;
-    memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
   }
 
-  // tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
+  memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
 
   SRpcMsg rpcMsg = {
       .msgType = pSql->cmd.msgType,
@@ -197,14 +195,18 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .code    = 0
   };
 
-  pSql->pRpcCtx = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
+  // NOTE: the rpc context should be acquired before sending data to server.
+  // Otherwise, the pSql object may have been released already during the response function, which is
+  // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
+  // cause crash.
+  /*pSql->pRpcCtx = */rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
   return TSDB_CODE_SUCCESS;
 }
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   SSqlObj *pSql = (SSqlObj *)rpcMsg->handle;
   if (pSql == NULL || pSql->signature != pSql) {
-    tscError("%p sql is already released", pSql->signature);
+    tscError("%p sql is already released", pSql);
     return;
   }
 
@@ -222,8 +224,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
 
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
   if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) {
-    tscTrace("%p sqlObj needs to be released or DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command,
+    tscTrace("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
-             pObj, pObj->signature);
+        pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
 
     tscFreeSqlObj(pSql);
     rpcFreeCont(rpcMsg->pCont);
@@ -449,18 +451,11 @@ void tscKillSTableQuery(SSqlObj *pSql) {
 }
 
 int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
-  char *pMsg, *pStart;
+  SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
-
-  pStart = pSql->cmd.payload + tsRpcHeadSize;
-  pMsg = pStart;
-
-  SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *)pMsg;
   pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
-  pMsg += sizeof(pSql->res.qhandle);
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   pRetrieveMsg->free = htons(pQueryInfo->type);
-  pMsg += sizeof(pQueryInfo->type);
 
   // todo valid the vgroupId at the client side
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@@ -475,11 +470,11 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
   }
 
-  pMsg += sizeof(SRetrieveTableMsg);
+  pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg);
-  
-  pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen);
-  
   pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH;
+
+  pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -487,30 +482,30 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
   
-  char* pMsg = pSql->cmd.payload + tsRpcHeadSize;
+  char* pMsg = pSql->cmd.payload;
   
   // NOTE: shell message size should not include SMsgDesc
   int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc);
-  
-  SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
-  
-  pMsgDesc->numOfVnodes = htonl(1);       //todo set the right number of vnodes
-  pMsg += sizeof(SMsgDesc);
-  
-  SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
   int32_t vgId = pTableMeta->vgroupInfo.vgId;
 
+  SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
+  pMsgDesc->numOfVnodes = htonl(1); // always one vnode
+
+  pMsg += sizeof(SMsgDesc);
+  SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
+
   pShellMsg->header.vgId = htonl(vgId);
-  pShellMsg->header.contLen = htonl(size);
+  pShellMsg->header.contLen = htonl(size);      // the length not includes the size of SMsgDesc
   pShellMsg->length = pShellMsg->header.contLen;
   
-  pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit);  // number of meters to be inserted
+  pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit);  // number of tables to be inserted
 
   // pSql->cmd.payloadLen is set during copying data into payload
   pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
   tscSetDnodeIpList(pSql, &pTableMeta->vgroupInfo);
   
-  tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d numberOfIP:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes), pSql->ipList.numOfIps);
+  tscTrace("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit,
+      pSql->ipList.numOfIps);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -620,9 +615,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     return -1;
   }
 
-  char *pStart = pCmd->payload + tsRpcHeadSize;
+  SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload;
-
-  SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart;
 
   int32_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList);
   
@@ -821,7 +814,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   }
 
   // compressed ts block
-  pQueryMsg->tsOffset = htonl(pMsg - pStart);
+  pQueryMsg->tsOffset = htonl(pMsg - pCmd->payload);
   int32_t tsLen = 0;
   int32_t numOfBlocks = 0;
 
@@ -830,8 +823,18 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL);  // this query should not be sent
 
     // todo refactor
-    fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET);
+    if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
-    fread(pMsg, pBlockInfo->compLen, 1, pQueryInfo->tsBuf->f);
+      int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
+      tscError("%p: fseek failed: %s", pSql, tstrerror(code));
+      return code;
+    }
+
+    size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f);
+    if (s != pBlockInfo->compLen) {
+      int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
+      tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+      return code;
+    }
 
     pMsg += pBlockInfo->compLen;
     tsLen = pBlockInfo->compLen;
@@ -844,7 +847,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
   }
 
-  int32_t msgLen = pMsg - pStart;
+  int32_t msgLen = pMsg - pCmd->payload;
 
   tscTrace("%p msg built success,len:%d bytes", pSql, msgLen);
   pCmd->payloadLen = msgLen;
@@ -1286,10 +1289,12 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
   SSqlCmd* pCmd = &pSql->cmd;
   pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL;
   
-  SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) (pCmd->payload + tsRpcHeadSize);
+  SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
   pCmd->payloadLen = htonl(pUpdateMsg->head.contLen);
+
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
   tscSetDnodeIpList(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo);
 
   return TSDB_CODE_SUCCESS;
@@ -1552,150 +1557,6 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 //}
 
 int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
-
-#if 0
-  SSuperTableMetaMsg *pMetaMsg;
-  char *          pMsg, *pStart;
-  int             msgLen = 0;
-  int             tableIndex = 0;
-
-  SSqlCmd *   pCmd = &pSql->cmd;
-  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
-
-  STagCond *pTagCond = &pQueryInfo->tagCond;
-
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
-
-  int32_t size = tscEstimateMetricMetaMsgSize(pCmd);
-  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
-    tscError("%p failed to malloc for metric meter msg", pSql);
-    return -1;
-  }
-
-  pStart = pCmd->payload + tsRpcHeadSize;
-  pMsg = pStart;
-
-  SMgmtHead *pMgmt = (SMgmtHead *)pMsg;
-  tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pMgmt->db);
-
-  pMsg += sizeof(SMgmtHead);
-
-  pMetaMsg = (SSuperTableMetaMsg *)pMsg;
-  pMetaMsg->numOfTables = htonl(pQueryInfo->numOfTables);
-
-  pMsg += sizeof(SSuperTableMetaMsg);
-
-  int32_t offset = pMsg - (char *)pMetaMsg;
-  pMetaMsg->join = htonl(offset);
-
-  // todo refactor
-  pMetaMsg->joinCondLen = htonl((TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2);
-
-  memcpy(pMsg, pTagCond->joinInfo.left.tableId, TSDB_TABLE_ID_LEN);
-  pMsg += TSDB_TABLE_ID_LEN;
-
-  *(int16_t *)pMsg = pTagCond->joinInfo.left.tagCol;
-  pMsg += sizeof(int16_t);
-
-  memcpy(pMsg, pTagCond->joinInfo.right.tableId, TSDB_TABLE_ID_LEN);
-  pMsg += TSDB_TABLE_ID_LEN;
-
-  *(int16_t *)pMsg = pTagCond->joinInfo.right.tagCol;
-  pMsg += sizeof(int16_t);
-
-  for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
-    pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
-    uint64_t uid = pTableMetaInfo->pTableMeta->uid;
-
-    offset = pMsg - (char *)pMetaMsg;
-    pMetaMsg->metaElem[i] = htonl(offset);
-
-    SSuperTableMetaElemMsg *pElem = (SSuperTableMetaElemMsg *)pMsg;
-    pMsg += sizeof(SSuperTableMetaElemMsg);
-
-    // convert to unicode before sending to mnode for metric query
-    int32_t condLen = 0;
-    if (pTagCond->numOfTagCond > 0) {
-      SCond *pCond = tsGetSTableQueryCond(pTagCond, uid);
-      if (pCond != NULL && pCond->cond != NULL) {
-        condLen = strlen(pCond->cond) + 1;
-
-        bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE);
-        if (!ret) {
-          tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCond(pTagCond, uid));
-          return 0;
-        }
-      }
-    }
-
-    pElem->condLen = htonl(condLen);
-
-    offset = pMsg - (char *)pMetaMsg;
-    pElem->cond = htonl(offset);
-    pMsg += condLen * TSDB_NCHAR_SIZE;
-
-    pElem->rel = htons(pTagCond->relType);
-    if (pTagCond->tbnameCond.uid == uid) {
-      offset = pMsg - (char *)pMetaMsg;
-
-      pElem->tableCond = htonl(offset);
-      
-      uint32_t len = 0;
-      if (pTagCond->tbnameCond.cond != NULL) {
-        len = strlen(pTagCond->tbnameCond.cond);
-        memcpy(pMsg, pTagCond->tbnameCond.cond, len);
-      }
-      
-      pElem->tableCondLen = htonl(len);
-      pMsg += len;
-    }
-
-    SSqlGroupbyExpr *pGroupby = &pQueryInfo->groupbyExpr;
-
-    if (pGroupby->tableIndex != i && pGroupby->numOfGroupCols > 0) {
-      pElem->orderType = 0;
-      pElem->orderIndex = 0;
-      pElem->numOfGroupCols = 0;
-    } else {
-      pElem->numOfGroupCols = htons(pGroupby->numOfGroupCols);
-      for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) {
-        pElem->tagCols[j] = htons(pTableMetaInfo->tagColumnIndex[j]);
-      }
-
-      if (pGroupby->numOfGroupCols != 0) {
-        pElem->orderIndex = htons(pGroupby->orderIndex);
-        pElem->orderType = htons(pGroupby->orderType);
-        offset = pMsg - (char *)pMetaMsg;
-
-        pElem->groupbyTagColumnList = htonl(offset);
-        for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) {
-          SColIndex *pCol = &pQueryInfo->groupbyExpr.columnInfo[j];
-          SColIndex *pDestCol = (SColIndex *)pMsg;
-
-          pDestCol->colIdxInBuf = 0;
-          pDestCol->colIndex = htons(pCol->colIndex);
-          pDestCol->colId = htons(pDestCol->colId);
-          pDestCol->flag = htons(pDestCol->flag);
-          strncpy(pDestCol->name, pCol->name, tListLen(pCol->name));
-
-          pMsg += sizeof(SColIndex);
-        }
-      }
-    }
-
-    strcpy(pElem->tableId, pTableMetaInfo->name);
-    pElem->numOfTags = htons(pTableMetaInfo->numOfTags);
-
-    int16_t len = pMsg - (char *)pElem;
-    pElem->elemLen = htons(len);  // redundant data for integrate check
-  }
-
-  msgLen = pMsg - pStart;
-  pCmd->payloadLen = msgLen;
-  pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP;
-  assert(msgLen + minMsgSize() <= size);
-#endif
-  
   SSqlCmd *pCmd = &pSql->cmd;
   
   char* pMsg = pCmd->payload;
@@ -1775,7 +1636,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  if (pMetaMsg->numOfTags > TSDB_MAX_TAGS || pMetaMsg->numOfTags < 0) {
+  if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) {
     tscError("invalid numOfTags:%d", pMetaMsg->numOfTags);
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
@@ -1796,6 +1657,10 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
     pSchema->bytes = htons(pSchema->bytes);
     pSchema->colId = htons(pSchema->colId);
 
+    if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+      assert(i == 0);
+    }
+
     assert(pSchema->type >= TSDB_DATA_TYPE_BOOL && pSchema->type <= TSDB_DATA_TYPE_NCHAR);
     pSchema++;
   }

src/client/src/tscSql.c

@@ -133,8 +133,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     return NULL;
   }
 
-  // tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid
+  tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
-  tsInsertHeadSize = tsRpcHeadSize + sizeof(SMsgDesc) + sizeof(SSubmitMsg);
   return pSql;
 }
 

src/client/src/tscSub.c

@@ -308,11 +308,9 @@ void tscSaveSubscriptionProgress(void* sub) {
 
   char path[256];
   sprintf(path, "%s/subscribe", tsDataDir);
-  if (access(path, 0) != 0) {
+  if (tmkdir(path, 0777) != 0) {
-    if (mkdir(path, 0777) != 0 && errno != EEXIST) {
     tscError("failed to create subscribe dir: %s", path);
   }
-  }
 
   sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
   FILE* fp = fopen(path, "w+");

src/client/src/tscSubquery.c

@@ -180,6 +180,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
   getTmpfilePath("join-", pSupporter->path);
   pSupporter->f = fopen(pSupporter->path, "w");
 
+  // todo handle error
   if (pSupporter->f == NULL) {
     tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno));
   }
@@ -234,7 +235,7 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
 /*
  * launch secondary stage query to fetch the result that contains timestamp in set
  */
-static int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
+static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
   int32_t         numOfSub = 0;
   SJoinSupporter* pSupporter = NULL;
   
@@ -249,7 +250,7 @@ static int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
   assert(numOfSub > 0);
   
   // scan all subquery, if one sub query has only ts, ignore it
-  tscTrace("%p start to launch secondary subquery, total:%d, only:%d needs to query", pSql, pSql->numOfSubs, numOfSub);
+  tscTrace("%p start to launch secondary subqueries, total:%d, only:%d needs to query", pSql, pSql->numOfSubs, numOfSub);
 
   //the subqueries that do not actually launch the secondary query to virtual node is set as completed.
   SSubqueryState* pState = pSupporter->pState;
@@ -451,7 +452,7 @@ static UNUSED_FUNC void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupport
       freeJoinSubqueryObj(pParentSql);
     } else {
       updateQueryTimeRange(pParentQueryInfo, &win);
-      tscLaunchSecondPhaseSubqueries(pParentSql);
+      tscLaunchRealSubqueries(pParentSql);
     }
 }
 
@@ -851,7 +852,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
   // launch the query the retrieve actual results from vnode along with the filtered timestamp
   SQueryInfo* pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
   updateQueryTimeRange(pPQueryInfo, &win);
-  tscLaunchSecondPhaseSubqueries(pParentSql);
+  tscLaunchRealSubqueries(pParentSql);
 }
 
 static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfRows) {
@@ -1159,7 +1160,6 @@ static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code);
 
 static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj);
 
-// todo merge with callback
 int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) {
   SSqlCmd *   pCmd = &pSql->cmd;
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
@@ -1302,7 +1302,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
   pState->numOfTotal = pQueryInfo->numOfTables;
   pState->numOfRemain = pState->numOfTotal;
 
-  tscTrace("%p start launch subquery, total:%d", pSql, pQueryInfo->numOfTables);
+  tscTrace("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables);
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i);
     
@@ -1378,6 +1378,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   if (ret != 0) {
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscQueueAsyncRes(pSql);
+    tfree(pMemoryBuf);
     return ret;
   }
   
@@ -1729,7 +1730,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
                                pRes->numOfRows, pQueryInfo->groupbyExpr.orderType);
     if (ret != 0) { // set no disk space error info, and abort retry
       tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE);
-      pthread_mutex_unlock(&trsupport->queryMutex);
       
     } else if (pRes->completed) {
       tscAllDataRetrievedFromDnode(trsupport, pSql);
@@ -1848,8 +1848,6 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
 static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) {
   SInsertSupporter *pSupporter = (SInsertSupporter *)param;
   SSqlObj* pParentObj = pSupporter->pSql;
-  SSqlCmd* pParentCmd = &pParentObj->cmd;
-  
   SSubqueryState* pState = pSupporter->pState;
 
   // record the total inserted rows
@@ -1864,7 +1862,6 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
     pParentObj->res.code = pSql->res.code;
   }
 
-  // it is not the initial sqlObj, free it
   taos_free_result(tres);
   tfree(pSupporter);
 
@@ -1876,7 +1873,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
 
   // release data block data
   tfree(pState);
-  pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks);
+//  pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks);
   
   // restore user defined fp
   pParentObj->fp = pParentObj->fetchFp;
@@ -1889,12 +1886,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
   
-  SDataBlockList *pDataBlocks = pCmd->pDataBlocks;
+  size_t size = taosArrayGetSize(pCmd->pDataBlocks);
-  pSql->pSubs = calloc(pDataBlocks->nSize, POINTER_BYTES);
+  assert(size > 0);
-  pSql->numOfSubs = pDataBlocks->nSize;
+
-  assert(pDataBlocks->nSize > 0);
+  pSql->pSubs = calloc(size, POINTER_BYTES);
+  pSql->numOfSubs = size;
+
+  tscTrace("%p submit data to %zu vnode(s)", pSql, size);
 
-  tscTrace("%p submit data to %d vnode(s)", pSql, pDataBlocks->nSize);
   SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
   pState->numOfTotal = pSql->numOfSubs;
   pState->numOfRemain = pSql->numOfSubs;
@@ -1920,12 +1919,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
     pNew->fetchFp = pNew->fp;
     pSql->pSubs[numOfSub] = pNew;
 
-    pRes->code = tscCopyDataBlockToPayload(pNew, pDataBlocks->pData[numOfSub++]);
+    STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, numOfSub);
+    pRes->code = tscCopyDataBlockToPayload(pNew, pTableDataBlock);
     if (pRes->code == TSDB_CODE_SUCCESS) {
       tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub);
+      numOfSub++;
     } else {
-      tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
+      tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%zu, code:%s", pSql, numOfSub,
-               pDataBlocks->nSize, tstrerror(pRes->code));
+               size, tstrerror(pRes->code));
       goto _error;
     }
   }
@@ -1943,6 +1944,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
     tscProcessSql(pSub);
   }
 
+  pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
   return TSDB_CODE_SUCCESS;
 
   _error:

src/client/src/tscSystem.c

@@ -314,22 +314,6 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       }
       break;
 
-    case TSDB_OPTION_SOCKET_TYPE:
-      cfg = taosGetConfigOption("sockettype");
-      assert(cfg != NULL);
-    
-      if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-//        if (strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_UDP) != 0 && strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_TCP) != 0) {
-//          tscError("only 'tcp' or 'udp' allowed for configuring the socket type");
-//          return -1;
-//        }
-
-        tstrncpy(tsSocketType, pStr, sizeof(tsSocketType));
-        cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
-        tscPrint("socket type is set:%s", tsSocketType);
-      }
-      break;
-
     default:
       // TODO return the correct error code to client in the format for taos_errstr()
       tscError("Invalid option %d", option);

src/client/src/tscUtil.c

@@ -13,8 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tscUtil.h"
-#include "hash.h"
 #include "os.h"
 #include "qast.h"
 #include "taosmsg.h"
@@ -29,6 +27,8 @@
 #include "ttimer.h"
 #include "ttokendef.h"
 #include "tscLog.h"
+#include "tscUtil.h"
+#include "hash.h"
 
 static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
 static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache);
@@ -428,48 +428,18 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
   return param;
 }
 
-SDataBlockList* tscCreateBlockArrayList() {
+void*  tscDestroyBlockArrayList(SArray* pDataBlockList) {
-  const int32_t DEFAULT_INITIAL_NUM_OF_BLOCK = 16;
+  if (pDataBlockList == NULL) {
-
-  SDataBlockList* pDataBlockArrayList = calloc(1, sizeof(SDataBlockList));
-  if (pDataBlockArrayList == NULL) {
     return NULL;
   }
 
-  pDataBlockArrayList->nAlloc = DEFAULT_INITIAL_NUM_OF_BLOCK;
+  size_t size = taosArrayGetSize(pDataBlockList);
-  pDataBlockArrayList->pData = calloc(1, POINTER_BYTES * pDataBlockArrayList->nAlloc);
+  for (int32_t i = 0; i < size; i++) {
-  if (pDataBlockArrayList->pData == NULL) {
+    void* d = taosArrayGetP(pDataBlockList, i);
-    free(pDataBlockArrayList);
+    tscDestroyDataBlock(d);
-    return NULL;
   }
 
-  return pDataBlockArrayList;
+  taosArrayDestroy(pDataBlockList);
-}
-
-void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks) {
-  if (pList->nSize >= pList->nAlloc) {
-    pList->nAlloc = (pList->nAlloc) << 1U;
-    pList->pData = realloc(pList->pData, POINTER_BYTES * (size_t)pList->nAlloc);
-
-    // reset allocated memory
-    memset(pList->pData + pList->nSize, 0, POINTER_BYTES * (pList->nAlloc - pList->nSize));
-  }
-
-  pList->pData[pList->nSize++] = pBlocks;
-}
-
-void* tscDestroyBlockArrayList(SDataBlockList* pList) {
-  if (pList == NULL) {
-    return NULL;
-  }
-
-  for (int32_t i = 0; i < pList->nSize; i++) {
-    tscDestroyDataBlock(pList->pData[i]);
-  }
-
-  tfree(pList->pData);
-  tfree(pList);
-
   return NULL;
 }
 
@@ -484,7 +454,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
 
   // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache
   if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) {
-    strcpy(pTableMetaInfo->name, pDataBlock->tableId);
+    tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
     taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
 
     pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta);
@@ -497,31 +467,32 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
    * the dataBlock only includes the RPC Header buffer and actual submit message body, space for digest needs
    * additional space.
    */
-  int ret = tscAllocPayload(pCmd, pDataBlock->nAllocSize + 100);
+  int ret = tscAllocPayload(pCmd, pDataBlock->size + 100);
   if (TSDB_CODE_SUCCESS != ret) {
     return ret;
   }
 
-  memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->nAllocSize);
+  assert(pDataBlock->size <= pDataBlock->nAllocSize);
+  memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->size);
 
   /*
    * the payloadLen should be actual message body size
    * the old value of payloadLen is the allocated payload size
    */
-  pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize;
+  pCmd->payloadLen = pDataBlock->size;
 
-  assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0);
+  assert(pCmd->allocSize >= pCmd->payloadLen + 100 && pCmd->payloadLen > 0);
   return TSDB_CODE_SUCCESS;
 }
 
-void tscFreeUnusedDataBlocks(SDataBlockList* pList) {
+//void tscFreeUnusedDataBlocks(SDataBlockList* pList) {
-  /* release additional memory consumption */
+//  /* release additional memory consumption */
-  for (int32_t i = 0; i < pList->nSize; ++i) {
+//  for (int32_t i = 0; i < pList->nSize; ++i) {
-    STableDataBlocks* pDataBlock = pList->pData[i];
+//    STableDataBlocks* pDataBlock = pList->pData[i];
-    pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size);
+//    pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size);
-    pDataBlock->nAllocSize = (uint32_t)pDataBlock->size;
+//    pDataBlock->nAllocSize = (uint32_t)pDataBlock->size;
-  }
+//  }
-}
+//}
 
 /**
  * create the in-memory buffer for each table to keep the submitted data block
@@ -568,7 +539,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size,
+int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size,
                                 int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
                                 STableDataBlocks** dataBlocks) {
   *dataBlocks = NULL;
@@ -585,7 +556,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
     }
 
     taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES);
-    tscAppendDataBlock(pDataBlockList, *dataBlocks);
+    taosArrayPush(pDataBlockList, dataBlocks);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -634,14 +605,18 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
   return len;
 }
 
-int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) {
+int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
   SSqlCmd* pCmd = &pSql->cmd;
 
-  void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+  // the expanded size when a row data is converted to SDataRow format
-  SDataBlockList* pVnodeDataBlockList = tscCreateBlockArrayList();
+  const int32_t MAX_EXPAND_SIZE = TD_DATA_ROW_HEAD_SIZE + TYPE_BYTES[TSDB_DATA_TYPE_BINARY];
 
-  for (int32_t i = 0; i < pTableDataBlockList->nSize; ++i) {
+  void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
-    STableDataBlocks* pOneTableBlock = pTableDataBlockList->pData[i];
+  SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
+
+  size_t total = taosArrayGetSize(pTableDataBlockList);
+  for (int32_t i = 0; i < total; ++i) {
+    STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i);
 
     STableDataBlocks* dataBuf = NULL;
     
@@ -655,7 +630,9 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
       return ret;
     }
 
-    int64_t destSize = dataBuf->size + pOneTableBlock->size + pOneTableBlock->size*sizeof(int32_t)*2;
+    SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
+    int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * MAX_EXPAND_SIZE;
+
     if (dataBuf->nAllocSize < destSize) {
       while (dataBuf->nAllocSize < destSize) {
         dataBuf->nAllocSize = dataBuf->nAllocSize * 1.5;
@@ -676,26 +653,30 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
       }
     }
 
-    SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
     tscSortRemoveDataBlockDupRows(pOneTableBlock);
-
+    char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
-    char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
     
     tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId,
-        pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(e));
+        pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey));
 
-    int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + sizeof(int32_t) * 2);
+
+    int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + MAX_EXPAND_SIZE);
 
     pBlocks->tid = htonl(pBlocks->tid);
     pBlocks->uid = htobe64(pBlocks->uid);
     pBlocks->sversion = htonl(pBlocks->sversion);
     pBlocks->numOfRows = htons(pBlocks->numOfRows);
 
-    pBlocks->len = htonl(len);
-    
     // erase the empty space reserved for binary data
-    len = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
+    int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
-    dataBuf->size += (len + sizeof(SSubmitBlk));
+    assert(finalLen <= len);
+
+    dataBuf->size += (finalLen + sizeof(SSubmitBlk));
+    assert(dataBuf->size <= dataBuf->nAllocSize);
+
+    // the length does not include the SSubmitBlk structure
+    pBlocks->len = htonl(finalLen);
+
     dataBuf->numOfTables += 1;
   }
 
@@ -704,7 +685,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
   // free the table data blocks;
   pCmd->pDataBlocks = pVnodeDataBlockList;
 
-  tscFreeUnusedDataBlocks(pCmd->pDataBlocks);
+//  tscFreeUnusedDataBlocks(pCmd->pDataBlocks);
   taosHashCleanup(pVnodeDataBlockHashList);
 
   return TSDB_CODE_SUCCESS;
@@ -1664,6 +1645,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
 
   SSqlCmd* pCmd = &pNew->cmd;
   pCmd->command = cmd;
+  pCmd->parseFinished = 1;
 
   if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
@@ -1724,6 +1706,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   pnCmd->numOfClause = 0;
   pnCmd->clauseIndex = 0;
   pnCmd->pDataBlocks = NULL;
+  pnCmd->parseFinished = 1;
 
   if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
@@ -1897,7 +1880,7 @@ void tscDoQuery(SSqlObj* pSql) {
   }
 
   if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) {
-    tscProcessMultiVnodesInsertFromFile(pSql);
+    tscProcessMultiVnodesImportFromFile(pSql);
   } else {
     SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
     uint16_t type = pQueryInfo->type;

src/common/inc/tglobal.h

@@ -20,37 +20,7 @@
 extern "C" {
 #endif
 
-extern char configDir[];
+// cluster
-extern char tsVnodeDir[];
-extern char tsDnodeDir[];
-extern char tsMnodeDir[];
-extern char tsDataDir[];
-extern char tsLogDir[];
-extern char tsScriptDir[];
-extern char tsOsName[];
-
-// system info
-extern int64_t tsPageSize;
-extern int64_t tsOpenMax;
-extern int64_t tsStreamMax;
-extern int32_t tsNumOfCores;
-extern int32_t tsAlternativeRole;
-extern float   tsTotalLogDirGB;
-extern float   tsTotalTmpDirGB;
-extern float   tsTotalDataDirGB;
-extern float   tsAvailLogDirGB;
-extern float   tsAvailTmpDirGB;
-extern float   tsAvailDataDirGB;
-extern float   tsMinimalLogDirGB;
-extern float   tsMinimalTmpDirGB;
-extern float   tsMinimalDataDirGB;
-extern int32_t tsEnableCoreFile;
-extern int32_t tsTotalMemoryMB;
-extern int32_t tsVersion;
-
-extern int32_t tscEmbedded;
-extern int64_t tsMsPerDay[3];
-
 extern char     tsFirst[];
 extern char     tsSecond[];
 extern char     tsLocalFqdn[];
@@ -59,21 +29,42 @@ extern uint16_t tsServerPort;
 extern uint16_t tsDnodeShellPort;
 extern uint16_t tsDnodeDnodePort;
 extern uint16_t tsSyncPort;
-
 extern int32_t  tsStatusInterval;
-extern int32_t tsShellActivityTimer;
-extern int32_t tsVnodePeerHBTimer;
-extern int32_t tsMgmtPeerHBTimer;
-extern int32_t tsTableMetaKeepTimer;
-
-extern float    tsNumOfThreadsPerCore;
-extern float    tsRatioOfQueryThreads;
-extern char     tsPublicIp[];
-extern char     tsPrivateIp[];
 extern int16_t  tsNumOfVnodesPerCore;
 extern int16_t  tsNumOfTotalVnodes;
-extern uint32_t tsPublicIpInt;
+extern int32_t  tsNumOfMnodes;
 
+// common
+extern int      tsRpcTimer;
+extern int      tsRpcMaxTime;
+extern int32_t  tsMaxConnections;
+extern int32_t  tsMaxShellConns;
+extern int32_t  tsShellActivityTimer;
+extern uint32_t tsMaxTmrCtrl;
+extern float    tsNumOfThreadsPerCore;
+extern float    tsRatioOfQueryThreads;
+extern int8_t   tsDaylight;
+extern char     tsTimezone[64];
+extern char     tsLocale[64];
+extern char     tsCharset[64];  // default encode string
+extern int32_t  tsEnableCoreFile;
+extern int32_t  tsCompressMsgSize;
+
+// client
+extern int32_t tsTableMetaKeepTimer;
+extern int32_t tsMaxSQLStringLen;
+extern int32_t tsTscEnableRecordSql;
+extern int32_t tsMaxNumOfOrderedResults;
+extern int32_t tsMinSlidingTime;
+extern int32_t tsMinIntervalTime;
+extern int32_t tsMaxStreamComputDelay;
+extern int32_t tsStreamCompStartDelay;
+extern int32_t tsStreamCompRetryDelay;
+extern float   tsStreamComputDelayRatio;  // the delayed computing ration of the whole time window
+extern int32_t tsProjectExecInterval;
+extern int64_t tsMaxRetentWindow;
+
+// db parameters in client
 extern int32_t tsCacheBlockSize;
 extern int32_t tsBlocksPerVnode;
 extern int32_t tsMaxTablePerVnode;
@@ -87,42 +78,16 @@ extern int16_t tsCompression;
 extern int16_t tsWAL;
 extern int32_t tsReplications;
 
-extern int16_t tsAffectedRowsMod;
+// balance
-extern int32_t tsNumOfMnodes;
+extern int32_t tsEnableBalance;
-extern int32_t tsMaxShellConns;
+extern int32_t tsAlternativeRole;
-extern int32_t tsMaxTables;
-
-extern char tsMqttBrokerAddress[];
-extern char tsMqttBrokerClientId[];
-
-extern int32_t tsMaxConnections;
-
 extern int32_t tsBalanceInterval;
 extern int32_t tsOfflineThreshold;
 extern int32_t tsMnodeEqualVnodeNum;
 
+// restful
 extern int32_t  tsEnableHttpModule;
-extern int32_t tsEnableMqttModule;
-extern int32_t tsEnableMonitorModule;
-
 extern int32_t  tsRestRowLimit;
-extern int32_t tsMaxSQLStringLen;
-extern int32_t tsCompressMsgSize;
-extern int32_t tsMaxNumOfOrderedResults;
-
-extern char tsSocketType[4];
-
-extern int32_t tsMinSlidingTime;
-extern int32_t tsMinIntervalTime;
-extern int32_t tsMaxStreamComputDelay;
-extern int32_t tsStreamCompStartDelay;
-extern int32_t tsStreamCompRetryDelay;
-extern float tsStreamComputDelayRatio;   // the delayed computing ration of the whole time window
-
-extern int     tsProjectExecInterval;
-extern int64_t tsMaxRetentWindow;
-
-extern char     tsHttpIp[];
 extern uint16_t tsHttpPort;
 extern int32_t  tsHttpCacheSessions;
 extern int32_t  tsHttpSessionExpire;
@@ -131,12 +96,54 @@ extern int32_t  tsHttpEnableCompress;
 extern int32_t  tsHttpEnableRecordSql;
 extern int32_t  tsTelegrafUseFieldNum;
 
-extern int32_t  tsTscEnableRecordSql;
+// mqtt
+extern int32_t tsEnableMqttModule;
+extern char    tsMqttBrokerAddress[];
+extern char    tsMqttBrokerClientId[];
 
+// monitor
+extern int32_t tsEnableMonitorModule;
 extern char    tsMonitorDbName[];
 extern char    tsInternalPass[];
 extern int32_t tsMonitorInterval;
 
+// internal
+extern int32_t tscEmbedded;
+extern char    configDir[];
+extern char    tsVnodeDir[];
+extern char    tsDnodeDir[];
+extern char    tsMnodeDir[];
+extern char    tsDataDir[];
+extern char    tsLogDir[];
+extern char    tsScriptDir[];
+extern int64_t tsMsPerDay[3];
+
+// system info
+extern char    tsOsName[];
+extern int64_t tsPageSize;
+extern int64_t tsOpenMax;
+extern int64_t tsStreamMax;
+extern int32_t tsNumOfCores;
+extern float   tsTotalLogDirGB;
+extern float   tsTotalTmpDirGB;
+extern float   tsTotalDataDirGB;
+extern float   tsAvailLogDirGB;
+extern float   tsAvailTmpDirGB;
+extern float   tsAvailDataDirGB;
+extern float   tsMinimalLogDirGB;
+extern float   tsMinimalTmpDirGB;
+extern float   tsMinimalDataDirGB;
+extern int32_t tsTotalMemoryMB;
+extern int32_t tsVersion;
+
+// build info
+extern char version[];
+extern char compatible_version[];
+extern char gitinfo[];
+extern char gitinfoOfInternal[];
+extern char buildinfo[];
+
+// log
 extern int32_t tsAsyncLog;
 extern int32_t tsNumOfLogLines;
 extern int32_t dDebugFlag;
@@ -155,22 +162,6 @@ extern int32_t debugFlag;
 extern int32_t odbcDebugFlag;
 extern int32_t qDebugFlag;
 
-extern uint32_t taosMaxTmrCtrl;
-
-extern int  tsRpcTimer;
-extern int  tsRpcMaxTime;
-extern int  tsUdpDelay;
-extern char version[];
-extern char compatible_version[];
-extern char gitinfo[];
-extern char gitinfoOfInternal[];
-extern char buildinfo[];
-
-extern int8_t tsDaylight;
-extern char tsTimezone[64];
-extern char tsLocale[64];
-extern char tsCharset[64];  // default encode string
-
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 void taosInitGlobalCfg();

src/common/src/tglobal.c

@@ -27,41 +27,7 @@
 #include "ttimezone.h"
 #include "tsync.h"
 
-char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
+// cluster
-char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
-char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
-char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
-char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
-char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
-char tsOsName[10] = "Linux";
-
-// system info, not configurable
-int64_t tsPageSize;
-int64_t tsOpenMax;
-int64_t tsStreamMax;
-int32_t tsNumOfCores = 1;
-int32_t tsAlternativeRole = 0;
-float   tsTotalTmpDirGB = 0;
-float   tsTotalDataDirGB = 0;
-float   tsAvailTmpDirGB = 0;
-float   tsAvailDataDirGB = 0;
-float   tsMinimalTmpDirGB = 0.1;
-float   tsMinimalDataDirGB = 0.5;
-int32_t tsTotalMemoryMB = 0;
-int32_t tsVersion = 0;
-int32_t tsEnableCoreFile = 0;
-
-// global, not configurable
-int32_t tscEmbedded = 0;
-
-/*
- * minimum scale for whole system, millisecond by default
- * for TSDB_TIME_PRECISION_MILLI: 86400000L
- *     TSDB_TIME_PRECISION_MICRO: 86400000000L
- *     TSDB_TIME_PRECISION_NANO:  86400000000000L
- */
-int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
-
 char     tsFirst[TSDB_EP_LEN] = {0};
 char     tsSecond[TSDB_EP_LEN] = {0};
 char     tsArbitrator[TSDB_EP_LEN] = {0};
@@ -71,78 +37,25 @@ uint16_t tsServerPort = 6030;
 uint16_t tsDnodeShellPort = 6030;  // udp[6035-6039] tcp[6035]
 uint16_t tsDnodeDnodePort = 6035;  // udp/tcp
 uint16_t tsSyncPort = 6040;
-
 int32_t  tsStatusInterval = 1;  // second
-int32_t tsShellActivityTimer = 3;     // second
-int32_t tsTableMetaKeepTimer = 7200;  // second
-int32_t tsRpcTimer = 300;
-int32_t tsRpcMaxTime = 600;      // seconds;
-
-float   tsNumOfThreadsPerCore = 1.0;
-float   tsRatioOfQueryThreads = 0.5;
 int16_t  tsNumOfVnodesPerCore = 8;
 int16_t  tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
-
-#ifdef _TD_ARM_32_
-int32_t tsMaxTablePerVnode = 100;
-#else
-int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
-#endif
-
-int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
-int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
-int16_t tsDaysPerFile    = TSDB_DEFAULT_DAYS_PER_FILE;
-int32_t tsDaysToKeep     = TSDB_DEFAULT_KEEP;
-int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
-int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
-int16_t tsCommitTime    = TSDB_DEFAULT_COMMIT_TIME;  // seconds
-int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
-int16_t tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
-int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
-int32_t tsReplications  = TSDB_DEFAULT_REPLICA_NUM;
-
-/**
- * Change the meaning of affected rows:
- * 0: affected rows not include those duplicate records
- * 1: affected rows include those duplicate records
- */
-int16_t tsAffectedRowsMod = 0;
 int32_t  tsNumOfMnodes = 3;
+
+// common
+int32_t tsRpcTimer = 300;
+int32_t tsRpcMaxTime = 600;  // seconds;
 int32_t tsMaxShellConns = 5000;
 int32_t tsMaxConnections = 5000;
-
+int32_t tsShellActivityTimer = 3;  // second
-int32_t tsBalanceInterval = 300;  // seconds
+float   tsNumOfThreadsPerCore = 1.0;
-int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
+float   tsRatioOfQueryThreads = 0.5;
-int32_t tsMnodeEqualVnodeNum = 4;
+int8_t  tsDaylight = 0;
-
+char    tsTimezone[64] = {0};
-int32_t tsEnableHttpModule = 1;
+char    tsLocale[TSDB_LOCALE_LEN] = {0};
-int32_t tsEnableMqttModule = 0;   // not finished yet, not started it by default
+char    tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
-int32_t tsEnableMonitorModule = 0;
+int32_t tsEnableCoreFile = 0;
-
+int32_t tsMaxBinaryDisplayWidth = 30;
-int32_t tsRestRowLimit = 10240;
-int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
-
-int32_t tsNumOfLogLines = 10000000;
-int32_t mDebugFlag = 135;
-int32_t sdbDebugFlag = 135;
-int32_t dDebugFlag = 135;
-int32_t vDebugFlag = 135;
-int32_t cDebugFlag = 135;
-int32_t jniDebugFlag = 131;
-int32_t odbcDebugFlag = 131;
-int32_t httpDebugFlag = 131;
-int32_t mqttDebugFlag = 131;
-int32_t monitorDebugFlag = 131;
-int32_t qDebugFlag = 131;
-int32_t rpcDebugFlag = 135;
-int32_t uDebugFlag = 131;
-int32_t debugFlag = 131;
-int32_t sDebugFlag = 135;
-int32_t tsdbDebugFlag = 135;
-
-// the maximum number of results for projection query on super table that are returned from
-// one virtual node, to order according to timestamp
-int32_t tsMaxNumOfOrderedResults = 100000;
 
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
@@ -154,8 +67,14 @@ int32_t tsMaxNumOfOrderedResults = 100000;
  */
 int32_t tsCompressMsgSize = -1;
 
-// use UDP by default[option: udp, tcp]
+// client
-char tsSocketType[4] = "udp";
+int32_t tsTableMetaKeepTimer = 7200;  // second
+int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
+int32_t tsTscEnableRecordSql = 0;
+
+// the maximum number of results for projection query on super table that are returned from
+// one virtual node, to order according to timestamp
+int32_t tsMaxNumOfOrderedResults = 100000;
 
 // 10 ms for sliding time, the value will changed in case of time precision changed
 int32_t tsMinSlidingTime = 10;
@@ -178,8 +97,36 @@ float tsStreamComputDelayRatio = 0.1;
 int32_t tsProjectExecInterval = 10000;   // every 10sec, the projection will be executed once
 int64_t tsMaxRetentWindow = 24 * 3600L;  // maximum time window tolerance
 
+// db parameters
+int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
+int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
+int16_t tsDaysPerFile    = TSDB_DEFAULT_DAYS_PER_FILE;
+int32_t tsDaysToKeep     = TSDB_DEFAULT_KEEP;
+int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
+int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
+int16_t tsCommitTime    = TSDB_DEFAULT_COMMIT_TIME;  // seconds
+int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
+int16_t tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
+int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
+int32_t tsReplications  = TSDB_DEFAULT_REPLICA_NUM;
+
+#ifdef _TD_ARM_32_
+  int32_t tsMaxTablePerVnode = 100;
+#else
+  int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
+#endif
+
+// balance
+int32_t tsEnableBalance = 1;
+int32_t tsAlternativeRole = 0;
+int32_t tsBalanceInterval = 300;  // seconds
+int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
+int32_t tsMnodeEqualVnodeNum = 4;
+
+// restful
+int32_t  tsEnableHttpModule = 1;
+int32_t  tsRestRowLimit = 10240;
 uint16_t tsHttpPort = 6020;  // only tcp, range tcp[6020]
-// uint16_t tsNginxPort = 6060;             //only tcp, range tcp[6060]
 int32_t  tsHttpCacheSessions = 100;
 int32_t  tsHttpSessionExpire = 36000;
 int32_t  tsHttpMaxThreads = 2;
@@ -187,21 +134,67 @@ int32_t tsHttpEnableCompress = 0;
 int32_t  tsHttpEnableRecordSql = 0;
 int32_t  tsTelegrafUseFieldNum = 0;
 
-int32_t  tsTscEnableRecordSql = 0;
+// mqtt
-uint32_t tsPublicIpInt = 0;
+int32_t tsEnableMqttModule = 0;  // not finished yet, not started it by default
+char    tsMqttBrokerAddress[128] = {0};
+char    tsMqttBrokerClientId[128] = {0};
 
+// monitor
+int32_t tsEnableMonitorModule = 0;
 char    tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char    tsInternalPass[] = "secretkey";
 int32_t tsMonitorInterval = 30;  // seconds
 
-int8_t tsDaylight = 0;
+// internal
-char tsTimezone[64] = {0};
+int32_t tscEmbedded = 0;
-char tsLocale[TSDB_LOCALE_LEN] = {0};
+char    configDir[TSDB_FILENAME_LEN] = "/etc/taos";
-char tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
+char    tsVnodeDir[TSDB_FILENAME_LEN] = {0};
-char tsMqttBrokerAddress[128] = {0}; 
+char    tsDnodeDir[TSDB_FILENAME_LEN] = {0};
-char tsMqttBrokerClientId[128] = {0};
+char    tsMnodeDir[TSDB_FILENAME_LEN] = {0};
+char    tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
+char    tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
 
-int32_t tsMaxBinaryDisplayWidth = 30;
+/*
+ * minimum scale for whole system, millisecond by default
+ * for TSDB_TIME_PRECISION_MILLI: 86400000L
+ *     TSDB_TIME_PRECISION_MICRO: 86400000000L
+ *     TSDB_TIME_PRECISION_NANO:  86400000000000L
+ */
+int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
+
+// system info
+char    tsOsName[10] = "Linux";
+int64_t tsPageSize;
+int64_t tsOpenMax;
+int64_t tsStreamMax;
+int32_t tsNumOfCores = 1;
+float   tsTotalTmpDirGB = 0;
+float   tsTotalDataDirGB = 0;
+float   tsAvailTmpDirGB = 0;
+float   tsAvailDataDirGB = 0;
+float   tsMinimalTmpDirGB = 0.1;
+float   tsMinimalDataDirGB = 0.5;
+int32_t tsTotalMemoryMB = 0;
+int32_t tsVersion = 0;
+
+// log
+int32_t tsNumOfLogLines = 10000000;
+int32_t mDebugFlag = 135;
+int32_t sdbDebugFlag = 135;
+int32_t dDebugFlag = 135;
+int32_t vDebugFlag = 135;
+int32_t cDebugFlag = 135;
+int32_t jniDebugFlag = 131;
+int32_t odbcDebugFlag = 131;
+int32_t httpDebugFlag = 131;
+int32_t mqttDebugFlag = 131;
+int32_t monitorDebugFlag = 131;
+int32_t qDebugFlag = 131;
+int32_t rpcDebugFlag = 135;
+int32_t uDebugFlag = 131;
+int32_t debugFlag = 131;
+int32_t sDebugFlag = 135;
+int32_t tsdbDebugFlag = 135;
 
 static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
 
@@ -426,6 +419,16 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "balance";
+  cfg.ptr = &tsEnableBalance;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "balanceInterval";
   cfg.ptr = &tsBalanceInterval;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -447,19 +450,9 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "affectedRowsMod";
-  cfg.ptr = &tsAffectedRowsMod;
-  cfg.valType = TAOS_CFG_VTYPE_INT16;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   // timer
   cfg.option = "maxTmrCtrl";
-  cfg.ptr = &taosMaxTmrCtrl;
+  cfg.ptr = &tsMaxTmrCtrl;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 8;
@@ -669,16 +662,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "ctime";
-  cfg.ptr = &tsCommitTime;
-  cfg.valType = TAOS_CFG_VTYPE_INT16;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_COMMIT_TIME;
-  cfg.maxValue = TSDB_MAX_COMMIT_TIME;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_SECOND;
-  taosInitConfigOption(cfg);
-
   cfg.option = "comp";
   cfg.ptr = &tsCompression;
   cfg.valType = TAOS_CFG_VTYPE_INT16;
@@ -729,17 +712,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
  
-  // socket type; udp by default
-  cfg.option = "sockettype";
-  cfg.ptr = tsSocketType;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = 3;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   cfg.option = "compressMsgSize";
   cfg.ptr = &tsCompressMsgSize;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -873,7 +845,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  
   cfg.option = "mqtt";
   cfg.ptr = &tsEnableMqttModule;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -894,16 +865,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "monitorDbName";
-  cfg.ptr = tsMonitorDbName;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   // http configs
   cfg.option = "httpCacheSessions";
   cfg.ptr = &tsHttpCacheSessions;
@@ -955,16 +916,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "httpEnableCompress";
-  cfg.ptr = &tsHttpEnableCompress;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   // debug flag
   cfg.option = "numOfLogLines";
   cfg.ptr = &tsNumOfLogLines;
@@ -1232,9 +1183,7 @@ bool taosCheckGlobalCfg() {
     taosGetFqdn(tsLocalFqdn);
   }
 
-  strcpy(tsLocalEp, tsLocalFqdn);
+  snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%d", tsLocalFqdn, tsServerPort);
-
-  snprintf(tsLocalEp + strlen(tsLocalEp), sizeof(tsLocalEp), ":%d", tsServerPort);
   uPrint("localEp is: %s", tsLocalEp);
 
   if (tsFirst[0] == 0) {

src/connector/jdbc/buildTDengine.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+ulimit -c unlimited
+
+function buildTDengine {    
+	cd /root/TDengine
+
+	git remote update
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "repo up-to-date"
+	else
+		echo "repo need to pull"        
+		git pull
+
+		LOCAL_COMMIT=`git rev-parse --short @`        
+		cd /root/TDengine/debug        
+		rm -rf /root/TDengine/debug/*        
+		cmake ..
+		make > /dev/null
+		make install
+	fi
+}
+
+function restartTaosd {
+	systemctl stop taosd
+	pkill -KILL -x taosd
+	sleep 10
+	
+    	logDir=`grep 'logDir' /etc/taos/taos.cfg|awk 'END{print $2}'`
+    	dataDir=`grep 'dataDir' /etc/taos/taos.cfg|awk '{print $2}'`    
+    
+	rm -rf $logDir/*
+	rm -rf $dataDir/*
+	
+	taosd 2>&1 > /dev/null &
+	sleep 10
+}
+
+buildTDengine
+restartTaosd

src/connector/jdbc/src/test/java/com/taosdata/jdbc/AsyncSubscribeTest.java

@@ -12,7 +12,7 @@ import java.util.Properties;
 
 import static org.junit.Assert.assertTrue;
 
-public class AsyncSubscribeTest {
+public class AsyncSubscribeTest extends BaseTest {
     Connection connection = null;
     Statement statement = null;
     String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java

@@ -0,0 +1,27 @@
+package com.taosdata.jdbc;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.junit.BeforeClass;
+
+public class BaseTest {
+    
+    @BeforeClass
+    public static void setupEnv() {
+        try{
+            String path = System.getProperty("user.dir");
+            String bashPath = path + "/buildTDengine.sh";
+
+            Process ps = Runtime.getRuntime().exec(bashPath);
+            ps.waitFor();
+
+            BufferedReader br = new BufferedReader(new InputStreamReader(ps.getInputStream()));
+            while(br.readLine() != null) {
+                System.out.println(br.readLine());
+            }            
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java

@@ -10,7 +10,7 @@ import java.util.Properties;
 
 import static org.junit.Assert.assertTrue;
 
-public class ConnectionTest {
+public class ConnectionTest extends BaseTest {
     static Connection connection = null;
     static Statement statement = null;
     static String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java

@@ -7,7 +7,7 @@ import org.junit.Test;
 import java.sql.*;
 import java.util.Properties;
 
-public class DatabaseMetaDataTest {
+public class DatabaseMetaDataTest extends BaseTest {
     static Connection connection = null;
     static PreparedStatement statement = null;
     static String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java

@@ -9,7 +9,7 @@ import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
-public class ImportTest {
+public class ImportTest extends BaseTest {
     Connection connection = null;
     Statement statement = null;
     String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java

@@ -13,7 +13,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 @FixMethodOrder(MethodSorters.DEFAULT)
-public class PreparedStatementTest {
+public class PreparedStatementTest extends BaseTest {
     static Connection connection = null;
     static PreparedStatement statement = null;
     static String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java

@@ -13,7 +13,7 @@ import java.util.Properties;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class ResultSetTest {
+public class ResultSetTest extends BaseTest {
     static Connection connection = null;
     static Statement statement = null;
     static String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java

@@ -9,7 +9,7 @@ import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 
-public class SelectTest {
+public class SelectTest extends BaseTest {
     Connection connection = null;
     Statement statement = null;
     String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java

@@ -12,7 +12,7 @@ import java.util.Properties;
 import static org.junit.Assert.assertEquals;
 
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public class StableTest {
+public class StableTest extends BaseTest {
     static Connection connection = null;
     static Statement statement = null;
     static String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java

@@ -10,7 +10,7 @@ import java.util.Properties;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class StatementTest {
+public class StatementTest extends BaseTest {
     static Connection connection = null;
     static Statement statement = null;
     static String dbName = "test";

src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java

@@ -12,7 +12,7 @@ import java.util.Properties;
 
 import static org.junit.Assert.assertTrue;
 
-public class SubscribeTest {
+public class SubscribeTest extends BaseTest {
     Connection connection = null;
     Statement statement = null;
     String dbName = "test";

src/dnode/src/dnodeMPeer.c

@@ -74,6 +74,8 @@ void dnodeCleanupMnodePeer() {
     }
   }
 
+  taosCloseQset(tsMPeerQset);
+  tfree(tsMPeerPool.peerWorker);
   dPrint("dnode mpeer is closed");
 }
 

src/dnode/src/dnodeMWrite.c

@@ -76,6 +76,8 @@ void dnodeCleanupMnodeWrite() {
     }
   }
 
+  taosCloseQset(tsMWriteQset);
+  tfree(tsMWritePool.writeWorker);
   dPrint("dnode mwrite is closed");
 }
 

src/inc/mnode.h

@@ -48,6 +48,7 @@ typedef struct SMnodeMsg {
   struct SDbObj *   pDb;
   struct SVgObj *   pVgroup;
   struct STableObj *pTable;
+  struct SSuperTableObj *pSTable;
 } SMnodeMsg;
 
 void    mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);

src/inc/taos.h

@@ -48,7 +48,6 @@ typedef enum {
   TSDB_OPTION_TIMEZONE,
   TSDB_OPTION_CONFIGDIR,
   TSDB_OPTION_SHELL_ACTIVITY_TIMER,
-  TSDB_OPTION_SOCKET_TYPE,
   TSDB_MAX_OPTIONS
 } TSDB_OPTION;
 

src/inc/taosdef.h

@@ -244,8 +244,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 
 #define TSDB_DEFAULT_PKT_SIZE     65480  //same as RPC_MAX_UDP_SIZE
 
-#define TSDB_PAYLOAD_SIZE         (TSDB_DEFAULT_PKT_SIZE - 100)
+#define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
-#define TSDB_DEFAULT_PAYLOAD_SIZE 2048   // default payload size
+#define TSDB_DEFAULT_PAYLOAD_SIZE 5120   // default payload size, greater than PATH_MAX value
 #define TSDB_EXTRA_PAYLOAD_SIZE   128    // extra bytes for auth
 #define TSDB_CQ_SQL_SIZE          1024
 #define TSDB_MAX_VNODES           256

src/inc/tbalance.h

@@ -25,8 +25,8 @@ struct SDnodeObj;
 
 int32_t balanceInit();
 void    balanceCleanUp();
-void    balanceNotify();
+void    balanceAsyncNotify();
-void    balanceUpdateMnode();
+void    balanceSyncNotify();
 void    balanceReset();
 int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
 int32_t balanceDropDnode(struct SDnodeObj *pDnode);

src/inc/tsdb.h

@@ -105,8 +105,9 @@ typedef struct {
 
 void tsdbClearTableCfg(STableCfg *config);
 
-void *     tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes);
+void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes);
-char *     tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id);
+char* tsdbGetTableName(void *pTable);
+STableId tsdbGetTableId(void *pTable);
 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
 
 int   tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
@@ -176,18 +177,16 @@ typedef struct SQueryRowCond {
   TSKEY   ts;
 } SQueryRowCond;
 
-typedef void *TsdbPosT;
-
 /**
  * Get the data block iterator, starting from position according to the query condition
  *
  * @param tsdb       tsdb handle
  * @param pCond      query condition, including time window, result set order, and basic required columns for each block
- * @param groupInfo  tableId list in the form of set, seperated into different groups according to group by condition
+ * @param tableqinfoGroupInfo  tableId list in the form of set, seperated into different groups according to group by condition
  * @param qinfo      query info handle from query processor
  * @return
  */
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo);
+TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo);
 
 /**
  * Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -197,13 +196,26 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
  * @param tsdb        tsdb handle
  * @param pCond       query condition, including time window, result set order, and basic required columns for each
  * block
- * @param groupInfo   tableId list.
+ * @param tableqinfoGroupInfo   tableId list.
  * @return
  */
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo);
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo);
 
-SArray *tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle);
+/**
+ * get the queried table object list
+ * @param pHandle
+ * @return
+ */
+SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
 
+/**
+ * get the group list according to table id from client
+ * @param tsdb
+ * @param pCond
+ * @param groupList
+ * @param qinfo
+ * @return
+ */
 TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
                                                void *qinfo);
 
@@ -247,37 +259,6 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataSta
  */
 SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList);
 
-/**
- * todo remove this function later
- * @param pQueryHandle
- * @param pIdList
- * @return
- */
-SArray *tsdbRetrieveDataRow(TsdbQueryHandleT *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond);
-
-/**
- *  Get iterator for super tables, of which tags values satisfy the tag filter info
- *
- *  NOTE: the tagFilterStr is an bin-expression for tag filter, such as ((tag_col = 5) and (tag_col2 > 7))
- *  The filter string is sent from client directly.
- *  The build of the tags filter expression from string is done in the iterator generating function.
- *
- * @param pCond         query condition
- * @param pTagFilterStr tag filter info
- * @return
- */
-TsdbQueryHandleT *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
-
-/**
- * Get the qualified tables for (super) table query.
- * Used to handle the super table projection queries, the last_row query, the group by on normal columns query,
- * the interpolation query, and timestamp-comp query for join processing.
- *
- * @param pQueryHandle
- * @return table sid list. the invoker is responsible for the release of this the sid list.
- */
-SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
-
 /**
  * Get the qualified table id for a super table according to the tag query expression.
  * @param stableid. super table sid
@@ -287,6 +268,12 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pT
                                  int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
                                  SColIndex *pColIndex, int32_t numOfCols);
 
+/**
+ * destory the created table group list, which is generated by tag query
+ * @param pGroupList
+ */
+void tsdbDestoryTableGroup(STableGroupInfo *pGroupList);
+
 /**
  * create the table group result including only one table, used to handle the normal table query
  *
@@ -297,6 +284,15 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pT
  */
 int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo);
 
+/**
+ *
+ * @param tsdb
+ * @param pTableIdList
+ * @param pGroupInfo
+ * @return
+ */
+int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
+
 /**
  * clean up the query handle
  * @param queryHandle

src/inc/ttokendef.h

@@ -221,6 +221,16 @@
 #define TK_INTO                           203
 #define TK_VALUES                         204
 
+
+#define TK_SPACE                          300
+#define TK_COMMENT                        301
+#define TK_ILLEGAL                        302
+#define TK_HEX                            303   // hex number  0x123
+#define TK_OCT                            304   // oct number
+#define TK_BIN                            305   // bin format data 0b111
+#define TK_FILE                           306
+#define TK_QUESTION                       307   // denoting the placeholder of "?",when invoking statement bind query
+
 #endif
 
 

src/kit/shell/src/shellEngine.c

@@ -474,7 +474,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
   } while( row != NULL);
 
   result = NULL;
-  taos_free_result(tres);
+  //taos_free_result(tres);
   fclose(fp);
 
   return numOfRows;
@@ -803,6 +803,7 @@ void source_file(TAOS *con, char *fptr) {
 
   char *fname = full_path.we_wordv[0];
 
+  /*
   if (access(fname, F_OK) != 0) {
     fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
     
@@ -810,6 +811,7 @@ void source_file(TAOS *con, char *fptr) {
     free(cmd);
     return;
   }
+  */
   
   FILE *f = fopen(fname, "r");
   if (f == NULL) {
@@ -849,7 +851,7 @@ void source_file(TAOS *con, char *fptr) {
 
 void shellGetGrantInfo(void *con) {
   return;
-
+#if 0
   char sql[] = "show grants";
 
   TAOS_RES* tres = taos_query(con, sql);
@@ -900,4 +902,5 @@ void shellGetGrantInfo(void *con) {
   }
 
   fprintf(stdout, "\n");
+  #endif
 }

src/kit/shell/src/shellImport.c

@@ -73,7 +73,7 @@ static void shellParseDirectory(const char *directoryName, const char *prefix, c
   }
 
   int fileNum = 0;
-  while (fscanf(fp, "%s", fileArray[fileNum++])) {
+  while (fscanf(fp, "%128s", fileArray[fileNum++])) {
     if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) {
       fileNum--;
     }
@@ -150,9 +150,11 @@ static void shellSourceFile(TAOS *con, char *fptr) {
   char *fname = full_path.we_wordv[0];
   if (fname == NULL) {
     fprintf(stderr, "ERROR: invalid filename\n");
+    free(cmd);
     return;
   }
 
+  /*
   if (access(fname, F_OK) != 0) {
     fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
     
@@ -168,6 +170,7 @@ static void shellSourceFile(TAOS *con, char *fptr) {
     free(cmd);
     return;
   }
+  */
 
   FILE *f = fopen(fname, "r");
   if (f == NULL) {

src/kit/shell/src/shellLinux.c

@@ -86,7 +86,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         wordfree(&full_path);
         return -1;
       }
-      strcpy(configDir, full_path.we_wordv[0]);
+      tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 's':
@@ -100,7 +100,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
-      strcpy(arguments->file, full_path.we_wordv[0]);
+      tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 'D':
@@ -108,7 +108,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
-      strcpy(arguments->dir, full_path.we_wordv[0]);
+      tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 'T':
@@ -162,13 +162,13 @@ void shellReadCommand(TAOS *con, char *command) {
   // Read input.
   char c;
   while (1) {
-    c = getchar();
+    c = (char)getchar(); // getchar() return an 'int' value
 
     if (c < 0) {  // For UTF-8
       int count = countPrefixOnes(c);
       utf8_array[0] = c;
       for (int k = 1; k < count; k++) {
-        c = getchar();
+        c = (char)getchar();
         utf8_array[k] = c;
       }
       insertChar(&cmd, utf8_array, count);
@@ -214,10 +214,10 @@ void shellReadCommand(TAOS *con, char *command) {
           break;
       }
     } else if (c == '\033') {
-      c = getchar();
+      c = (char)getchar();
       switch (c) {
         case '[':
-          c = getchar();
+          c = (char)getchar();
           switch (c) {
             case 'A':  // Up arrow
               if (hist_counter != history.hstart) {
@@ -244,35 +244,35 @@ void shellReadCommand(TAOS *con, char *command) {
               moveCursorLeft(&cmd);
               break;
             case '1':
-              if ((c = getchar()) == '~') {
+              if ((c = (char)getchar()) == '~') {
                 // Home key
                 positionCursorHome(&cmd);
               }
               break;
             case '2':
-              if ((c = getchar()) == '~') {
+              if ((c = (char)getchar()) == '~') {
                 // Insert key
               }
               break;
             case '3':
-              if ((c = getchar()) == '~') {
+              if ((c = (char)getchar()) == '~') {
                 // Delete key
                 deleteChar(&cmd);
               }
               break;
             case '4':
-              if ((c = getchar()) == '~') {
+              if ((c = (char)getchar()) == '~') {
                 // End key
                 positionCursorEnd(&cmd);
               }
               break;
             case '5':
-              if ((c = getchar()) == '~') {
+              if ((c = (char)getchar()) == '~') {
                 // Page up key
               }
               break;
             case '6':
-              if ((c = getchar()) == '~') {
+              if ((c = (char)getchar()) == '~') {
                 // Page down key
               }
               break;

src/kit/taosdemo/taosdemo.c

@@ -34,6 +34,7 @@
 #include <wordexp.h>
 
 #include "taos.h"
+#include "tutil.h"
 
 extern char configDir[];
 
@@ -82,7 +83,7 @@ typedef struct DemoArguments {
   bool   insert_only;
   char  *output_file;
   int    mode;
-  char  *datatype[MAX_NUM_DATATYPE];
+  char  *datatype[MAX_NUM_DATATYPE+1];
   int    len_of_binary;
   int    num_of_CPR;
   int    num_of_threads;
@@ -432,7 +433,7 @@ int main(int argc, char *argv[]) {
           tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
   printf("###################################################################\n\n");
   printf("Press enter key to continue");
-  getchar();
+  (void)getchar();
 
   fprintf(fp, "###################################################################\n");
   fprintf(fp, "# Server IP:                         %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);
@@ -469,11 +470,12 @@ int main(int argc, char *argv[]) {
   char command[BUFFER_SIZE] = "\0";
 
   sprintf(command, "drop database %s;", db_name);
-  taos_query(taos, command);
+  TAOS_RES* res = taos_query(taos, command);
-  
+  taos_free_result(res);
 
   sprintf(command, "create database %s;", db_name);
-  taos_query(taos, command);
+  res = taos_query(taos, command);
+  taos_free_result(res);
 
   char cols[STRING_LEN] = "\0";
   int colIndex = 0;
@@ -550,8 +552,8 @@ int main(int argc, char *argv[]) {
   for (int i = 0; i < threads; i++) {
     info *t_info = infos + i;
     t_info->threadID = i;
-    strcpy(t_info->db_name, db_name);
+    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
-    strcpy(t_info->tb_prefix, tb_prefix);
+    tstrncpy(t_info->tb_prefix, tb_prefix, MAX_TB_NAME_SIZE);
     t_info->datatype = data_type;
     t_info->ncols_per_record = ncols_per_record;
     t_info->nrecords_per_table = nrecords_per_table;
@@ -845,10 +847,10 @@ void *syncWrite(void *sarg) {
       pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, tID);
       int k;
       for (k = 0; k < winfo->nrecords_per_request;) {
-        int rand_num = rand() % 100;
+        int rand_num = trand() % 100;
         int len = -1;
         if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) {
-          long d = tmp_time - rand() % 1000000 + rand_num;
+          long d = tmp_time - trand() % 1000000 + rand_num;
           len = generateData(data, data_type, ncols_per_record, d, len_of_binary);
         } else {
           len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
@@ -940,10 +942,10 @@ void callBack(void *param, TAOS_RES *res, int code) {
   pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name);
 
   for (int i = 0; i < tb_info->nrecords_per_request; i++) {
-    int rand_num = rand() % 100;
+    int rand_num = trand() % 100;
     if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate)
     {
-      long d = tmp_time - rand() % 1000000 + rand_num;
+      long d = tmp_time - trand() % 1000000 + rand_num;
       generateData(data, datatype, ncols_per_record, d, len_of_binary);
     } else 
     {
@@ -985,22 +987,27 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times
     }
   }
 
+  if (0 == c) {
+    perror("data type error!");
+    exit(-1);
+  }
+
   for (int i = 0; i < num_of_cols; i++) {
     if (strcasecmp(data_type[i % c], "tinyint") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(rand() % 128));
+      pstr += sprintf(pstr, ", %d", (int)(trand() % 128));
     } else if (strcasecmp(data_type[i % c], "smallint") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(rand() % 32767));
+      pstr += sprintf(pstr, ", %d", (int)(trand() % 32767));
     } else if (strcasecmp(data_type[i % c], "int") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(rand() % 10)); 
+      pstr += sprintf(pstr, ", %d", (int)(trand() % 10)); 
     } else if (strcasecmp(data_type[i % c], "bigint") == 0) {
-      pstr += sprintf(pstr, ", %" PRId64, rand() % 2147483648);
+      pstr += sprintf(pstr, ", %" PRId64, trand() % 2147483648);
     } else if (strcasecmp(data_type[i % c], "float") == 0) {
-      pstr += sprintf(pstr, ", %10.4f", (float)(rand() / 1000));
+      pstr += sprintf(pstr, ", %10.4f", (float)(trand() / 1000.0));
     } else if (strcasecmp(data_type[i % c], "double") == 0) {
-      double t = (double)(rand() / 1000000);
+      double t = (double)(trand() / 1000000.0);
       pstr += sprintf(pstr, ", %20.8f", t);
     } else if (strcasecmp(data_type[i % c], "bool") == 0) {
-      bool b = rand() & 1;
+      bool b = trand() & 1;
       pstr += sprintf(pstr, ", %s", b ? "true" : "false");
     } else if (strcasecmp(data_type[i % c], "binary") == 0) {
       char s[len_of_binary];
@@ -1026,7 +1033,7 @@ void rand_string(char *str, int size) {
     --size;
     int n;
     for (n = 0; n < size; n++) {
-      int key = rand() % (int)(sizeof charset - 1);
+      int key = trand() % (int)(sizeof charset - 1);
       str[n] = charset[key];
     }
     str[n] = 0;

src/kit/taosdump/taosdump.c

@@ -229,7 +229,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
-      strcpy(arguments->output, full_path.we_wordv[0]);
+      tstrncpy(arguments->output, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 'i':
@@ -238,7 +238,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
-      strcpy(arguments->input, full_path.we_wordv[0]);
+      tstrncpy(arguments->input, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 'c':
@@ -246,7 +246,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
-      strcpy(configDir, full_path.we_wordv[0]);
+      tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
     case 'e':
@@ -411,7 +411,7 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) {
   if ((row = taos_fetch_row(result)) != NULL) {
     isSet = true;
     pTableRecordInfo->isMetric = true;
-    strcpy(pTableRecordInfo->tableRecord.metric, table);
+    tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN);
   }
 
   taos_free_result(result);
@@ -537,11 +537,11 @@ int taosDumpOut(SDumpArguments *arguments) {
 
   if (arguments->databases || arguments->all_databases) {
     for (int i = 0; i < count; i++) {
-      taosDumpDb(dbInfos[i], arguments, fp);
+      (void)taosDumpDb(dbInfos[i], arguments, fp);
     }
   } else {
     if (arguments->arg_list_len == 1) {
-      taosDumpDb(dbInfos[0], arguments, fp);
+      (void)taosDumpDb(dbInfos[0], arguments, fp);
     } else {
       taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp);
 
@@ -560,9 +560,9 @@ int taosDumpOut(SDumpArguments *arguments) {
         }
 
         if (tableRecordInfo.isMetric) {  // dump whole metric
-          taosDumpMetric(tableRecordInfo.tableRecord.metric, arguments, fp);
+          (void)taosDumpMetric(tableRecordInfo.tableRecord.metric, arguments, fp);
         } else {  // dump MTable and NTable
-          taosDumpTable(tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, arguments, fp);
+          (void)taosDumpTable(tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, arguments, fp);
         }
       }
     }
@@ -642,17 +642,23 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
 
   taos_free_result(result);
 
-  lseek(fd, 0, SEEK_SET);
+  (void)lseek(fd, 0, SEEK_SET);
 
-  while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
+  STableRecord tableInfo;
-    tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
+  while (1) {
-    tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
+    memset(&tableInfo, 0, sizeof(STableRecord));
-    taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
+    ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
+    if (ret <= 0) break;
+    
+    tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
+    tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;
+    taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
   }
 
   close(fd);
+  (void)remove(".table.tmp");
 
-  return remove(".table.tmp");
+  return 0;
 }
 
 void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) {
@@ -807,7 +813,7 @@ int taosGetTableDes(char *table, STableDef *tableDes) {
 
   TAOS_FIELD *fields = taos_fetch_fields(result);
 
-  strcpy(tableDes->name, table);
+  tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN);
 
   while ((row = taos_fetch_row(result)) != NULL) {
     strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
@@ -874,7 +880,7 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
   int fd = -1;
   STableRecord tableRecord;
 
-  tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
+  //tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
 
   sprintf(command, "select tbname from %s", metric);
   TAOS_RES* result = taos_query(taos, command);
@@ -895,24 +901,42 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
 
   while ((row = taos_fetch_row(result)) != NULL) {
     memset(&tableRecord, 0, sizeof(STableRecord));
-    strncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
+    tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
-    strcpy(tableRecord.metric, metric);
+    tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
     twrite(fd, &tableRecord, sizeof(STableRecord));
   }
 
   taos_free_result(result);
   result = NULL;
 
-  lseek(fd, 0, SEEK_SET);
+  (void)lseek(fd, 0, SEEK_SET);
 
-  while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
+  //STableRecord tableInfo;
-    tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
+  char tableName[TSDB_TABLE_NAME_LEN] ;
-    tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
+  char metricName[TSDB_TABLE_NAME_LEN];
-    taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
+  ssize_t ret;
+  while (1) {
+    //memset(&tableInfo, 0, sizeof(STableRecord));      
+    memset(tableName, 0, TSDB_TABLE_NAME_LEN);    
+    memset(metricName, 0, TSDB_TABLE_NAME_LEN);
+    //ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
+    //if (ret <= 0) break;
+    ret = read(fd, tableName, TSDB_TABLE_NAME_LEN);
+    if (ret <= 0) break;
+    
+    ret = read(fd, metricName, TSDB_TABLE_NAME_LEN);
+    if (ret <= 0) break;
+    
+    //tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
+    //tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;    
+    //taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
+    //tstrncpy(tableName, tableInfo.name, TSDB_TABLE_NAME_LEN-1);
+    //tstrncpy(metricName, tableInfo.metric, TSDB_TABLE_NAME_LEN-1);
+    taosDumpTable(tableName, metricName, arguments, fp);
   }
 
-  tclose(fd);
+  close(fd);
-  remove(".table.tmp");
+  (void)remove(".table.tmp");
 
   return 0;
 }
@@ -1004,7 +1028,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
           break;
       }
     }
-    pstr += sprintf(pstr, ")");
+    sprintf(pstr, ")");
 
     count++;
     fprintf(fp, "%s", buffer);
@@ -1327,7 +1351,7 @@ int convertNCharToReadable(char *str, int size, char *buf, int bufsize) {
 
     if ((int)wc < 256) {
       pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]);
-    } else {
+    } else if (byte_width > 0) {
       memcpy(pbuf, pstr, byte_width);
       pbuf += byte_width;
     }

src/mnode/src/mnodeAcct.c

@@ -126,8 +126,8 @@ int32_t mnodeInitAccts() {
 }
 
 void mnodeCleanupAccts() {
-  sdbCloseTable(tsAcctSdb);
   acctCleanUp();
+  sdbCloseTable(tsAcctSdb);
 }
 
 void *mnodeGetAcct(char *name) {

src/mnode/src/mnodeBalance.c

@@ -25,8 +25,8 @@
 
 int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
 void    balanceCleanUp() {}
-void    balanceNotify() {}
+void    balanceAsyncNotify() {}
-void    balanceUpdateMnode() {}
+void    balanceSyncNotify() {}
 void    balanceReset() {}
 
 int32_t balanceAllocVnodes(SVgObj *pVgroup) {

src/mnode/src/mnodeDb.c

@@ -311,6 +311,15 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->replications < 0) pCfg->replications = tsReplications;
 }
 
+static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
+  SDbObj *pDb = pMsg->pDb;
+  if (pDb != NULL) {
+    mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+  }
+
+  return code;
+}
+
 static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg) {
   int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
   if (code != 0) return code;
@@ -364,18 +373,18 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs
     .table   = tsDbSdb,
     .pObj    = pDb,
     .rowSize = sizeof(SDbObj),
-    .pMsg    = pMsg
+    .pMsg    = pMsg,
+    .cb      = mnodeCreateDbCb
   };
 
   code = sdbInsertRow(&oper);
   if (code != TSDB_CODE_SUCCESS) {
     tfree(pDb);
-  } else {
+    mLPrint("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code));
-    mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
-    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
     return code;
+  } else {
+    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  }
 }
 
 bool mnodeCheckIsMonitorDB(char *db, char *monitordb) {
@@ -910,7 +919,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
   mTrace("db:%s, all vgroups is altered", pDb->name);
   mLPrint("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
 
-  balanceNotify();
+  balanceAsyncNotify();
 
   return TSDB_CODE_SUCCESS;
 }

src/mnode/src/mnodeDnode.c

@@ -81,7 +81,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
   mnodeDropAllDnodeVgroups(pDnode);
 #endif  
   mnodeDropMnodeLocal(pDnode->dnodeId);
-  balanceNotify();
+  balanceAsyncNotify();
 
   mTrace("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
   return TSDB_CODE_SUCCESS;
@@ -348,7 +348,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
   pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
   pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
-  mnodeGetMnodeInfos(&pRsp->mnodes);
   SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
 
   for (int32_t j = 0; j < openVnodes; ++j) {
@@ -383,8 +382,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
     
     mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
     pDnode->status = TAOS_DN_STATUS_READY;
-    balanceUpdateMnode();
+    balanceSyncNotify();
-    balanceNotify();
+    balanceAsyncNotify();
   }
 
   if (openVnodes != pDnode->openVnodes) {
@@ -392,6 +391,10 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   }
 
   pDnode->lastAccess = tsAccessSquence;
+
+  //this func should be called after sdb replica changed
+  mnodeGetMnodeInfos(&pRsp->mnodes);
+  
   mnodeDecDnodeRef(pDnode);
 
   pMsg->rpcRsp.len = contLen;

src/mnode/src/mnodeInt.c

@@ -54,6 +54,7 @@ void mnodeCleanupMsg(SMnodeMsg *pMsg) {
     if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb);
     if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup);
     if (pMsg->pTable) mnodeDecTableRef(pMsg->pTable);
+    if (pMsg->pSTable) mnodeDecTableRef(pMsg->pSTable);
     if (pMsg->pAcct) mnodeDecAcctRef(pMsg->pAcct);
     if (pMsg->pDnode) mnodeDecDnodeRef(pMsg->pDnode);
   }

src/mnode/src/mnodeSdb.c

@@ -63,7 +63,6 @@ typedef struct _SSdbTable {
   int32_t (*encodeFp)(SSdbOper *pOper);
   int32_t (*destroyFp)(SSdbOper *pOper);
   int32_t (*restoredFp)();
-  pthread_mutex_t mutex;
 } SSdbTable;
 
 typedef struct {
@@ -429,24 +428,18 @@ static SSdbRow *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) {
 
 void *sdbGetRow(void *handle, void *key) {
   SSdbTable *pTable = (SSdbTable *)handle;
-  SSdbRow * pMeta;
-
-  if (handle == NULL) return NULL;
-
-  pthread_mutex_lock(&pTable->mutex);
-
   int32_t keySize = sizeof(int32_t);
   if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
     keySize = strlen((char *)key);
   }
-  pMeta = taosHashGet(pTable->iHandle, key, keySize);
-
-  if (pMeta) sdbIncRef(pTable, pMeta->row);
-  pthread_mutex_unlock(&pTable->mutex);
-
-  if (pMeta == NULL) return NULL;
   
+  SSdbRow *pMeta = taosHashGet(pTable->iHandle, key, keySize);
+  if (pMeta) {
+    sdbIncRef(pTable, pMeta->row);
     return pMeta->row;
+  } else {
+    return NULL;
+  }
 }
 
 static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) {
@@ -458,8 +451,6 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
   rowMeta.rowSize = pOper->rowSize;
   rowMeta.row = pOper->pObj;
 
-  pthread_mutex_lock(&pTable->mutex);
-
   void *  key = sdbGetObjKey(pTable, pOper->pObj);
   int32_t keySize = sizeof(int32_t);
 
@@ -470,16 +461,14 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
   taosHashPut(pTable->iHandle, key, keySize, &rowMeta, sizeof(SSdbRow));
 
   sdbIncRef(pTable, pOper->pObj);
-  pTable->numOfRows++;
+  atomic_add_fetch_32(&pTable->numOfRows, 1);
 
   if (pTable->keyType == SDB_KEY_AUTO) {
     pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj));
   } else {
-    pTable->autoIndex++;
+    atomic_add_fetch_32(&pTable->autoIndex, 1);
   }
 
-  pthread_mutex_unlock(&pTable->mutex);
-
   sdbTrace("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
 
@@ -490,19 +479,14 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
 static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
   (*pTable->deleteFp)(pOper);
   
-  pthread_mutex_lock(&pTable->mutex);
-
   void *  key = sdbGetObjKey(pTable, pOper->pObj);
   int32_t keySize = sizeof(int32_t);
-
   if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
     keySize = strlen((char *)key);
   }
 
   taosHashRemove(pTable->iHandle, key, keySize);
-
+  atomic_sub_fetch_32(&pTable->numOfRows, 1);
-  pTable->numOfRows--;
-  pthread_mutex_unlock(&pTable->mutex);
   
   sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
@@ -608,14 +592,12 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
   }
 
   if (pTable->keyType == SDB_KEY_AUTO) {
-    pthread_mutex_lock(&pTable->mutex);
+    *((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
-    *((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
 
     // let vgId increase from 2
     if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) {
-      *((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
+      *((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
     }
-    pthread_mutex_unlock(&pTable->mutex);
   }
 
   int32_t code = sdbInsertHash(pTable, pOper);
@@ -805,8 +787,6 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
   }
   pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true);
 
-  pthread_mutex_init(&pTable->mutex, NULL);
-
   tsSdbObj.numOfTables++;
   tsSdbObj.tableList[pTable->tableId] = pTable;
   return pTable;
@@ -835,8 +815,6 @@ void sdbCloseTable(void *handle) {
   taosHashDestroyIter(pIter);
   taosHashCleanup(pTable->iHandle);
 
-  pthread_mutex_destroy(&pTable->mutex);
-  
   sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables);
   free(pTable);
 }
@@ -873,6 +851,7 @@ void sdbCleanupWriteWorker() {
   }
 
   sdbFreeWritequeue();
+  tfree(tsSdbPool.writeWorker);
 
   mPrint("sdb write is closed");
 }

src/mnode/src/mnodeTable.c

@@ -780,6 +780,16 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
   }
 }
 
+static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
+  SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+  if (pTable != NULL) {
+    mLPrint("app:%p:%p, stable:%s, is created in sdb, result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+            tstrerror(code));
+  }
+
+  return code;
+}
+
 static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
   if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
 
@@ -819,27 +829,38 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
     assert(tschema[col].type >= TSDB_DATA_TYPE_BOOL && tschema[col].type <= TSDB_DATA_TYPE_NCHAR);
   }
 
+  pMsg->pTable = (STableObj *)pStable;
+  mnodeIncTableRef(pMsg->pTable);
+
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsSuperTableSdb,
     .pObj = pStable,
     .rowSize = sizeof(SSuperTableObj) + schemaSize,
-    .pMsg = pMsg
+    .pMsg = pMsg,
+    .cb = mnodeCreateSuperTableCb
   };
 
   int32_t code = sdbInsertRow(&oper);
   if (code != TSDB_CODE_SUCCESS) {
     mnodeDestroySuperTable(pStable);
+    pMsg->pTable = NULL;
     mError("app:%p:%p, table:%s, failed to create, sdb error", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+    return code;
   } else {
-    mLPrint("app:%p:%p, table:%s, is created, tags:%d fields:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
-            pStable->numOfTags, pStable->numOfColumns);
+  }
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
+  SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+  if (pTable != NULL) {
+    mLPrint("app:%p:%p, stable:%s, is dropped from sdb, result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+            tstrerror(code));
   }
 
   return code;
 }
-
 static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
   if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
 
@@ -873,16 +894,17 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
     .type  = SDB_OPER_GLOBAL,
     .table = tsSuperTableSdb,
     .pObj  = pStable,
-    .pMsg = pMsg
+    .pMsg  = pMsg,
+    .cb    = mnodeDropSuperTableCb
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code == TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS) {
-    mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code));
+    mError("app:%p:%p, table:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
-    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
-  }
-
     return code;
+  } else {
+    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  }
 }
 
 static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {
@@ -1535,10 +1557,16 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO
 }
 
 static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
+  SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+  if (pTable != NULL) {
+    mTrace("app:%p:%p, table:%s, create table in id:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle, pMsg,
+           pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code));
+  }
+
   if (code != TSDB_CODE_SUCCESS) return code;
 
   SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
-  SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, (SChildTableObj *)pMsg->pTable);
+  SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, pTable);
   if (pMDCreate == NULL) {
     return terrno;
   }
@@ -1578,19 +1606,18 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
     
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     STagData *pTagData = (STagData *) pCreate->schema;  // it is a tag key
-    SSuperTableObj *pSuperTable = mnodeGetSuperTable(pTagData->name);
+    if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);
-    if (pSuperTable == NULL) {
+    if (pMsg->pSTable == NULL) {
       mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg,
              pCreate->tableId, pTagData->name);
       mnodeDestroyChildTable(pTable);
       return TSDB_CODE_MND_INVALID_TABLE_NAME;
     }
-    mnodeDecTableRef(pSuperTable);
     
-    pTable->suid = pSuperTable->uid;
+    pTable->suid = pMsg->pSTable->uid;
     pTable->uid  = (((uint64_t)pTable->vgId) << 40) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 16) +
                   (sdbGetVersion() & ((1ul << 16) - 1ul));
-    pTable->superTable = pSuperTable;
+    pTable->superTable = pMsg->pSTable;
   } else {
     pTable->uid          = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul));
     pTable->sversion     = 0;
@@ -1639,15 +1666,13 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
   
   int32_t code = sdbInsertRow(&desc);
   if (code != TSDB_CODE_SUCCESS) {
-    free(pTable);
+    mnodeDestroyChildTable(pTable);
+    pMsg->pTable = NULL;
     mError("app:%p:%p, table:%s, update sdb error, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
            tstrerror(code));
-    pMsg->pTable = NULL;
     return code;       
   } else {
-    mTrace("app:%p:%p, table:%s, create table in vgroup:%d, id:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
+    return TSDB_CODE_MND_ACTION_IN_PROGRESS;
-           pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid);
-    return TSDB_CODE_SUCCESS;
   }
 }
 
@@ -1683,12 +1708,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
       mTrace("app:%p:%p, table:%s, create table in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
              pVgroup->vgId, sid);
 
-      code = mnodeDoCreateChildTable(pMsg, sid);
+      return mnodeDoCreateChildTable(pMsg, sid);
-      if (code != TSDB_CODE_SUCCESS) {
-        return code;
-      } else {
-        return TSDB_CODE_MND_ACTION_IN_PROGRESS;
-      }
     }
   } else {
     if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId);

src/mnode/src/mnodeVgroup.c

@@ -678,8 +678,9 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
   SMnodeMsg *mnodeMsg = rpcMsg->handle;
   mnodeMsg->received++;
   if (rpcMsg->code == TSDB_CODE_SUCCESS) {
-    mnodeMsg->code = rpcMsg->code;
     mnodeMsg->successed++;
+  } else {
+    mnodeMsg->code = rpcMsg->code;
   }
 
   SVgObj *pVgroup = mnodeMsg->pVgroup;
@@ -702,7 +703,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
       code = TSDB_CODE_MND_SDB_ERROR;
     }
 
-    dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+    dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
   }
 }
 

src/os/linux/src/linuxPlatform.c

@@ -56,70 +56,8 @@ void taosMsleep(int mseconds) {
 
 bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
 
-void taosResetPthread(pthread_t *thread) { *thread = 0; }
-
 int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
 
-/*
-* Function to get the private ip address of current machine. If get IP
-* successfully, return 0, else, return -1. The return values is ip.
-*
-* Use:
-* if (taosGetPrivateIp(ip) != 0) {
-*     perror("Fail to get private IP address\n");
-*     exit(EXIT_FAILURE);
-* }
-*/
-int taosGetPrivateIp(char *const ip) {
-  bool hasLoCard = false;
-
-  struct ifaddrs *ifaddr, *ifa;
-  int             family, s;
-  char            host[NI_MAXHOST];
-
-  if (getifaddrs(&ifaddr) == -1) {
-    return -1;
-  }
-
-  /* Walk through linked list, maintaining head pointer so we can free list later */
-  int flag = 0;
-  for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
-    if (ifa->ifa_addr == NULL) continue;
-
-    family = ifa->ifa_addr->sa_family;
-    if (strcmp("lo", ifa->ifa_name) == 0) {
-      hasLoCard = true;
-      continue;
-    }
-
-    if (family == AF_INET) {
-      /* printf("%-8s", ifa->ifa_name); */
-      s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6),
-                      host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-      if (s != 0) {
-        freeifaddrs(ifaddr);
-        return -1;
-      }
-
-      strcpy(ip, host);
-      flag = 1;
-      break;
-    }
-  }
-
-  freeifaddrs(ifaddr);
-  if (flag) {
-    return 0;
-  } else {
-    if (hasLoCard) {
-      uPrint("no net card was found, use lo:127.0.0.1 as default");
-      strcpy(ip, "127.0.0.1");
-      return 0;
-    }
-    return -1;
-  }
-}
-
 int taosSetNonblocking(int sock, int on) {
   int flags = 0;
   if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
@@ -294,21 +232,6 @@ ssize_t twrite(int fd, void *buf, size_t n) {
   return n;
 }
 
-bool taosSkipSocketCheck() {
-  struct utsname buf;
-  if (uname(&buf)) {
-    uPrint("can't fetch os info");
-    return false;
-  }
-
-  if (strstr(buf.release, "Microsoft") != 0) {
-    uPrint("using WSLv1");
-    return true;
-  }
-
-  return false;
-}
-
 void taosBlockSIGPIPE() {
   sigset_t signal_mask;
   sigemptyset(&signal_mask);

src/os/linux/src/linuxSysPara.c

@@ -162,7 +162,13 @@ static void taosGetSystemTimezone() {
   FILE *f = fopen("/etc/timezone", "r");
   char  buf[65] = {0};
   if (f != NULL) {
-    fread(buf, 64, 1, f);
+    int len = fread(buf, 64, 1, f);
+    if(len < 64 && ferror(f)) {
+      fclose(f);
+      uError("read /etc/timezone error, reason:%s", strerror(errno));
+      return;
+    }
+    
     fclose(f);
   }
 
@@ -547,7 +553,7 @@ void taosSetCoreDump() {
   struct rlimit rlim;
   struct rlimit rlim_new;
   if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uPrint("the old unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max);
+    uPrint("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
     rlim_new.rlim_cur = RLIM_INFINITY;
     rlim_new.rlim_max = RLIM_INFINITY;
     if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
@@ -559,7 +565,7 @@ void taosSetCoreDump() {
   }
 
   if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uPrint("the new unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max);
+    uPrint("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
   }
 
 #ifndef _TD_ARM_
@@ -586,7 +592,7 @@ void taosSetCoreDump() {
       uPrint("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
   }
   
-  uPrint("The old core_uses_pid[%d]: %d", old_len, old_usespid);
+  uPrint("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
 
 
   old_usespid = 0;
@@ -603,7 +609,7 @@ void taosSetCoreDump() {
       uPrint("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
   }
   
-  uPrint("The new core_uses_pid[%d]: %d", old_len, old_usespid);
+  uPrint("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
 #endif
   
 #if 0

src/query/inc/qExecutor.h

@@ -102,7 +102,7 @@ typedef struct STableQueryInfo {  // todo merge with the STableQueryInfo struct
   int64_t     tag;
   STimeWindow win;
   STSCursor   cur;
-  STableId    id;  // for retrieve the page id list
+  void*       pTable;  // for retrieve the page id list
 
   SWindowResInfo windowResInfo;
 } STableQueryInfo;
@@ -126,10 +126,10 @@ typedef struct SQueryCostInfo {
   uint64_t computTime;
 } SQueryCostInfo;
 
-typedef struct SGroupItem {
+//typedef struct SGroupItem {
-  STableId         id;
+//  void            *pTable;
-  STableQueryInfo* info;
+//  STableQueryInfo *info;
-} SGroupItem;
+//} SGroupItem;
 
 typedef struct SQuery {
   int16_t          numOfCols;
@@ -187,8 +187,8 @@ typedef struct SQInfo {
   void*   tsdb;
   int32_t vgId;
 
-  STableGroupInfo  tableIdGroupInfo;  // table id list < only includes the STableId list>
+  STableGroupInfo  tableGroupInfo;       // table id list < only includes the STable list>
-  STableGroupInfo  groupInfo;         //
+  STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
   SQueryRuntimeEnv runtimeEnv;
   int32_t          groupIndex;
   int32_t          offset;  // offset in group result set of subgroup, todo refactor

src/query/src/qExecutor.c

@@ -113,7 +113,7 @@ static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static bool hasMainOutput(SQuery *pQuery);
 static void buildTagQueryResult(SQInfo *pQInfo);
 
-static int32_t setAdditionalInfo(SQInfo *pQInfo, STableId *pTableId, STableQueryInfo *pTableQueryInfo);
+static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo);
 static int32_t flushFromResultBuf(SQInfo *pQInfo);
 
 bool doFilterData(SQuery *pQuery, int32_t elemPos) {
@@ -1495,6 +1495,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 }
 
 static bool isQueryKilled(SQInfo *pQInfo) {
+  return false;
   return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED);
 }
 
@@ -1781,7 +1782,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) {
   if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
     num = 128;
   } else if (isIntervalQuery(pQuery)) {  // time window query, allocate one page for each table
-    size_t s = pQInfo->groupInfo.numOfTables;
+    size_t s = pQInfo->tableqinfoGroupInfo.numOfTables;
     num = MAX(s, INITIAL_RESULT_ROWS_VALUE);
   } else {    // for super table query, one page for each subset
     num = 1;  // pQInfo->pSidSet->numOfSubSet;
@@ -1815,18 +1816,18 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi
  */
 UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) {
   if (pQInfo != NULL) {
-    //    assert(taosHashGetSize(pQInfo->groupInfo) >= 1);
+    //    assert(taosHashGetSize(pQInfo->tableqinfoGroupInfo) >= 1);
   }
 
 #if 0
-  if (pQInfo == NULL || pQInfo->groupInfo.numOfTables == 1) {
+  if (pQInfo == NULL || pQInfo->tableqinfoGroupInfo.numOfTables == 1) {
     atomic_fetch_sub_32(&pQInfo->pObj->numOfQueries, 1);
     qTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pQInfo->pObj->vnode,
            pQInfo->pObj->sid, pQInfo->pObj->meterId, pQInfo->pObj->numOfQueries);
   } else {
     int32_t num = 0;
-    for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) {
+    for (int32_t i = 0; i < pQInfo->tableqinfoGroupInfo.numOfTables; ++i) {
-      SMeterObj *pMeter = getMeterObj(pQInfo->groupInfo, pQInfo->pSidSet->pTableIdList[i]->sid);
+      SMeterObj *pMeter = getMeterObj(pQInfo->tableqinfoGroupInfo, pQInfo->pSidSet->pTableIdList[i]->sid);
       atomic_fetch_sub_32(&(pMeter->numOfQueries), 1);
 
       if (pMeter->numOfQueries > 0) {
@@ -1840,9 +1841,9 @@ UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) {
      * in order to reduce log output, for all meters of which numOfQueries count are 0,
      * we do not output corresponding information
      */
-    num = pQInfo->groupInfo.numOfTables - num;
+    num = pQInfo->tableqinfoGroupInfo.numOfTables - num;
     qTrace("QInfo:%p metric query is over, dec query ref for %d meters, numOfQueries on %d meters are 0", pQInfo,
-           pQInfo->groupInfo.numOfTables, num);
+           pQInfo->tableqinfoGroupInfo.numOfTables, num);
   }
 #endif
 }
@@ -2188,16 +2189,16 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
  * set tag value in SQLFunctionCtx
  * e.g.,tag information into input buffer
  */
-static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) {
+static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) {
   tVariantDestroy(tag);
 
   if (tagColId == TSDB_TBNAME_COLUMN_INDEX) {
-    char* val = tsdbGetTableName(tsdb, pTableId);
+    char* val = tsdbGetTableName(pTable);
     assert(val != NULL);
     
     tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY);
   } else {
-    char* val = tsdbGetTableTagVal(tsdb, pTableId, tagColId, type, bytes);
+    char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes);
     if (val == NULL) {
       tag->nType = TSDB_DATA_TYPE_NULL;
       return;
@@ -2221,7 +2222,7 @@ static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColI
   }
 }
 
-void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
+void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
   SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
 
@@ -2238,7 +2239,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
       }
     }
 
-    doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
+    doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
   } else {
     // set tag value, by which the results are aggregated.
     for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) {
@@ -2250,7 +2251,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
       }
 
       // todo use tag column index to optimize performance
-      doSetTagValueInParam(tsdb, pTableId, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
+      doSetTagValueInParam(tsdb, pTable, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
                            pLocalExprInfo->type, pLocalExprInfo->bytes);
     }
 
@@ -2269,7 +2270,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
         }
       }
 
-      doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
+      doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
       qTrace("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64,
           pRuntimeEnv->pCtx[0].tag.i64Key)
     }
@@ -2474,10 +2475,10 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
   int64_t st = taosGetTimestampMs();
   int32_t ret = TSDB_CODE_SUCCESS;
 
-  int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+  int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
 
   while (pQInfo->groupIndex < numOfGroups) {
-    SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
+    SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, pQInfo->groupIndex);
     ret = mergeIntoGroupResultImpl(pQInfo, group);
     if (ret < 0) {  // not enough disk space to save the data into disk
       return -1;
@@ -2510,9 +2511,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
     }
 
     // check if all results has been sent to client
-    int32_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+    int32_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
     if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) {
-      pQInfo->tableIndex = pQInfo->groupInfo.numOfTables;  // set query completed
+      pQInfo->tableIndex = pQInfo->tableqinfoGroupInfo.numOfTables;  // set query completed
       return;
     }
   }
@@ -2599,12 +2600,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
   // todo opt for the case of one table per group
   int32_t numOfTables = 0;
   for (int32_t i = 0; i < size; ++i) {
-    SGroupItem *item = taosArrayGet(pGroup, i);
+    STableQueryInfo *item = taosArrayGetP(pGroup, i);
-    STableQueryInfo *pInfo = item->info;
 
-    SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, pInfo->id.tid);
+    SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, tsdbGetTableId(item->pTable).tid);
-    if (list.size > 0 && pInfo->windowResInfo.size > 0) {
+    if (list.size > 0 && item->windowResInfo.size > 0) {
-      pTableList[numOfTables] = pInfo;
+      pTableList[numOfTables] = item;
       numOfTables += 1;
     }
   }
@@ -2845,15 +2845,15 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
     }
   }
   
-  int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+  int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
   
   for(int32_t i = 0; i < numOfGroups; ++i) {
-    SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
+    SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
     
     size_t t = taosArrayGetSize(group);
     for (int32_t j = 0; j < t; ++j) {
-      SGroupItem *item = taosArrayGet(group, j);
+      STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);
-      updateTableQueryInfoForReverseScan(pQuery, item->info);
+      updateTableQueryInfoForReverseScan(pQuery, pCheckInfo);
     }
   }
 }
@@ -3088,7 +3088,8 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
     tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
   }
 
-  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
+  // add ref for table
+  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
 
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
   switchCtxOrder(pRuntimeEnv);
@@ -3160,7 +3161,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
       tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
     }
 
-    pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
+    pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
     pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex;
 
     setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
@@ -3236,17 +3237,13 @@ static bool hasMainOutput(SQuery *pQuery) {
   return false;
 }
 
-static STableQueryInfo *createTableQueryInfo(
+static STableQueryInfo *createTableQueryInfo( SQueryRuntimeEnv *pRuntimeEnv, void* pTable, STimeWindow win) {
-  SQueryRuntimeEnv *pRuntimeEnv,
-  STableId tableId,
-  STimeWindow win
-) {
   STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo));
 
   pTableQueryInfo->win = win;
   pTableQueryInfo->lastKey = win.skey;
 
-  pTableQueryInfo->id = tableId;
+  pTableQueryInfo->pTable = pTable;
   pTableQueryInfo->cur.vgroupIndex = -1;
 
   initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT);
@@ -3275,7 +3272,7 @@ void setCurrentQueryTable(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTable
  * @param pRuntimeEnv
  * @param pDataBlockInfo
  */
-void setExecutionContext(SQInfo *pQInfo, STableId* pTableId, int32_t groupIndex, TSKEY nextKey) {
+void setExecutionContext(SQInfo *pQInfo, void* pTable, int32_t groupIndex, TSKEY nextKey) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current;
   
@@ -3302,7 +3299,7 @@ void setExecutionContext(SQInfo *pQInfo, STableId* pTableId, int32_t groupIndex,
   initCtxOutputBuf(pRuntimeEnv);
 
   pTableQueryInfo->lastKey = nextKey;
-  setAdditionalInfo(pQInfo, pTableId, pTableQueryInfo);
+  setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo);
 }
 
 void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
@@ -3362,11 +3359,10 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
   }
 }
 
-int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) {
+int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQueryInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-  //assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL);
 
-  setTagVal(pRuntimeEnv, pTableId, pQInfo->tsdb);
+  setTagVal(pRuntimeEnv, pTable, pQInfo->tsdb);
 
   // both the master and supplement scan needs to set the correct ts comp start position
   if (pRuntimeEnv->pTSBuf != NULL) {
@@ -3469,7 +3465,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) {
   if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) {
     totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo);
   } else {
-    totalSubset = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+    totalSubset = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
   }
 
   return totalSubset;
@@ -3669,7 +3665,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
   // all data returned, set query over
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
     if (pQInfo->runtimeEnv.stableQuery) {
-      if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
+      if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) {
         setQueryStatus(pQuery, QUERY_OVER);
       }
     } else {
@@ -3979,23 +3975,23 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
   };
 
   if (!isSTableQuery
-    && (pQInfo->groupInfo.numOfTables == 1)
+    && (pQInfo->tableqinfoGroupInfo.numOfTables == 1)
     && (cond.order == TSDB_ORDER_ASC) 
     && (!isIntervalQuery(pQuery))
     && (!isGroupbyNormalCol(pQuery->pGroupbyExpr))
     && (!isFixedOutputQuery(pQuery))
   ) {
-    SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
+    SArray* pa = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
-    SGroupItem* pItem = taosArrayGet(pa, 0);
+    STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0);
-    cond.twindow = pItem->info->win;
+    cond.twindow = pCheckInfo->win;
   }
 
   if (isFirstLastRowQuery(pQuery)) {
-    pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
+    pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
   } else if (isPointInterpoQuery(pQuery)) {
-    pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
+    pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
   } else {
-    pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
+    pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
   }
 }
 
@@ -4132,18 +4128,18 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
     STableQueryInfo *pTableQueryInfo = NULL;
 
     // todo opt performance using hash table
-    size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+    size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
     for (int32_t i = 0; i < numOfGroup; ++i) {
-      SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
+      SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
 
       size_t num = taosArrayGetSize(group);
       for (int32_t j = 0; j < num; ++j) {
-        SGroupItem *item = taosArrayGet(group, j);
+        STableQueryInfo *p = taosArrayGetP(group, j);
-        STableQueryInfo *pInfo = item->info;
 
-        if (pInfo->id.tid == blockInfo.tid) {
+        STableId id = tsdbGetTableId(p->pTable);
-          assert(pInfo->id.uid == blockInfo.uid);
+        if (id.tid == blockInfo.tid) {
-          pTableQueryInfo = item->info;
+          assert(id.uid == blockInfo.uid);
+          pTableQueryInfo = p;
 
           break;
         }
@@ -4163,11 +4159,11 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
     if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
       if (!isIntervalQuery(pQuery)) {
         int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
-        setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
+        setExecutionContext(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
       } else {  // interval query
         TSKEY nextKey = blockInfo.window.skey;
         setIntervalQueryRange(pQInfo, nextKey);
-        /*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
+        /*int32_t ret = */setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo);
       }
     }
 
@@ -4187,16 +4183,17 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
-  SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
+  SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
-  SGroupItem* item = taosArrayGet(group, index);
+  STableQueryInfo* pCheckInfo = taosArrayGetP(group, index);
 
-  setTagVal(pRuntimeEnv, &item->id, pQInfo->tsdb);
+  setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb);
 
+  STableId id = tsdbGetTableId(pCheckInfo->pTable);
   qTrace("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index,
-         item->id.uid, item->id.tid, item->info->lastKey, item->info->win.ekey);
+         id.uid, id.tid, pCheckInfo->lastKey, pCheckInfo->win.ekey);
 
   STsdbQueryCond cond = {
-      .twindow   = {item->info->lastKey, item->info->win.ekey},
+      .twindow   = {pCheckInfo->lastKey, pCheckInfo->win.ekey},
       .order     = pQuery->order.order,
       .colList   = pQuery->colList,
       .numOfCols = pQuery->numOfCols,
@@ -4204,9 +4201,9 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
 
   // todo refactor
   SArray *g1 = taosArrayInit(1, POINTER_BYTES);
-  SArray *tx = taosArrayInit(1, sizeof(STableId));
+  SArray *tx = taosArrayInit(1, POINTER_BYTES);
 
-  taosArrayPush(tx, &item->info->id);
+  taosArrayPush(tx, &pCheckInfo->pTable);
   taosArrayPush(g1, &tx);
   STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1};
 
@@ -4250,14 +4247,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
   SQuery *          pQuery = pRuntimeEnv->pQuery;
   setQueryStatus(pQuery, QUERY_COMPLETED);
 
-  size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+  size_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
 
   if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) {
     resetCtxOutputBuf(pRuntimeEnv);
     assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0);
 
     while (pQInfo->groupIndex < numOfGroups) {
-      SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
+      SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
 
       qTrace("QInfo:%p last_row query on group:%d, total group:%zu, current group:%p", pQInfo, pQInfo->groupIndex,
              numOfGroups, group);
@@ -4289,10 +4286,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       
       initCtxOutputBuf(pRuntimeEnv);
       
-      SArray* s = tsdbGetQueriedTableIdList(pRuntimeEnv->pQueryHandle);
+      SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
       assert(taosArrayGetSize(s) >= 1);
       
-      setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
+      setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
       if (isFirstLastRowQuery(pQuery)) {
         assert(taosArrayGetSize(s) == 1);
       }
@@ -4300,7 +4297,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       taosArrayDestroy(s);
 
       // here we simply set the first table as current table
-      pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
+      pQuery->current = (STableQueryInfo*) taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
       scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
       
       int64_t numOfRes = getNumOfResult(pRuntimeEnv);
@@ -4322,7 +4319,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     }
   } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group-by on normal columns query
     while (pQInfo->groupIndex < numOfGroups) {
-      SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
+      SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
 
       qTrace("QInfo:%p group by normal columns group:%d, total group:%zu", pQInfo, pQInfo->groupIndex, numOfGroups);
 
@@ -4347,10 +4344,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
 
-      SArray* s = tsdbGetQueriedTableIdList(pRuntimeEnv->pQueryHandle);
+      SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
       assert(taosArrayGetSize(s) >= 1);
 
-      setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
+      setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
 
       // here we simply set the first table as current table
       scanMultiTableDataBlocks(pQInfo);
@@ -4406,25 +4403,23 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     }
 
     // all data have returned already
-    if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
+    if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) {
       return;
     }
 
     resetCtxOutputBuf(pRuntimeEnv);
     resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo);
 
-    SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
+    SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
-    assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables &&
+    assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables &&
-           1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList));
+           1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList));
 
-    while (pQInfo->tableIndex < pQInfo->groupInfo.numOfTables) {
+    while (pQInfo->tableIndex < pQInfo->tableqinfoGroupInfo.numOfTables) {
       if (isQueryKilled(pQInfo)) {
         return;
       }
 
-      SGroupItem *item = taosArrayGet(group, pQInfo->tableIndex);
+      pQuery->current = taosArrayGetP(group, pQInfo->tableIndex);
-      pQuery->current = item->info;
-      
       if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) {
         pQInfo->tableIndex++;
         continue;
@@ -4444,7 +4439,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       // the limitation of output result is reached, set the query completed
       if (limitResults(pRuntimeEnv)) {
-        pQInfo->tableIndex = pQInfo->groupInfo.numOfTables;
+        pQInfo->tableIndex = pQInfo->tableqinfoGroupInfo.numOfTables;
         break;
       }
 
@@ -4461,8 +4456,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         pQInfo->tableIndex++;
 
         STableIdInfo tidInfo;
-        tidInfo.uid = item->id.uid;
+        STableId id = tsdbGetTableId(pQuery->current->pTable);
-        tidInfo.tid = item->id.tid;
+
+        tidInfo.uid = id.uid;
+        tidInfo.tid = id.tid;
         tidInfo.key = pQuery->current->lastKey;
         taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
 
@@ -4485,7 +4482,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       }
     }
 
-    if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
+    if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) {
       setQueryStatus(pQuery, QUERY_COMPLETED);
     }
   }
@@ -4511,7 +4508,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
   qTrace(
       "QInfo %p numOfTables:%"PRIu64", index:%d, numOfGroups:%zu, %"PRId64" points returned, total:%"PRId64", offset:%" PRId64,
-      pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
+      pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
       pQuery->limit.offset);
 }
 
@@ -4539,7 +4536,7 @@ static void doSaveContext(SQInfo *pQInfo) {
     tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
   }
   
-  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo, pQInfo);
+  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
   
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
   switchCtxOrder(pRuntimeEnv);
@@ -4564,14 +4561,14 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
   if (isIntervalQuery(pQuery)) {
-    size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+    size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
     for (int32_t i = 0; i < numOfGroup; ++i) {
-      SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
+      SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
 
       size_t num = taosArrayGetSize(group);
       for (int32_t j = 0; j < num; ++j) {
-        SGroupItem* item = taosArrayGet(group, j);
+        STableQueryInfo* item = taosArrayGetP(group, j);
-        closeAllTimeWindow(&item->info->windowResInfo);
+        closeAllTimeWindow(&item->windowResInfo);
       }
     }
   } else {  // close results for group result
@@ -4732,8 +4729,10 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
         pQuery->current->lastKey, pQuery->window.ekey);
   } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
     STableIdInfo tidInfo;
-    tidInfo.uid = pQuery->current->id.uid;
+    STableId id = tsdbGetTableId(pQuery->current);
-    tidInfo.tid = pQuery->current->id.tid;
+
+    tidInfo.uid = id.uid;
+    tidInfo.tid = id.tid;
     tidInfo.key = pQuery->current->lastKey;
     taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
   }
@@ -4883,23 +4882,23 @@ static void tableQueryImpl(SQInfo *pQInfo) {
   pQuery->rec.rows = 0;
   int64_t st = taosGetTimestampUs();
   
-  assert(pQInfo->groupInfo.numOfTables == 1);
+  assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1);
-  SArray* g = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
+  SArray* g = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
-  SGroupItem* item = taosArrayGet(g, 0);
+  STableQueryInfo* item = taosArrayGetP(g, 0);
   
   // group by normal column, sliding window query, interval query are handled by interval query processor
   if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) {  // interval (down sampling operation)
-    tableIntervalProcess(pQInfo, item->info);
+    tableIntervalProcess(pQInfo, item);
   } else if (isFixedOutputQuery(pQuery)) {
-    tableFixedOutputProcess(pQInfo, item->info);
+    tableFixedOutputProcess(pQInfo, item);
   } else {  // diff/add/multiply/subtract/division
     assert(pQuery->checkBuffer == 1);
-    tableMultiOutputProcess(pQInfo, item->info);
+    tableMultiOutputProcess(pQInfo, item);
   }
 
   // record the total elapsed time
   pRuntimeEnv->summary.elapsedTime += (taosGetTimestampUs() - st);
-  assert(pQInfo->groupInfo.numOfTables == 1);
+  assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1);
 
   /* check if query is killed or not */
   if (isQueryKilled(pQInfo)) {
@@ -4931,7 +4930,7 @@ static void stableQueryImpl(SQInfo *pQInfo) {
   pQInfo->runtimeEnv.summary.elapsedTime += (taosGetTimestampUs() - st);
   
   if (pQuery->rec.rows == 0) {
-    qTrace("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total);
+    qTrace("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total);
   }
 }
 
@@ -5154,9 +5153,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
   }
 
   if (!validateQuerySourceCols(pQueryMsg, *pExpr)) {
-    tfree(*pExpr);
+    goto _cleanup;
-
-    return TSDB_CODE_QRY_INVALID_MSG;
   }
 
   pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList);
@@ -5228,8 +5225,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
          pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
          pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime,
          pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset);
-
   return 0;
+
+_cleanup:
+  tfree(*pExpr);
+  taosArrayDestroy(*pTableIdList);
+  *pTableIdList = NULL;
+  tfree(*tbnameCond);
+  tfree(*groupbyCols);
+  tfree(*tagCols);
+  tfree(*tagCond);
+  return TSDB_CODE_QRY_INVALID_MSG; 
 }
 
 static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) {
@@ -5487,7 +5493,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) {
 }
 
 
-static int compareTableIdInfo( const void* a, const void* b ) {
+static int compareTableIdInfo(const void* a, const void* b) {
   const STableIdInfo* x = (const STableIdInfo*)a;
   const STableIdInfo* y = (const STableIdInfo*)b;
   if (x->uid > y->uid) return 1;
@@ -5495,8 +5501,10 @@ static int compareTableIdInfo( const void* a, const void* b ) {
   return 0;
 }
 
+static void freeQInfo(SQInfo *pQInfo);
+
 static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
-                               STableGroupInfo *groupInfo, SColumnInfo* pTagCols) {
+                               STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols) {
   SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo));
   if (pQInfo == NULL) {
     return NULL;
@@ -5582,39 +5590,41 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   // to make sure third party won't overwrite this structure
   pQInfo->signature = pQInfo;
 
-  pQInfo->tableIdGroupInfo = *groupInfo;
+  pQInfo->tableGroupInfo = *pTableGroupInfo;
-  size_t numOfGroups = taosArrayGetSize(groupInfo->pGroupList);
+  size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList);
 
-  pQInfo->groupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
+  pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
-  pQInfo->groupInfo.numOfTables = groupInfo->numOfTables;
+  pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables;
   
   int tableIndex = 0;
   STimeWindow window = pQueryMsg->window;
   taosArraySort(pTableIdList, compareTableIdInfo);
 
   for(int32_t i = 0; i < numOfGroups; ++i) {
-    SArray* pa = taosArrayGetP(groupInfo->pGroupList, i);
+    SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i);
     size_t s = taosArrayGetSize(pa);
 
-    SArray* p1 = taosArrayInit(s, sizeof(SGroupItem));
+    SArray* p1 = taosArrayInit(s, POINTER_BYTES);
 
     for(int32_t j = 0; j < s; ++j) {
-      STableId id = *(STableId*) taosArrayGet(pa, j);
+      void* pTable = taosArrayGetP(pa, j);
-      SGroupItem item = { .id = id };
+
       // NOTE: compare STableIdInfo with STableId
-      STableIdInfo* pTableId = taosArraySearch( pTableIdList, &id, compareTableIdInfo);
+      STableId id = tsdbGetTableId(pTable);
+      STableIdInfo* pTableId = taosArraySearch(pTableIdList, &id, compareTableIdInfo);
       if (pTableId != NULL ) {
         window.skey = pTableId->key;
       } else {
         window.skey = pQueryMsg->window.skey;
       }
-      item.info = createTableQueryInfo(&pQInfo->runtimeEnv, item.id, window);
+
-      item.info->groupIndex = i;
+      STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, pTable, window);
-      item.info->tableIndex = tableIndex++;
+      item->groupIndex = i;
+      item->tableIndex = tableIndex++;
       taosArrayPush(p1, &item);
     }
 
-    taosArrayPush(pQInfo->groupInfo.pGroupList, &p1);
+    taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1);
   }
 
   pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
@@ -5633,22 +5643,27 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   return pQInfo;
 
 _cleanup:
-  tfree(pQuery->fillVal);
+  //tfree(pQuery->fillVal);
 
-  if (pQuery->sdata != NULL) {
+  //if (pQuery->sdata != NULL) {
-    for (int16_t col = 0; col < pQuery->numOfOutput; ++col) {
+  //  for (int16_t col = 0; col < pQuery->numOfOutput; ++col) {
-      tfree(pQuery->sdata[col]);
+  //    tfree(pQuery->sdata[col]);
-    }
+  //  }
-  }
+  //}
 
-  tfree(pQuery->sdata);
+  //
-  tfree(pQuery->pFilterInfo);
+  //tfree(pQuery->sdata);
-  tfree(pQuery->colList);
+  //tfree(pQuery->pFilterInfo);
+  //tfree(pQuery->colList);
 
-  tfree(pExprs);
+  //tfree(pExprs);
-  tfree(pGroupbyExpr);
+  //tfree(pGroupbyExpr);
 
-  tfree(pQInfo);
+  //taosArrayDestroy(pQInfo->arrTableIdInfo);
+  //tsdbDestoryTableGroup(&pQInfo->tableGroupInfo);
+  // 
+  //tfree(pQInfo);
+  freeQInfo(pQInfo);
 
   return NULL;
 }
@@ -5667,7 +5682,6 @@ static bool isValidQInfo(void *param) {
   return (sig == (uint64_t)pQInfo);
 }
 
-static void freeQInfo(SQInfo *pQInfo);
 
 static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) {
   int32_t code = TSDB_CODE_SUCCESS;
@@ -5693,7 +5707,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
     return TSDB_CODE_SUCCESS;
   }
   
-  if (pQInfo->groupInfo.numOfTables == 0) {
+  if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
     qTrace("QInfo:%p no table qualified for tag filter, abort query", pQInfo);
     setQueryStatus(pQuery, QUERY_COMPLETED);
   
@@ -5754,29 +5768,24 @@ static void freeQInfo(SQInfo *pQInfo) {
   }
 
   // todo refactor, extract method to destroytableDataInfo
-  int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+  int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
   for (int32_t i = 0; i < numOfGroups; ++i) {
-    SArray *p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
+    SArray *p = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i);
 
     size_t num = taosArrayGetSize(p);
     for(int32_t j = 0; j < num; ++j) {
-      SGroupItem* item = taosArrayGet(p, j);
+      STableQueryInfo* item = taosArrayGetP(p, j);
-      if (item->info != NULL) {
+      if (item != NULL) {
-        destroyTableQueryInfo(item->info, pQuery->numOfOutput);
+        destroyTableQueryInfo(item, pQuery->numOfOutput);
       }
     }
 
     taosArrayDestroy(p);
   }
 
-  taosArrayDestroy(pQInfo->groupInfo.pGroupList);
+  taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList);
 
-  for(int32_t i = 0; i < numOfGroups; ++i) {
+  tsdbDestoryTableGroup(&pQInfo->tableGroupInfo);
-    SArray* p = taosArrayGetP(pQInfo->tableIdGroupInfo.pGroupList, i);
-    taosArrayDestroy(p);
-  }
-
-  taosArrayDestroy(pQInfo->tableIdGroupInfo.pGroupList);
   taosArrayDestroy(pQInfo->arrTableIdInfo);
   
   if (pQuery->pGroupbyExpr != NULL) {
@@ -5861,8 +5870,6 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
   }
   
   return TSDB_CODE_SUCCESS;
-
-  // todo if interpolation exists, the result may be dump to client by several rounds
 }
 
 int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) {
@@ -5875,6 +5882,8 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
   SSqlFuncMsg **pExprMsg = NULL;
   SColIndex *   pGroupColIndex = NULL;
   SColumnInfo*  pTagColumnInfo = NULL;
+  SExprInfo     *pExprs = NULL;
+  SSqlGroupbyExpr *pGroupbyExpr = NULL;
 
   if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) !=
          TSDB_CODE_SUCCESS) {
@@ -5893,24 +5902,24 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
     goto _over;
   }
 
-  SExprInfo *pExprs = NULL;
   if ((code = createQFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) {
+    free(pExprMsg);
     goto _over;
   }
 
-  SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code);
+  pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code);
   if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
     goto _over;
   }
 
   bool isSTableQuery = false;
-  STableGroupInfo groupInfo = {0};
+  STableGroupInfo tableGroupInfo = {0};
   
   if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) {
     STableIdInfo *id = taosArrayGet(pTableIdList, 0);
 
     qTrace("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
-    if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
+    if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &tableGroupInfo)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }
   } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) {
@@ -5927,25 +5936,28 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
         numOfGroupByCols = 0;
       }
       
-      code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex,
+      code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex,
                                           numOfGroupByCols);
       if (code != TSDB_CODE_SUCCESS) {
         goto _over;
       }
     } else {
-      groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES);
+      code = tsdbGetTableGroupFromIdList(tsdb, pTableIdList, &tableGroupInfo);
-      groupInfo.numOfTables = taosArrayGetSize(pTableIdList);
+      if (code != TSDB_CODE_SUCCESS) {
+        goto _over;
+      }
 
-      SArray* p = taosArrayClone(pTableIdList);
+      qTrace("qmsg:%p query on %zu tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables);
-      taosArrayPush(groupInfo.pGroupList, &p);
-
-      qTrace("qmsg:%p query on %zu tables in one group from client", pQueryMsg, groupInfo.numOfTables);
     }
   } else {
     assert(0);
   }
 
-  (*pQInfo) = createQInfoImpl(pQueryMsg, pTableIdList, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo);
+  (*pQInfo) = createQInfoImpl(pQueryMsg, pTableIdList, pGroupbyExpr, pExprs, &tableGroupInfo, pTagColumnInfo);
+  pExprs = NULL;
+  pGroupbyExpr = NULL;
+  pTagColumnInfo = NULL;
+  
   if ((*pQInfo) == NULL) {
     code = TSDB_CODE_QRY_OUT_OF_MEMORY;
     goto _over;
@@ -5954,9 +5966,15 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
   code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery);
 
 _over:
-  tfree(tagCond);
+  free(tagCond);
-  tfree(tbnameCond);
+  free(tbnameCond);
-  tfree(pGroupColIndex);
+  free(pGroupColIndex);
+  if (pGroupbyExpr != NULL) {
+    taosArrayDestroy(pGroupbyExpr->columnInfo);
+    free(pGroupbyExpr);
+  } 
+  free(pTagColumnInfo);
+  free(pExprs);
   taosArrayDestroy(pTableIdList);
 
   //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
@@ -6134,17 +6152,17 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
-  size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+  size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList);
   assert(numOfGroup == 0 || numOfGroup == 1);
 
   if (numOfGroup == 0) {
     return;
   }
   
-  SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
+  SArray* pa = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0);
 
   size_t num = taosArrayGetSize(pa);
-  assert(num == pQInfo->groupInfo.numOfTables);
+  assert(num == pQInfo->tableqinfoGroupInfo.numOfTables);
 
   int32_t count = 0;
   int32_t functionId = pQuery->pSelectExpr[0].base.functionId;
@@ -6168,26 +6186,28 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
 
     while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) {
       int32_t i = pQInfo->tableIndex++;
-      SGroupItem *item = taosArrayGet(pa, i);
+      STableQueryInfo *item = taosArrayGetP(pa, i);
 
       char *output = pQuery->sdata[0]->data + i * rsize;
       varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);
 
       output = varDataVal(output);
-      *(int64_t *)output = item->id.uid;  // memory align problem, todo serialize
+      STableId id = tsdbGetTableId(item->pTable);
-      output += sizeof(item->id.uid);
 
-      *(int32_t *)output = item->id.tid;
+      *(int64_t *)output = id.uid;  // memory align problem, todo serialize
-      output += sizeof(item->id.tid);
+      output += sizeof(id.uid);
+
+      *(int32_t *)output = id.tid;
+      output += sizeof(id.tid);
 
       *(int32_t *)output = pQInfo->vgId;
       output += sizeof(pQInfo->vgId);
 
       if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
-        char *data = tsdbGetTableName(pQInfo->tsdb, &item->id);
+        char *data = tsdbGetTableName(item->pTable);
         memcpy(output, data, varDataTLen(data));
       } else {
-        char *val = tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo->base.colInfo.colId, type, bytes);
+        char *val = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes);
 
         // todo refactor
         if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
@@ -6223,18 +6243,18 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
       int32_t i = pQInfo->tableIndex++;
 
       SExprInfo* pExprInfo = pQuery->pSelectExpr;
-      SGroupItem* item = taosArrayGet(pa, i);
+      STableQueryInfo* item = taosArrayGetP(pa, i);
 
       for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
         if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
-          char* data = tsdbGetTableName(pQInfo->tsdb, &item->id);
+          char* data = tsdbGetTableName(item->pTable);
           char* dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes;
           memcpy(dst, data, varDataTLen(data));
         } else {// todo refactor
           int16_t type = pExprInfo[j].type;
           int16_t bytes = pExprInfo[j].bytes;
           
-          char* data = tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo[j].base.colInfo.colId, type, bytes);
+          char* data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes);
           char* dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes;
 
           if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {

src/query/src/qUtil.c

@@ -114,8 +114,6 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
     SWindowResult *pResult = &pWindowResInfo->pResult[i];
     if (pResult->status.closed) {  // remove the window slot from hash table
       taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type);
-      printf("remove ============>%ld, remain size:%ld\n", pResult->window.skey, pWindowResInfo->hashList->size);
-
     } else {
       break;
     }
@@ -140,7 +138,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
     SWindowResult *pResult = &pWindowResInfo->pResult[k];
     int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey,
         tDataTypeDesc[pWindowResInfo->type].nSize);
-    
+    assert(p != NULL); 
     int32_t  v = (*p - num);
     assert(v >= 0 && v <= pWindowResInfo->size);
     taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize,

src/query/src/qpercentile.c

@@ -879,8 +879,8 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
           UNUSED(ret);
 
           for (uint32_t jx = 0; jx < pFlushInfo->numOfPages; ++jx) {
-            ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
+            size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
-            UNUSED(ret);
+            UNUSED(sz);
             tMemBucketPut(pMemBucket, pPage->data, pPage->num);
           }
 
@@ -965,10 +965,11 @@ char *getFirstElemOfMemBuffer(tMemBucketSegment *pSeg, int32_t slotIdx, tFilePag
      */
     tFlushoutInfo *pFlushInfo = &pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[0];
     assert(pFlushInfo->numOfPages == pMemBuffer->fileMeta.nFileSize);
-
+    int32_t ret;
-    fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET);
+    ret = fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET);
-    size_t ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
     UNUSED(ret);
+    size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
+    UNUSED(sz);
     thisVal = pPage->data;
   }
   return thisVal;

src/query/src/qresultBuf.c

@@ -72,7 +72,7 @@ static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOf
   if (ret != 0) {
     //    dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
     //           strerror(errno));
-    return -TSDB_CODE_QRY_NO_DISKSPACE;
+    return TSDB_CODE_QRY_NO_DISKSPACE;
   }
 
   pResultBuf->totalBufSize = pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE;
@@ -80,7 +80,7 @@ static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOf
 
   if (pResultBuf->pBuf == MAP_FAILED) {
     //    dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
-    return -TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
   return TSDB_CODE_SUCCESS;

src/query/src/qtokenizer.c

@@ -25,7 +25,7 @@
 // All the keywords of the SQL language are stored in a hash table
 typedef struct SKeyword {
   const char* name;  // The keyword name
-  uint8_t     type;  // type
+  uint16_t     type;  // type
   uint8_t     len;   // length
 } SKeyword;
 

src/query/src/qtsbuf.c

@@ -65,8 +65,10 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
   
   // validate the file magic number
   STSBufFileHeader header = {0};
-  fseek(pTSBuf->f, 0, SEEK_SET);
+  int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET);
-  fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
+  UNUSED(ret);
+  size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
+  UNUSED(sz);
 
   // invalid file
   if (header.magic != TS_COMP_FILE_MAGIC) {
@@ -97,22 +99,30 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
   size_t infoSize = sizeof(STSVnodeBlockInfo) * pTSBuf->numOfVnodes;
   
   STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize);
+  if (buf == NULL) {
+    tsBufDestory(pTSBuf);
+    return NULL; 
+  } 
   
   //int64_t pos = ftell(pTSBuf->f); //pos not used
-  fread(buf, infoSize, 1, pTSBuf->f);
+  sz = fread(buf, infoSize, 1, pTSBuf->f);
+  UNUSED(sz);
   
   // the length value for each vnode is not kept in file, so does not set the length value
   for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) {
     STSVnodeBlockInfoEx* pBlockList = &pTSBuf->pData[i];
     memcpy(&pBlockList->info, &buf[i], sizeof(STSVnodeBlockInfo));
   }
-  
   free(buf);
   
-  fseek(pTSBuf->f, 0, SEEK_END);
+  ret = fseek(pTSBuf->f, 0, SEEK_END);
+  UNUSED(ret);
   
   struct stat fileStat;
-  fstat(fileno(pTSBuf->f), &fileStat);
+  if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
+    tsBufDestory(pTSBuf);
+    return NULL;
+  }
   
   pTSBuf->fileSize = (uint32_t)fileStat.st_size;
   tsBufResetPos(pTSBuf);
@@ -278,19 +288,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
      * set the right position for the reversed traverse, the reversed traverse is started from
      * the end of each comp data block
      */
-    fseek(pTSBuf->f, -sizeof(pBlock->padding), SEEK_CUR);
+    int32_t ret = fseek(pTSBuf->f, -sizeof(pBlock->padding), SEEK_CUR);
-    fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+    size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+    UNUSED(sz); 
     
     pBlock->compLen = pBlock->padding;
     int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
-    fseek(pTSBuf->f, -offset, SEEK_CUR);
+    ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
+    UNUSED(ret);
   }
   
-  fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f);
+  size_t sz = fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f);
-  fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
+  UNUSED(sz);
-  
+  sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
-  fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+  UNUSED(sz);
-  fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
+  sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+  UNUSED(sz);
+  sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
+  UNUSED(sz);
   
   if (decomp) {
     pTSBuf->tsData.len =
@@ -299,12 +314,13 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
   }
   
   // read the comp length at the length of comp block
-  fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+  sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+  UNUSED(sz);
   
   // for backwards traverse, set the start position at the end of previous block
   if (order == TSDB_ORDER_DESC) {
     int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
-    int64_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
+    int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
     UNUSED(r);
   }
   
@@ -441,7 +457,8 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int
     STSBlock* pBlock = &pTSBuf->block;
     int32_t   compBlockSize =
         pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
-    fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
+    int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
+    UNUSED(ret);
   }
   
   return 0;
@@ -538,7 +555,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
 
   assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
 
-  int64_t r = fseek(pTSBuf->f, 0, SEEK_SET);
+  int32_t r = fseek(pTSBuf->f, 0, SEEK_SET);
   if (r != 0) {
     return -1;
   }
@@ -743,7 +760,9 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
   int32_t oldSize = pDestBuf->fileSize;
   
   struct stat fileStat;
-  fstat(fileno(pDestBuf->f), &fileStat);
+  if (fstat(fileno(pDestBuf->f), &fileStat) != 0) {
+    return -1;  
+  }
   pDestBuf->fileSize = (uint32_t)fileStat.st_size;
   
   assert(pDestBuf->fileSize == oldSize + size);
@@ -766,8 +785,10 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
   // update prev vnode length info in file
   TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo);
   
-  fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
+  int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
-  fwrite((void*)pData, 1, len, pTSBuf->f);
+  UNUSED(ret);
+  size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f);
+  UNUSED(sz);
   pTSBuf->fileSize += len;
   
   pTSBuf->tsOrder = order;

src/rpc/inc/rpcTcp.h

@@ -21,9 +21,11 @@ extern "C" {
 #endif
 
 void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
+void taosStopTcpServer(void *param);
 void taosCleanUpTcpServer(void *param);
 
 void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle);
+void taosStopTcpClient(void *chandle);
 void taosCleanUpTcpClient(void *chandle);
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
 

src/rpc/inc/rpcUdp.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "taosdef.h"
 
 void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle);
+void  taosStopUdpConnection(void *handle);
 void  taosCleanUpUdpConnection(void *handle);
 int   taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle);
 void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);

src/rpc/src/rpcMain.c

@@ -58,7 +58,7 @@ typedef struct {
   void   (*cfp)(SRpcMsg *, SRpcIpSet *);
   int    (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); 
 
-  int      refCount;
+  int32_t   refCount;
   void     *idPool;   // handle to ID pool
   void     *tmrCtrl;  // handle to timer
   SHashObj *hash;     // handle returned by hash utility
@@ -153,6 +153,13 @@ void (*taosCleanUpConn[])(void *thandle) = {
     taosCleanUpTcpClient
 };
 
+void (*taosStopConn[])(void *thandle) = {
+    taosStopUdpConnection, 
+    taosStopUdpConnection, 
+    taosStopTcpServer,
+    taosStopTcpClient,
+};
+
 int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
     taosSendUdpData, 
     taosSendUdpData, 
@@ -289,12 +296,18 @@ void *rpcOpen(const SRpcInit *pInit) {
 void rpcClose(void *param) {
   SRpcInfo *pRpc = (SRpcInfo *)param;
 
+  // stop connection to outside first
+  (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
+  (*taosStopConn[pRpc->connType])(pRpc->udphandle);
+
+  // close all connections 
   for (int i = 0; i < pRpc->sessions; ++i) {
     if (pRpc->connList && pRpc->connList[i].user[0]) {
       rpcCloseConn((void *)(pRpc->connList + i));
     }
   }
 
+  // clean up
   (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
   (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
 
@@ -421,6 +434,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
   pConn->rspMsgLen = msgLen;
   if (pMsg->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) pConn->inTranId--;
 
+  // stop the progress timer
   taosTmrStopA(&pConn->pTimer);
 
   // set the idle timer to monitor the activity
@@ -588,6 +602,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
   pConn->inTranId = 0;
   pConn->outTranId = 0;
   pConn->secured = 0;
+  pConn->peerId = 0;
   pConn->peerIp = 0;
   pConn->peerPort = 0;
   pConn->pReqMsg = NULL;
@@ -627,6 +642,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
     pConn->spi = pRpc->spi;
     pConn->encrypt = pRpc->encrypt;
     if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN);
+    tTrace("%s %p client connection is allocated", pRpc->label, pConn);
   }
 
   return pConn;
@@ -681,6 +697,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
     }
   
     taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
+    tTrace("%s %p server connection is allocated", pRpc->label, pConn);
   }
 
   return pConn;
@@ -948,11 +965,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   terrno = 0;
   pConn = rpcProcessMsgHead(pRpc, pRecv);
 
-  if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) {
   tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
         pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, 
         pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
-  }
 
   int32_t code = terrno;
   if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) {
@@ -1007,7 +1022,11 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
   if ( rpcIsReq(pHead->msgType) ) {
     rpcMsg.handle = pConn;
     rpcAddRef(pRpc);  // add the refCount for requests
+
+    // start the progress timer to monitor the response from server app
     pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pRpc->tmrCtrl);
+ 
+    // notify the server app
     (*(pRpc->cfp))(&rpcMsg, NULL);
   } else {
     // it's a response
@@ -1180,13 +1199,11 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
   msgLen = rpcAddAuthPart(pConn, msg, msgLen);
 
   if ( rpcIsReq(pHead->msgType)) {
-    if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
     tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
            pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, 
            msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   } else {
     if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
-    if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
     tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
            pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, 
            htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
@@ -1480,12 +1497,12 @@ static void rpcUnlockConn(SRpcConn *pConn) {
 
 static void rpcAddRef(SRpcInfo *pRpc)
 {  
-   atomic_add_fetch_8(&pRpc->refCount, 1);
+   atomic_add_fetch_32(&pRpc->refCount, 1);
 }
 
 static void rpcDecRef(SRpcInfo *pRpc)
 { 
-  if (atomic_sub_fetch_8(&pRpc->refCount, 1) == 0) {
+  if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) {
     taosHashCleanup(pRpc->hash);
     taosTmrCleanUp(pRpc->tmrCtrl);
     taosIdPoolCleanUp(pRpc->idPool);

src/rpc/src/rpcTcp.c

@@ -190,22 +190,28 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   }
 }
 
-
+void taosStopTcpServer(void *handle) {
-void taosCleanUpTcpServer(void *handle) {
   SServerObj *pServerObj = handle;
-  SThreadObj *pThreadObj;
 
   if (pServerObj == NULL) return;
   if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD);
   if(pServerObj->thread) pthread_join(pServerObj->thread, NULL);
 
+  tTrace("%s TCP server is stopped", pServerObj->label);
+}
+
+void taosCleanUpTcpServer(void *handle) {
+  SServerObj *pServerObj = handle;
+  SThreadObj *pThreadObj;
+  if (pServerObj == NULL) return;
+
   for (int i = 0; i < pServerObj->numOfThreads; ++i) {
     pThreadObj = pServerObj->pThreadObj + i;
     taosStopTcpThread(pThreadObj);
     pthread_mutex_destroy(&(pThreadObj->mutex));
   }
 
-  tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label);
+  tTrace("%s TCP server is cleaned up", pServerObj->label);
 
   tfree(pServerObj->pThreadObj);
   tfree(pServerObj);
@@ -226,7 +232,7 @@ static void *taosAcceptTcpConnection(void *arg) {
     connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen);
     if (connFd == -1) {
       if (errno == EINVAL) {
-        tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label);
+        tTrace("%s TCP server stop accepting new connections, exiting", pServerObj->label);
         break;
       }
 
@@ -242,13 +248,13 @@ static void *taosAcceptTcpConnection(void *arg) {
     SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
     if (pFdObj) {
       pFdObj->ip = caddr.sin_addr.s_addr;
-      pFdObj->port = caddr.sin_port;
+      pFdObj->port = htons(caddr.sin_port);
-      tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, 
+      tTrace("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, 
-              inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
+              inet_ntoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds);
     } else {
       close(connFd);
       tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
-             inet_ntoa(caddr.sin_addr), caddr.sin_port);
+             inet_ntoa(caddr.sin_addr), htons(caddr.sin_port));
     }  
 
     // pick up next thread for next connection
@@ -304,12 +310,19 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *
   return pThreadObj;
 }
 
+void taosStopTcpClient(void *chandle) {
+  SThreadObj *pThreadObj = chandle;
+  if (pThreadObj == NULL) return;
+
+  tTrace ("%s TCP client is stopped", pThreadObj->label);
+}
+
 void taosCleanUpTcpClient(void *chandle) {
   SThreadObj *pThreadObj = chandle;
   if (pThreadObj == NULL) return;
 
   taosStopTcpThread(pThreadObj);
-  tTrace ("%s, all connections are cleaned up", pThreadObj->label);
+  tTrace ("%s TCP client is cleaned up", pThreadObj->label);
 
   tfree(pThreadObj);
 }
@@ -320,14 +333,22 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
   int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
   if (fd < 0) return NULL;
 
+  struct sockaddr_in sin;
+  uint16_t localPort = 0;
+  unsigned int addrlen = sizeof(sin);
+  if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 &&
+      sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
+    localPort = (uint16_t)ntohs(sin.sin_port);
+  }
+
   SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
   
   if (pFdObj) {
     pFdObj->thandle = thandle;
     pFdObj->port = port;
     pFdObj->ip = ip;
-    tTrace("%s %p, TCP connection to 0x%x:%hu is created, FD:%p numOfFds:%d", 
+    tTrace("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", 
-            pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds);
+            pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
   } else {
     close(fd);
     tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
@@ -340,7 +361,10 @@ void taosCloseTcpConnection(void *chandle) {
   SFdObj *pFdObj = chandle;
   if (pFdObj == NULL) return;
 
-  pFdObj->thandle = NULL;
+  SThreadObj *pThreadObj = pFdObj->pThreadObj;
+  tTrace("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); 
+
+  // pFdObj->thandle = NULL;
   pFdObj->closedByApp = 1;
   shutdown(pFdObj->fd, SHUT_WR);
 }
@@ -385,14 +409,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
 
   headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
   if (headLen != sizeof(SRpcHead)) {
-    tTrace("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
+    tTrace("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
     return -1; 
   }
 
   msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
   buffer = malloc(msgLen + tsRpcOverhead);
   if ( NULL == buffer) {
-    tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
+    tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
     return -1;
   }
 
@@ -401,8 +425,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
   retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
 
   if (leftLen != retLen) {
-    tError("%s %p, read error, leftLen:%d retLen:%d", 
+    tError("%s %p read error, leftLen:%d retLen:%d FD:%p", 
-            pThreadObj->label, pFdObj->thandle, leftLen, retLen);
+            pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
     free(buffer);
     return -1;
   }
@@ -446,19 +470,19 @@ static void *taosProcessTcpData(void *param) {
       pFdObj = events[i].data.ptr;
 
       if (events[i].events & EPOLLERR) {
-        tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle);
+        tTrace("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj);
         taosReportBrokenLink(pFdObj);
         continue;
       }
 
       if (events[i].events & EPOLLRDHUP) {
-        tTrace("%s %p, FD RD hang up", pThreadObj->label, pFdObj->thandle);
+        tTrace("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj);
         taosReportBrokenLink(pFdObj);
         continue;
       }
 
       if (events[i].events & EPOLLHUP) {
-        tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle);
+        tTrace("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj);
         taosReportBrokenLink(pFdObj);
         continue;
       }
@@ -527,7 +551,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
 
   pThreadObj->numOfFds--;
   if (pThreadObj->numOfFds < 0)
-    tError("%s %p, TCP thread:%d, number of FDs is negative!!!", 
+    tError("%s %p TCP thread:%d, number of FDs is negative!!!", 
             pThreadObj->label, pFdObj->thandle, pThreadObj->threadId);
 
   if (pFdObj->prev) {
@@ -542,7 +566,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
 
   pthread_mutex_unlock(&pThreadObj->mutex);
 
-  tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", 
+  tTrace("%s %p TCP connection is closed, FD:%p numOfFds:%d", 
           pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
 
   tfree(pFdObj);

src/rpc/src/rpcUdp.c

@@ -30,7 +30,6 @@
 #define RPC_MAX_UDP_SIZE 65480
 
 typedef struct {
-  void           *signature;
   int             index;
   int             fd;
   uint16_t        port;       // peer port
@@ -111,7 +110,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
     pConn->processData = fp;
     pConn->index = i;
     pConn->pSet = pSet;
-    pConn->signature = pConn;
 
     int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
     if (code != 0) {
@@ -132,7 +130,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
   return pSet;
 }
 
-void taosCleanUpUdpConnection(void *handle) {
+void taosStopUdpConnection(void *handle) {
   SUdpConnSet *pSet = (SUdpConnSet *)handle;
   SUdpConn    *pConn;
 
@@ -140,8 +138,6 @@ void taosCleanUpUdpConnection(void *handle) {
 
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
-    pConn->signature = NULL;
-
     if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR);
     if (pConn->fd >=0) taosCloseSocket(pConn->fd);
   }
@@ -150,9 +146,24 @@ void taosCleanUpUdpConnection(void *handle) {
     pConn = pSet->udpConn + i;
     if (pConn->thread) pthread_join(pConn->thread, NULL);
     tfree(pConn->buffer);
-    tTrace("%s UDP thread is closed, inedx:%d", pConn->label, i);
+    // tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
   }
 
+  tTrace("%s UDP is stopped", pSet->label);
+}
+
+void taosCleanUpUdpConnection(void *handle) {
+  SUdpConnSet *pSet = (SUdpConnSet *)handle;
+  SUdpConn    *pConn;
+
+  if (pSet == NULL) return;
+
+  for (int i = 0; i < pSet->threads; ++i) {
+    pConn = pSet->udpConn + i;
+    if (pConn->fd >=0) taosCloseSocket(pConn->fd);
+  }
+
+  tTrace("%s UDP is cleaned up", pSet->label);
   tfree(pSet);
 }
 
@@ -164,7 +175,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
   SUdpConn *pConn = pSet->udpConn + pSet->index;
   pConn->port = port;
 
-  tTrace("%s UDP connection is setup, ip:%x:%hu", pConn->label, ip, port);
+  tTrace("%s UDP connection is setup, ip:%x:%hu localPort:%hu", pConn->label, ip, port, pConn->localPort);
 
   return pConn;
 }
@@ -185,7 +196,7 @@ static void *taosRecvUdpData(void *param) {
   while (1) {
     dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
     if(dataLen <= 0) {
-      tTrace("%s UDP socket was closed, exiting", pConn->label);
+      tTrace("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno));
       break;
     }
 
@@ -221,7 +232,7 @@ static void *taosRecvUdpData(void *param) {
 int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) {
   SUdpConn *pConn = (SUdpConn *)chandle;
 
-  if (pConn == NULL || pConn->signature != pConn) return -1;
+  if (pConn == NULL) return -1;
 
   struct sockaddr_in destAdd;
   memset(&destAdd, 0, sizeof(destAdd));

src/tsdb/inc/tsdbMain.h

@@ -25,6 +25,7 @@
 #include "tsdb.h"
 #include "tskiplist.h"
 #include "tutil.h"
+#include "trwlatch.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -59,6 +60,7 @@ typedef struct STable {
   TSKEY          lastKey;        // lastkey inserted in this table, initialized as 0, TODO: make a structure
   char*          sql;
   void*          cqhandle;
+  SRWLatch       latch;  // TODO: implementa latch functions
   T_REF_DECLARE();
 } STable;
 

src/tsdb/src/tsdbBuffer.c

@@ -120,10 +120,13 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
   STsdbBufPool *pBufPool = pRepo->pPool;
 
   while (POOL_IS_EMPTY(pBufPool)) {
+    pRepo->repoLocked = false;
     pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex));
+    pRepo->repoLocked = true;
   }
 
   SListNode *    pNode = tdListPopHead(pBufPool->bufBlockList);
+  ASSERT(pNode != NULL);
   STsdbBufBlock *pBufBlock = NULL;
   tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
 

src/tsdb/src/tsdbFile.c

@@ -358,7 +358,9 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) {
   ASSERT(pFileH->nFGroups >= 0);
 
   for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
-    remove(fileGroup.files[type].fname);
+    if (remove(fileGroup.files[type].fname) < 0) {
+      tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname);
+    }
     tsdbDestroyFile(&fileGroup.files[type]);
   }
 }

src/tsdb/src/tsdbMain.c

@@ -179,7 +179,7 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *
       return -1;
     }
   }
-  pRsp->affectedRows = htonl(affectedrows);
+  if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
   return 0;
 }
 
@@ -213,10 +213,10 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
       SFileGroup *pFGroup =
           taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE);
       if (pFGroup->fileId == fid) {
-        strcpy(fname, pFGroup->files[(*index) % 3].fname);
+        fname = strdup(pFGroup->files[(*index) % 3].fname);
       } else {
         if (pFGroup->fileId * 3 + 2 < eindex) {
-          strcpy(fname, pFGroup->files[0].fname);
+          fname = strdup(pFGroup->files[0].fname);
           *index = pFGroup->fileId * 3;
         } else {
           tfree(sdup);
@@ -237,12 +237,13 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
       }
 
       SFile *pFile = &pFGroup->files[(*index) % 3];
-      strcpy(fname, pFile->fname);
+      fname = strdup(pFile->fname);
     }
   }
 
   if (stat(fname, &fState) < 0) {
     tfree(sdup);
+    tfree(fname);
     return 0;
   }
 
@@ -566,7 +567,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
 
 _err:
   tfree(fname);
-  if (fd > 0) close(fd);
+  if (fd >= 0) close(fd);
   return -1;
 }
 
@@ -609,7 +610,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
 
 _err:
   tfree(fname);
-  if (fd > 0) close(fd);
+  if (fd >= 0) close(fd);
   return -1;
 }
 
@@ -647,7 +648,7 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
   }
 
   pRepo->config = *pCfg;
-  pRepo->appH = *pAppH;
+  if (pAppH) pRepo->appH = *pAppH;
 
   pRepo->tsdbMeta = tsdbNewMeta(pCfg);
   if (pRepo->tsdbMeta == NULL) {

src/tsdb/src/tsdbMemTable.c

@@ -99,6 +99,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
   if (tSkipListPut(pTableData->pData, pNode) == NULL) {
     tsdbFreeBytes(pRepo, (void *)pNode, bytes);
   } else {
+    if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
     if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
     if (pMemTable->keyLast < key) pMemTable->keyLast = key;
     pMemTable->numOfRows++;
@@ -222,11 +223,12 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
     }
+    pRepo->commit = 0;
   }
 
   ASSERT(pRepo->commit == 0);
-  if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
   if (pRepo->mem != NULL) {
+    if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
     if (tsdbLockRepo(pRepo) < 0) return -1;
     pRepo->imem = pRepo->mem;
     pRepo->mem = NULL;
@@ -468,9 +470,6 @@ _err:
 
 static void tsdbEndCommit(STsdbRepo *pRepo) {
   ASSERT(pRepo->commit == 1);
-  tsdbLockRepo(pRepo);
-  pRepo->commit = 0;
-  tsdbUnlockRepo(pRepo);
   if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
 }
 
@@ -526,8 +525,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
     goto _err;
   }
 
-  free(dataDir);
-
   // Open files for write/read
   if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
     tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
@@ -590,6 +587,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
     goto _err;
   }
 
+  tfree(dataDir);
   tsdbCloseHelperFile(pHelper, 0);
 
   pthread_rwlock_wrlock(&(pFileH->fhlock));
@@ -601,7 +599,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
   return 0;
 
 _err:
-  // ASSERT(false);
+  tfree(dataDir);
   tsdbCloseHelperFile(pHelper, 1);
   return -1;
 }

src/tsdb/src/tsdbMeta.c

@@ -147,6 +147,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
       tsdbInsertTableAct(pRepo, TSDB_DROP_META, buf, tTable);
       tsdbRemoveTableFromMeta(pRepo, tTable, false, true);
     }
+    tSkipListDestroyIter(pIter);
   }
 
   tsdbRemoveTableFromMeta(pRepo, pTable, true, true);
@@ -157,18 +158,16 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
   return 0;
 }
 
-void *tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes) {
+void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes) {
   // TODO: this function should be changed also
-  STsdbMeta *pMeta = tsdbGetMeta(repo);
-  STable *   pTable = tsdbGetTableByUid(pMeta, id->uid);
 
-  STSchema *pSchema = tsdbGetTableTagSchema(pTable);
+  STSchema *pSchema = tsdbGetTableTagSchema((STable*) pTable);
   STColumn *pCol = tdGetColOfID(pSchema, colId);
   if (pCol == NULL) {
     return NULL;  // No matched tag volumn
   }
 
-  char *val = tdGetKVRowValOfCol(pTable->tagVal, colId);
+  char *val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId);
   assert(type == pCol->type && bytes == pCol->bytes);
 
   if (val != NULL && IS_VAR_DATA_TYPE(type)) {
@@ -178,20 +177,21 @@ void *tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, i
   return val;
 }
 
-char *tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id) {
+char *tsdbGetTableName(void* pTable) {
   // TODO: need to change as thread-safe
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
-
-  STable *pTable = tsdbGetTableByUid(pMeta, id->uid);
 
   if (pTable == NULL) {
     return NULL;
   } else {
-    return (char *)pTable->name;
+    return (char*) (((STable *)pTable)->name);
   }
 }
 
+STableId tsdbGetTableId(void *pTable) {
+  assert(pTable);
+  return ((STable*)pTable)->tableId;
+}
+
 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) {
   if (pMsg == NULL) return NULL;
 
@@ -270,7 +270,6 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) {
 _err:
   tdDestroyTSchemaBuilder(&schemaBuilder);
   tsdbClearTableCfg(pCfg);
-  tfree(pCfg);
   return NULL;
 }
 
@@ -309,6 +308,7 @@ int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) {
 
     int32_t code = tsdbUpdateTable(pRepo, super, pTableCfg);
     if (code != TSDB_CODE_SUCCESS) {
+      tsdbClearTableCfg(pTableCfg);
       return code;
     }
     tsdbClearTableCfg(pTableCfg);
@@ -788,6 +788,9 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) {
   }
 
   if (addIdx && tsdbUnlockRepoMeta(pRepo) < 0) return -1;
+  if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) {
+    pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, tsdbGetTableSchema(pTable));
+  }
 
   tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
             TABLE_TID(pTable), TABLE_UID(pTable));

src/tsdb/src/tsdbRWHelper.c

@@ -173,9 +173,14 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
       close(pHelper->files.nHeadF.fd);
       pHelper->files.nHeadF.fd = -1;
       if (hasError) {
-        remove(pHelper->files.nHeadF.fname);
+        (void)remove(pHelper->files.nHeadF.fname);
       } else {
-        rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname);
+        if (rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname) < 0) {
+          tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nHeadF.fname,
+                    pHelper->files.headF.fname, strerror(errno));
+          terrno = TAOS_SYSTEM_ERROR(errno);
+          return -1;
+        }
         pHelper->files.headF.info = pHelper->files.nHeadF.info;
       }
     }
@@ -186,9 +191,14 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
       close(pHelper->files.nLastF.fd);
       pHelper->files.nLastF.fd = -1;
       if (hasError) {
-        remove(pHelper->files.nLastF.fname);
+        (void)remove(pHelper->files.nLastF.fname);
       } else {
-        rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname);
+        if (rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname) < 0) {
+          tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nLastF.fname,
+                    pHelper->files.lastF.fname, strerror(errno));
+          terrno = TAOS_SYSTEM_ERROR(errno);
+          return -1;
+        }
         pHelper->files.lastF.info = pHelper->files.nLastF.info;
       }
     }
@@ -306,8 +316,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
 
     if (pCompBlock->numOfSubBlocks > 1) {
       if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1;
-      ASSERT(pHelper->pDataCols[0]->numOfRows > 0 &&
+      ASSERT(pHelper->pDataCols[0]->numOfRows > 0 && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
-             pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
       if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0],
                                pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0)
         return -1;
@@ -330,14 +339,27 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
 }
 
 int tsdbWriteCompInfo(SRWHelper *pHelper) {
+  off_t offset = 0;
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
   if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) {
     if (pIdx->offset > 0) {
-      pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
+      offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
-      if (pIdx->offset < 0) return -1;
+      if (offset < 0) {
+        tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname,
+                  strerror(errno));
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return -1;
+      }
+
+      pIdx->offset = offset;
       ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
-      if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1;
+      if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) {
+        tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
+                  pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno));
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return -1;
+      }
     }
   } else {
     pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER;
@@ -345,12 +367,23 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
     pHelper->pCompInfo->checksum = 0;
     ASSERT((pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0);
     taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len);
-    pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
+    offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
+    if (offset < 0) {
+      tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname,
+                strerror(errno));
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
+    pIdx->offset = offset;
     pIdx->uid = pHelper->tableInfo.uid;
-    if (pIdx->offset < 0) return -1;
     ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
-    if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1;
+    if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+      tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
+                pHelper->files.nHeadF.fname, strerror(errno));
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
   }
 
   return 0;
@@ -567,24 +600,24 @@ _err:
 static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
   ASSERT(pHelper->files.lastF.fd > 0);
   struct stat st;
-  fstat(pHelper->files.lastF.fd, &st);
+  if (fstat(pHelper->files.lastF.fd, &st) < 0) return true;
   if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true;
   return false;
 }
 
 static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite,
                                 SCompBlock *pCompBlock, bool isLast, bool isSuperBlock) {
-  STsdbCfg *pCfg = &(pHelper->pRepo->config);
+  STsdbCfg * pCfg = &(pHelper->pRepo->config);
   SCompData *pCompData = (SCompData *)(pHelper->pBuffer);
   int64_t    offset = 0;
 
   ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pCfg->maxRowsPerFileBlock);
   ASSERT(isLast ? rowsToWrite < pCfg->minRowsPerFileBlock : true);
 
-
   offset = lseek(pFile->fd, 0, SEEK_END);
   if (offset < 0) {
-    tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
+    tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname,
+              strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
@@ -639,8 +672,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
         }
       }
 
-      pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))(
+      pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr,
-          (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, pCfg->compression,
+                                                                  tsizeof(pHelper->pBuffer) - lsize, pCfg->compression,
                                                                   pHelper->compBuffer, tsizeof(pHelper->compBuffer));
     } else {
       pCompCol->len = tlen;
@@ -725,8 +758,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
   // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
 
   if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) {  // Merge with the last block by append
-    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock &&
+    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks - 1);
-           blkIdx == pIdx->numOfBlocks - 1);
     int defaultRowsToWrite = pCfg->maxRowsPerFileBlock * 4 / 5;  // TODO: make a interface
 
     rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfRows), pDataCols->numOfRows);
@@ -1099,10 +1131,8 @@ static int tsdbInitHelperBlock(SRWHelper *pHelper) {
   STsdbRepo *pRepo = helperRepo(pHelper);
   STsdbMeta *pMeta = pHelper->pRepo->tsdbMeta;
 
-  pHelper->pDataCols[0] =
+  pHelper->pDataCols[0] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
-      tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
+  pHelper->pDataCols[1] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
-  pHelper->pDataCols[1] =
-      tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
   if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     return -1;
@@ -1222,12 +1252,16 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32
 static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
   ASSERT(pCompBlock->numOfSubBlocks <= 1);
 
-  ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len);
-
   SCompData *pCompData = (SCompData *)pHelper->pBuffer;
 
   SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF);
 
+  pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompBlock->len);
+  if (pHelper->pBuffer == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    goto _err;
+  }
+
   int fd = pFile->fd;
   if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) {
     tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,

src/tsdb/src/tsdbRead.c

@@ -184,15 +184,17 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
     assert(gsize > 0);
     
     for (int32_t j = 0; j < gsize; ++j) {
-      STableId* id = (STableId*) taosArrayGet(group, j);
+      STable* pTable = (STable*) taosArrayGetP(group, j);
       
       STableCheckInfo info = {
           .lastKey = pQueryHandle->window.skey,
-          .tableId = *id,
+          .tableId = pTable->tableId,
-          .pTableObj = tsdbGetTableByUid(pMeta, id->uid),
+          .pTableObj = pTable,
       };
       
-      assert(info.pTableObj != NULL && info.pTableObj->tableId.tid == id->tid);
+      assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
+      info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
+
       taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
     }
   }
@@ -215,17 +217,17 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
   return pQueryHandle;
 }
 
-SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle) {
+SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   assert(pHandle != NULL);
   
   STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle;
   
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  SArray* res = taosArrayInit(size, sizeof(STableId));
+  SArray* res = taosArrayInit(size, POINTER_BYTES);
   
   for(int32_t i = 0; i < size; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
-    taosArrayPush(res, &pCheckInfo->tableId);
+    taosArrayPush(res, &pCheckInfo->pTableObj);
   }
   
   return res;
@@ -1052,7 +1054,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
 
         int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
         if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
-          tSkipListIterNext(pCheckInfo->iter);
+          moveToNextRow(pCheckInfo);
         }
         
         int32_t start = -1;
@@ -1607,6 +1609,7 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
   }
   
   if (index == -1) {
+    // todo add failure test cases
     return;
   }
   
@@ -1856,42 +1859,19 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
 
 SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
 
-TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) {
+static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
-  return NULL;
-}
-
-SArray* tsdbGetTableList(TsdbQueryHandleT* pQueryHandle) { return NULL; }
-
-static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
   SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex);
   while (tSkipListIterNext(iter)) {
     SSkipListNode* pNode = tSkipListIterGet(iter);
     
     STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
-    taosArrayPush(list, &(*pTable)->tableId);
+    taosArrayPush(list, pTable);
   }
   
   tSkipListDestroyIter(iter);
   return TSDB_CODE_SUCCESS;
 }
 
-/**
- * convert the result pointer to table id instead of table object pointer
- * todo remove it by using callback function to change the final result in-time.
- * @param pRes
- */
-static void convertQueryResult(SArray* pRes, SArray* pTableList) {
-  if (pTableList == NULL || taosArrayGetSize(pTableList) == 0) {
-    return;
-  }
-
-  size_t size = taosArrayGetSize(pTableList);
-  for (int32_t i = 0; i < size; ++i) {  // todo speedup  by using reserve space.
-    STable* pTable = taosArrayGetP(pTableList, i);
-    taosArrayPush(pRes, &pTable->tableId);
-  }
-}
-
 static void destroyHelper(void* param) {
   if (param == NULL) {
     return;
@@ -1960,16 +1940,13 @@ typedef struct STableGroupSupporter {
   int32_t    numOfCols;
   SColIndex* pCols;
   STSchema*  pTagSchema;
-  void*      tsdbMeta;
+//  void*      tsdbMeta;
 } STableGroupSupporter;
 
 int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
   STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param;
-  STableId* id1 = (STableId*) p1;
+  STable* pTable1 = *(STable**) p1;
-  STableId* id2 = (STableId*) p2;
+  STable* pTable2 = *(STable**) p2;
-  
-  STable *pTable1 = tsdbGetTableByUid(pTableGroupSupp->tsdbMeta, id1->uid);
-  STable *pTable2 = tsdbGetTableByUid(pTableGroupSupp->tsdbMeta, id2->uid);
   
   for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) {
     SColIndex* pColIndex = &pTableGroupSupp->pCols[i];
@@ -2019,26 +1996,29 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
   return 0;
 }
 
-void createTableGroupImpl(SArray* pGroups, SArray* pTableIdList, size_t numOfTables, STableGroupSupporter* pSupp,
+void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp,
     __ext_compar_fn_t compareFn) {
-  STableId* pId = taosArrayGet(pTableIdList, 0);
+  STable* pTable = taosArrayGetP(pTableList, 0);
   
-  SArray* g = taosArrayInit(16, sizeof(STableId));
+  SArray* g = taosArrayInit(16, POINTER_BYTES);
-  taosArrayPush(g, pId);
+  taosArrayPush(g, &pTable);
+  tsdbRefTable(pTable);
 
   for (int32_t i = 1; i < numOfTables; ++i) {
-    STableId* prev = taosArrayGet(pTableIdList, i - 1);
+    STable** prev = taosArrayGet(pTableList, i - 1);
-    STableId* p = taosArrayGet(pTableIdList, i);
+    STable** p = taosArrayGet(pTableList, i);
     
     int32_t ret = compareFn(prev, p, pSupp);
     assert(ret == 0 || ret == -1);
     
+    tsdbRefTable(*p);
+    assert((*p)->type == TSDB_CHILD_TABLE);
+
     if (ret == 0) {
       taosArrayPush(g, p);
     } else {
       taosArrayPush(pGroups, &g);  // current group is ended, start a new group
-      g = taosArrayInit(16, sizeof(STableId));
+      g = taosArrayInit(16, POINTER_BYTES);
-      
       taosArrayPush(g, p);
     }
   }
@@ -2046,8 +2026,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableIdList, size_t numOfTab
   taosArrayPush(pGroups, &g);
 }
 
-SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols,
+SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) {
-    TSDB_REPO_T* tsdb) {
   assert(pTableList != NULL);
   SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
   
@@ -2058,22 +2037,24 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
   }
   
   if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table
-    SArray* sa = taosArrayInit(size, sizeof(STableId));
+    SArray* sa = taosArrayInit(size, POINTER_BYTES);
     for(int32_t i = 0; i < size; ++i) {
-      STableId* tableId = taosArrayGet(pTableList, i);
+      STable** pTable = taosArrayGet(pTableList, i);
-      taosArrayPush(sa, tableId);
+      assert((*pTable)->type == TSDB_CHILD_TABLE);
+
+      tsdbRefTable(*pTable);
+      taosArrayPush(sa, pTable);
     }
     
     taosArrayPush(pTableGroup, &sa);
     tsdbTrace("all %zu tables belong to one group", size);
   } else {
     STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter));
-    pSupp->tsdbMeta = tsdbGetMeta(tsdb);
     pSupp->numOfCols = numOfOrderCols;
     pSupp->pTagSchema = pTagSchema;
     pSupp->pCols = pCols;
     
-    taosqsort(pTableList->pData, size, sizeof(STableId), pSupp, tableGroupComparFn);
+    taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn);
     createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn);
     tfree(pSupp);
   }
@@ -2149,48 +2130,53 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
       .pExtInfo = pSTable->tagSchema,
       };
 
-  SArray* pTableList = taosArrayInit(8, POINTER_BYTES);
+  tExprTreeTraverse(pExpr, pSTable->pIndex, pRes, &supp);
-
-  tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp);
   tExprTreeDestroy(&pExpr, destroyHelper);
-
-  convertQueryResult(pRes, pTableList);
-  taosArrayDestroy(pTableList);
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pTagCond, size_t len,
                                  int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
                                  SColIndex* pColIndex, int32_t numOfCols) {
+  if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
+
   STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
   if (pTable == NULL) {
     tsdbError("%p failed to get stable, uid:%" PRIu64, tsdb, uid);
-    return TSDB_CODE_TDB_INVALID_TABLE_ID;
+    terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+    tsdbUnlockRepoMeta(tsdb);
+
+    goto _error;
   }
   
   if (pTable->type != TSDB_SUPER_TABLE) {
     tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid,
         pTable->name->data);
+    terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
 
-    return TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
+    tsdbUnlockRepoMeta(tsdb);
+    goto _error;
   }
 
-  SArray* res = taosArrayInit(8, sizeof(STableId));
+  //NOTE: not add ref count for super table
+  SArray* res = taosArrayInit(8, POINTER_BYTES);
   STSchema* pTagSchema = tsdbGetTableTagSchema(pTable);
   
   // no tags and tbname condition, all child tables of this stable are involved
   if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {
-    int32_t ret = getAllTableIdList(pTable, res);
+    int32_t ret = getAllTableList(pTable, res);
-    if (ret == TSDB_CODE_SUCCESS) {
+    if (ret != TSDB_CODE_SUCCESS) {
-      pGroupInfo->numOfTables = taosArrayGetSize(res);
+      tsdbUnlockRepoMeta(tsdb);
-      pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb);
+      goto _error;
-      
-      tsdbTrace("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
-    } else {
-      // todo add error
     }
 
+    pGroupInfo->numOfTables = taosArrayGetSize(res);
+    pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols);
+      
+    tsdbTrace("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
     taosArrayDestroy(res);
+
+    if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
     return ret;
   }
 
@@ -2227,31 +2213,84 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
   doQueryTableList(pTable, res, expr);
   pGroupInfo->numOfTables = taosArrayGetSize(res);
-  pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb);
+  pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols);
 
   tsdbTrace("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%zu, belong to %zu groups", tsdb, pTable->tableId.tid,
       pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList));
 
   taosArrayDestroy(res);
+
+  if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
   return ret;
+
+  _error:
+  return terrno;
 }
 
 int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) {
+  if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
+
   STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
   if (pTable == NULL) {
-    return TSDB_CODE_TDB_INVALID_TABLE_ID;
+    terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+    tsdbUnlockRepoMeta(tsdb);
+    goto _error;
   }
 
-  //todo assert table type, add the table ref count
+  assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE);
+  tsdbRefTable(pTable);
+  if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
+
   pGroupInfo->numOfTables = 1;
   pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
   
-  SArray* group = taosArrayInit(1, sizeof(STableId));
+  SArray* group = taosArrayInit(1, POINTER_BYTES);
   
-  taosArrayPush(group, &pTable->tableId);
+  taosArrayPush(group, &pTable);
   taosArrayPush(pGroupInfo->pGroupList, &group);
   
   return TSDB_CODE_SUCCESS;
+
+  _error:
+  return terrno;
+}
+
+int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) {
+  if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
+
+  assert(pTableIdList != NULL);
+  size_t size = taosArrayGetSize(pTableIdList);
+  pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
+  SArray* group = taosArrayInit(1, POINTER_BYTES);
+
+  int32_t i = 0;
+  for(; i < size; ++i) {
+    STableIdInfo *id = taosArrayGet(pTableIdList, i);
+
+    STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), id->uid);
+    if (pTable == NULL) {
+      tsdbWarn("table uid:%"PRIu64", tid:%d has been drop already", id->uid, id->tid);
+      continue;
+    }
+
+    if (pTable->type == TSDB_SUPER_TABLE) {
+      tsdbError("direct query on super tale is not allowed, table uid:%"PRIu64", tid:%d", id->uid, id->tid);
+      terrno = TSDB_CODE_QRY_INVALID_MSG;
+    }
+
+    tsdbRefTable(pTable);
+    taosArrayPush(group, &pTable);
+  }
+
+  if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
+
+  pGroupInfo->numOfTables = i;
+  taosArrayPush(pGroupInfo->pGroupList, &group);
+
+  return TSDB_CODE_SUCCESS;
+
+  _error:
+  return terrno;
 }
 
 void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
@@ -2263,12 +2302,11 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   for (int32_t i = 0; i < size; ++i) {
     STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
+    tSkipListDestroyIter(pTableCheckInfo->iter);
 
     tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->mem);
     tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->imem);
 
-    tSkipListDestroyIter(pTableCheckInfo->iter);
-
     if (pTableCheckInfo->pDataCols != NULL) {
       tfree(pTableCheckInfo->pDataCols->buf);
     }
@@ -2293,3 +2331,26 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   
   tfree(pQueryHandle);
 }
+
+void tsdbDestoryTableGroup(STableGroupInfo *pGroupList) {
+  assert(pGroupList != NULL);
+
+  size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList);
+
+  for(int32_t i = 0; i < numOfGroup; ++i) {
+    SArray* p = taosArrayGetP(pGroupList->pGroupList, i);
+
+    size_t numOfTables = taosArrayGetSize(p);
+    for(int32_t j = 0; j < numOfTables; ++j) {
+      STable* pTable = taosArrayGetP(p, j);
+      assert(pTable != NULL);
+
+      tsdbUnRefTable(pTable);
+    }
+
+    taosArrayDestroy(p);
+  }
+
+  taosArrayDestroy(pGroupList->pGroupList);
+}
+

src/tsdb/tests/CMakeLists.txt

@@ -1,6 +1,6 @@
 aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
 add_executable(tsdbTests ${SOURCE_LIST})
-target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb)
+target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb tutil trpc)
 
 add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests)

src/tsdb/tests/tsdbTests.cpp

@@ -2,9 +2,8 @@
 #include <stdlib.h>
 #include <sys/time.h>
 
-#include "tdataformat.h"
+#include "tsdb.h"
 #include "tsdbMain.h"
-#include "tskiplist.h"
 
 static double getCurTime() {
   struct timeval tv;
@@ -77,7 +76,7 @@ static int insertData(SInsertInfo *pInfo) {
     pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
     pMsg->compressed = htonl(pMsg->numOfBlocks);
 
-    if (tsdbInsertData(pInfo->pRepo, pMsg) < 0) {
+    if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
       tfree(pMsg);
       return -1;
     }
@@ -90,222 +89,72 @@ static int insertData(SInsertInfo *pInfo) {
   return 0;
 }
 
-TEST(TsdbTest, DISABLED_tableEncodeDecode) {
+static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, int32_t totalBlocks, int32_t maxTables,
-// TEST(TsdbTest, tableEncodeDecode) {
+                       int32_t daysPerFile, int32_t keep, int32_t minRows, int32_t maxRows, int8_t precision,
-  STable *pTable = (STable *)malloc(sizeof(STable));
+                       int8_t compression) {
-
+  pCfg->tsdbId = tsdbId;
-  pTable->type = TSDB_NORMAL_TABLE;
+  pCfg->cacheBlockSize = cacheBlockSize;
-  pTable->tableId.uid = 987607499877672L;
+  pCfg->totalBlocks = totalBlocks;
-  pTable->tableId.tid = 0;
+  pCfg->maxTables = maxTables;
-  pTable->superUid = -1;
+  pCfg->daysPerFile = daysPerFile;
-  pTable->sversion = 0;
+  pCfg->keep = keep;
-  pTable->tagSchema = NULL;
+  pCfg->minRowsPerFileBlock = minRows;
-  pTable->tagVal = NULL;
+  pCfg->maxRowsPerFileBlock = maxRows;
-  int nCols = 5;
+  pCfg->precision = precision;
-  STSchema *schema = tdNewSchema(nCols);
+  pCfg->compression = compression;
-
-  for (int i = 0; i < nCols; i++) {
-    if (i == 0) {
-      tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
-    } else {
-      tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
-    }
-  }
-
-  pTable->schema = schema;
-
-  int bufLen = 0;
-  void *buf = tsdbEncodeTable(pTable, &bufLen);
-
-  STable *tTable = tsdbDecodeTable(buf, bufLen);
-
-  ASSERT_EQ(pTable->type, tTable->type);
-  ASSERT_EQ(pTable->tableId.uid, tTable->tableId.uid);
-  ASSERT_EQ(pTable->tableId.tid, tTable->tableId.tid);
-  ASSERT_EQ(pTable->superUid, tTable->superUid);
-  ASSERT_EQ(pTable->sversion, tTable->sversion);
-  ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0);
 }
 
-// TEST(TsdbTest, DISABLED_createRepo) {
+static void tsdbSetTableCfg(STableCfg *pCfg) {
-TEST(TsdbTest, createRepo) {
+  STSchemaBuilder schemaBuilder = {0};
-  STsdbCfg config;
-  STsdbRepo *repo;
 
-  // 1. Create a tsdb repository
+  pCfg->type = TSDB_NORMAL_TABLE;
-  tsdbSetDefaultCfg(&config);
+  pCfg->superUid = TSDB_INVALID_SUPER_TABLE_ID;
-  ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0);
+  pCfg->tableId.tid = 1;
+  pCfg->tableId.uid = 5849583783847394;
+  tdInitTSchemaBuilder(&schemaBuilder, 0);
 
-  TSDB_REPO_T *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
+  int colId = 0;
-  ASSERT_NE(pRepo, nullptr);
+  for (int i = 0; i < 5; i++) {
-
+    tdAddColToSchema(&schemaBuilder, (colId == 0) ? TSDB_DATA_TYPE_TIMESTAMP : TSDB_DATA_TYPE_INT, colId, 0);
-  // 2. Create a normal table
+    colId++;
-  STableCfg tCfg;
-  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
-  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
-  tsdbTableSetName(&tCfg, "test", false);
-
-  int       nCols = 5;
-  STSchema *schema = tdNewSchema(nCols);
-
-  for (int i = 0; i < nCols; i++) {
-    if (i == 0) {
-      tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
-    } else {
-      tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
-    }
   }
 
-  tsdbTableSetSchema(&tCfg, schema, true);
+  pCfg->schema = tdGetSchemaFromBuilder(&schemaBuilder);
+  pCfg->name = strdup("t1");
 
-  tsdbCreateTable(pRepo, &tCfg);
+  tdDestroyTSchemaBuilder(&schemaBuilder);
-
-  // Insert Some Data
-  SInsertInfo iInfo = {
-    .pRepo = pRepo,
-    // .isAscend = true,
-    .isAscend = false,
-    .tid = tCfg.tableId.tid,
-    .uid = tCfg.tableId.uid,
-    .sversion = tCfg.sversion,
-    .startTime = 1584081000000,
-    .interval = 1000,
-    .totalRows = 10000000,
-    .rowsPerSubmit = 1,
-    .pSchema = schema
-  };
-
-  ASSERT_EQ(insertData(&iInfo), 0);
-
-  // Close the repository
-  tsdbCloseRepo(pRepo);
-
-  // Open the repository again
-  pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
-  repo = (STsdbRepo *)pRepo;
-  ASSERT_NE(pRepo, nullptr);
-
-  // // Insert more data
-  // iInfo.startTime = iInfo.startTime + iInfo.interval * iInfo.totalRows;
-  // iInfo.totalRows = 10;
-  // iInfo.pRepo = pRepo;
-  // ASSERT_EQ(insertData(&iInfo), 0);
-
-  // // Close the repository
-  // tsdbCloseRepo(pRepo);
-
-  // // Open the repository again
-  // pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
-  // repo = (STsdbRepo *)pRepo;
-  // ASSERT_NE(pRepo, nullptr);
-
-  // // Read from file
-  // SRWHelper rhelper;
-  // tsdbInitReadHelper(&rhelper, repo);
-
-  // SFileGroup *pFGroup = tsdbSearchFGroup(repo->tsdbFileH, 1833);
-  // ASSERT_NE(pFGroup, nullptr);
-  // ASSERT_GE(tsdbSetAndOpenHelperFile(&rhelper, pFGroup), 0);
-
-  // STable *pTable = tsdbGetTableByUid(repo->tsdbMeta, tCfg.tableId.uid);
-  // ASSERT_NE(pTable, nullptr);
-  // tsdbSetHelperTable(&rhelper, pTable, repo);
-
-  // ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0);
-  // ASSERT_EQ(tsdbLoadBlockData(&rhelper, blockAtIdx(&rhelper, 0), NULL), 0);
-
-  int k = 0;
 }
 
-TEST(TsdbTest, DISABLED_openRepo) {
+TEST(TsdbTest, testInsertSpeed) {
-// TEST(TsdbTest, openRepo) {
+  int         vnode = 1;
-  // tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
+  int         ret = 0;
-  // ASSERT_NE(repo, nullptr);
+  STsdbCfg    tsdbCfg;
+  STableCfg   tableCfg;
+  std::string testDir = "./test";
+  char *      rootDir = strdup((testDir + "/vnode" + std::to_string(vnode)).c_str());
 
-  // STsdbRepo *pRepo = (STsdbRepo *)repo;
+  tsdbDebugFlag = 131; //NOTE: you must set the flag
 
-  // SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655);
+  taosRemoveDir(rootDir);
 
-//   for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+  // Create and open repository
-//     tsdbOpenFile(&pGroup->files[type], O_RDONLY);
+  tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1);
-//   }
+  tsdbCreateRepo(rootDir, &tsdbCfg);
+  TSDB_REPO_T *repo = tsdbOpenRepo(rootDir, NULL);
+  ASSERT_NE(repo, nullptr);
 
-//   SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx));
+  // Create table
-//   tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables);
+  tsdbSetTableCfg(&tableCfg);
+  tsdbCreateTable(repo, &tableCfg);
 
-//   SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len);
+  // Insert data
+  SInsertInfo iInfo = {repo, true, 1, 5849583783847394, 0, 1590000000000, 10, 10000000, 100, tableCfg.schema};
 
-  // tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo);
+  insertData(&iInfo);
 
-//   int blockIdx = 0;
+  tsdbCloseRepo(repo, 1);
-//   SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
-
-//   SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
-
-//   tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
-
-  // STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
-  // SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5);
-  // tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
-
-//   tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
-
-  // tdResetDataCols(pDataCols);
-
-  // tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData);
-
-
-//   int k = 0;
-
-}
-
-TEST(TsdbTest, DISABLED_createFileGroup) {
-  SFileGroup fGroup;
-
-  // ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0);
-
-  int k = 0;
 }
 
 static char *getTKey(const void *data) {
   return (char *)data;
 }
-
-static void insertSkipList(bool isAscend) {
-  TSKEY start_time = 1587393453000;
-  TSKEY interval = 1000;
-
-  SSkipList *pList = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(TSKEY), 0, 0, 1, getTKey);
-  ASSERT_NE(pList, nullptr);
-
-  for (size_t i = 0; i < 20000000; i++)
-  {
-    TSKEY time = isAscend ? (start_time + i * interval) : (start_time - i * interval);
-    int32_t level = 0;
-    int32_t headSize = 0;
-
-    tSkipListNewNodeInfo(pList, &level, &headSize);
-    SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(TSKEY));
-    ASSERT_NE(pNode, nullptr);
-    pNode->level = level;
-    *(TSKEY *)((char *)pNode + headSize) = time;
-    tSkipListPut(pList, pNode);
-  }
-
-  tSkipListDestroy(pList);
-}
-
-TEST(TsdbTest, DISABLED_testSkipList) {
-// TEST(TsdbTest, testSkipList) {
-  double stime = getCurTime();
-  insertSkipList(true);
-  double etime = getCurTime();
-
-  printf("Time used to insert 100000000 records takes %f seconds\n", etime-stime);
-
-  stime = getCurTime();
-  insertSkipList(false);
-  etime = getCurTime();
-
-  printf("Time used to insert 100000000 records takes %f seconds\n", etime-stime);
-}

src/util/inc/trwlatch.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __TD_RWLATCH_H__
+#define __TD_RWLATCH_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+typedef int32_t SRWLatch;
+
+void taosInitRWLatch(SRWLatch *pLatch);
+void taosWLockLatch(SRWLatch *pLatch);
+void taosWUnLockLatch(SRWLatch *pLatch);
+void taosRLockLatch(SRWLatch *pLatch);
+void taosRUnLockLatch(SRWLatch *pLatch);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/util/inc/tstoken.h

@@ -24,14 +24,7 @@ extern "C" {
 #include "tutil.h"
 #include "ttokendef.h"
 
-#define TK_SPACE      200
+
-#define TK_COMMENT    201
-#define TK_ILLEGAL    202
-#define TK_HEX        203   // hex number  0x123
-#define TK_OCT        204   // oct number
-#define TK_BIN        205   // bin format data 0b111
-#define TK_FILE       206
-#define TK_QUESTION   207   // denoting the placeholder of "?",when invoking statement bind query
 
 #define TSQL_TBNAME   "TBNAME"
 #define TSQL_TBNAME_L "tbname"

src/util/inc/ttimer.h

@@ -24,7 +24,7 @@ typedef void *tmr_h;
 typedef void (*TAOS_TMR_CALLBACK)(void *, void *);
 
 extern int      taosTmrThreads;
-extern uint32_t taosMaxTmrCtrl;
+extern uint32_t tsMaxTmrCtrl;
 
 #define MSECONDS_PER_TICK 5
 

src/util/inc/tutil.h

@@ -119,6 +119,8 @@ extern "C" {
 
 uint32_t taosRand(void);
 
+uint32_t trand(void);
+
 size_t twcslen(const wchar_t *wcs);
 
 int32_t strdequote(char *src);
@@ -133,7 +135,7 @@ char* strtolower(char *dst, const char *src);
 
 int64_t strnatoi(char *num, int32_t len);
 
-char* strreplace(const char* str, const char* pattern, const char* rep);
+//char* strreplace(const char* str, const char* pattern, const char* rep);
 
 char *strbetween(char *string, char *begin, char *end);
 
@@ -181,6 +183,7 @@ char *taosIpStr(uint32_t ipInt);
 uint32_t ip2uint(const char *const ip_addr);
 
 void taosRemoveDir(char *rootDir);
+int  tmkdir(const char *pathname, mode_t mode); 
 
 #define TAOS_ALLOC_MODE_DEFAULT 0
 #define TAOS_ALLOC_MODE_RANDOM_FAIL 1

src/util/src/hash.c

@@ -468,7 +468,8 @@ void taosHashTableResize(SHashObj *pHashObj) {
     return;
   }
 
-  void *pNewEntry = realloc(pHashObj->hashList, POINTER_BYTES * newSize);
+  int32_t pointerSize = POINTER_BYTES;
+  void *pNewEntry = realloc(pHashObj->hashList, pointerSize * newSize);
   if (pNewEntry == NULL) {// todo handle error
 //    uTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity);
     return;

src/util/src/tconfig.c

@@ -111,21 +111,18 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
         wordfree(&full_path);
         return;
       }
+      
       if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
         strcpy(option, full_path.we_wordv[0]);
       }
+      
       wordfree(&full_path);
 
-      struct stat dirstat;
+      int code = tmkdir(option, 0755);
-      if (stat(option, &dirstat) < 0) {
+      if (code != 0) {
-        int code = mkdir(option, 0755);
+        terrno = TAOS_SYSTEM_ERROR(errno);
-        if (code < 0) {
+        uError("config option:%s, input value:%s, directory not exist, create fail:%s",
-          uError("config option:%s, input value:%s, directory not exist, create fail with return code:%d",
+             cfg->option, input_value, strerror(errno)); 
-               cfg->option, input_value, code); 
-        } else {
-          uPrint("config option:%s, input value:%s, directory not exist, create with return code:%d",
-               cfg->option, input_value, code); 
-        }
       }
       cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
     } else {

src/util/src/tkvstore.c

@@ -78,8 +78,8 @@ int tdCreateKVStore(char *fname) {
   return 0;
 
 _err:
-  if (fd > 0) close(fd);
+  if (fd >= 0) close(fd);
-  remove(fname);
+  (void)remove(fname);
   return -1;
 }
 
@@ -106,15 +106,15 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
     goto _err;
   }
 
-  if (access(pStore->fsnap, F_OK) == 0) { // .snap file exists
-    uTrace("file %s exists, try to recover the KV store", pStore->fsnap);
   pStore->sfd = open(pStore->fsnap, O_RDONLY);
   if (pStore->sfd < 0) {
+    if (errno != ENOENT) {
       uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-
+  } else {
+    uTrace("file %s exists, try to recover the KV store", pStore->fsnap);
     if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info) < 0) {
       if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err;
     } else {
@@ -133,7 +133,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH
 
     close(pStore->sfd);
     pStore->sfd = -1;
-    remove(pStore->fsnap);
+    (void)remove(pStore->fsnap);
   }
 
   if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
@@ -212,7 +212,7 @@ _err:
   if (pStore->sfd > 0) {
     close(pStore->sfd);
     pStore->sfd = -1;
-    remove(pStore->fsnap);
+    (void)remove(pStore->fsnap);
   }
   if (pStore->fd > 0) {
     close(pStore->fd);
@@ -314,7 +314,7 @@ int tdKVStoreEndCommit(SKVStore *pStore) {
   }
   pStore->fd = -1;
 
-  remove(pStore->fsnap);
+  (void)remove(pStore->fsnap);
   return 0;
 }
 

src/util/src/tlog.c

@@ -147,7 +147,7 @@ static void *taosThreadToOpenNewFile(void *param) {
     return NULL;
   }
   taosLockFile(fd);
-  lseek(fd, 0, SEEK_SET);
+  (void)lseek(fd, 0, SEEK_SET);
 
   int32_t oldFd = tsLogObj.logHandle->fd;
   tsLogObj.logHandle->fd = fd;
@@ -276,14 +276,15 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
     }
   }
 
-  sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
+  char fileName[LOG_FILE_NAME_LEN + 50] = "\0";
+  sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag);
   pthread_mutex_init(&tsLogObj.logMutex, NULL);
 
   umask(0);
-  tsLogObj.logHandle->fd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+  tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
   if (tsLogObj.logHandle->fd < 0) {
-    printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno));
+    printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
   taosLockFile(tsLogObj.logHandle->fd);
@@ -291,7 +292,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   // only an estimate for number of lines
   struct stat filestat;
   if (fstat(tsLogObj.logHandle->fd, &filestat) < 0) {
-    printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
+    printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
   size = (int32_t)filestat.st_size;

src/util/src/tnote.c

@@ -92,7 +92,7 @@ void *taosThreadToOpenNewNote(void *param)
     }
 
     taosLockNote(fd, pNote);
-    lseek(fd, 0, SEEK_SET);
+    (void)lseek(fd, 0, SEEK_SET);
 
     int oldFd = pNote->taosNoteFd;
     pNote->taosNoteFd = fd;
@@ -128,10 +128,12 @@ int taosOpenNewNote(taosNoteInfo * pNote)
 
 bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote)
 {
+    /*
     int exist = access(noteName, F_OK);
     if (exist != 0) {
         return false;
     }
+    */
 
     int fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
     if (fd < 0) {
@@ -206,14 +208,15 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
         }
     }
 
-    sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
+    char noteName[NOTE_FILE_NAME_LEN * 2] = "\0";
+    sprintf(noteName, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
     pthread_mutex_init(&pNote->taosNoteMutex, NULL);
 
     umask(0);
-    pNote->taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+    pNote->taosNoteFd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
     if (pNote->taosNoteFd < 0) {
-        fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno));
+        fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno));
         return -1;
     }
     taosLockNote(pNote->taosNoteFd, pNote);
@@ -221,7 +224,7 @@ int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInf
     // only an estimate for number of lines
     struct stat filestat;
     if (fstat(pNote->taosNoteFd, &filestat) < 0) {
-      fprintf(stderr, "failed to fstat note file:%s reason:%s\n", name, strerror(errno));
+      fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno));
       return -1;
     }    
     size = (int)filestat.st_size;

src/util/src/trwlatch.c

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+// #define _GNU_SOURCE
+// #include <pthread.h>
+
+#include "trwlatch.h"
+#include "os.h"
+
+#define TD_RWLATCH_WRITE_FLAG 0x40000000
+
+void taosInitRWLatch(SRWLatch *pLatch) { *pLatch = 0; }
+
+void taosWLockLatch(SRWLatch *pLatch) {
+  SRWLatch oLatch, nLatch;
+  int      nLoops = 0;
+
+  // Set write flag
+  while (1) {
+    oLatch = atomic_load_32(pLatch);
+    if (oLatch & TD_RWLATCH_WRITE_FLAG) {
+      nLoops++;
+      if (nLoops > 1000) {
+        sched_yield();
+        nLoops = 0;
+      }
+      continue;
+    }
+
+    nLatch = oLatch | TD_RWLATCH_WRITE_FLAG;
+    if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) break;
+  }
+
+  // wait for all reads end
+  nLoops = 0;
+  while (1) {
+    oLatch = atomic_load_32(pLatch);
+    if (oLatch == TD_RWLATCH_WRITE_FLAG) break;
+    nLoops++;
+    if (nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+}
+
+void taosWUnLockLatch(SRWLatch *pLatch) { atomic_store_32(pLatch, 0); }
+
+void taosRLockLatch(SRWLatch *pLatch) {
+  SRWLatch oLatch, nLatch;
+  int      nLoops = 0;
+
+  while (1) {
+    oLatch = atomic_load_32(pLatch);
+    if (oLatch & TD_RWLATCH_WRITE_FLAG) {
+      nLoops++;
+      if (nLoops > 1000) {
+        sched_yield();
+        nLoops = 0;
+      }
+      continue;
+    }
+
+    nLatch = oLatch + 1;
+    if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) break;
+  }
+}
+
+void taosRUnLockLatch(SRWLatch *pLatch) { atomic_fetch_sub_32(pLatch, 1); }

src/util/src/tskiplist.c

@@ -194,13 +194,12 @@ void *tSkipListDestroy(SSkipList *pSkipList) {
     pthread_rwlock_wrlock(pSkipList->lock);
   }
 
+  if (pSkipList->keyInfo.freeNode) {
     SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0);
 
     while (pNode != pSkipList->pTail) {
       SSkipListNode *pTemp = pNode;
       pNode = SL_GET_FORWARD_POINTER(pNode, 0);
-    
-    if (pSkipList->keyInfo.freeNode) {
       tfree(pTemp);
     }
   }

src/util/src/tsocket.c

@@ -222,9 +222,7 @@ int taosReadn(int fd, char *ptr, int nbytes) {
 int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
   struct sockaddr_in localAddr;
   int                sockFd;
-  int                ttl = 128;
+  int                bufSize = 1024000;
-  int                reuse, nocheck;
-  int                bufSize = 8192000;
 
   uTrace("open udp socket:0x%x:%hu", ip, port);
 
@@ -238,31 +236,6 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
     return -1;
   }
 
-  reuse = 1;
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
-    uError("setsockopt SO_REUSEADDR failed): %d (%s)", errno, strerror(errno));
-    close(sockFd);
-    return -1;
-  };
-
-  nocheck = 1;
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_NO_CHECK, (void *)&nocheck, sizeof(nocheck)) < 0) {
-    if (!taosSkipSocketCheck()) {
-      uError("setsockopt SO_NO_CHECK failed: %d (%s)", errno, strerror(errno));
-      close(sockFd);
-      return -1;
-    } else {
-      uPrint("Skipping setsockopt SO_NO_CHECK error: %d (%s)", errno, strerror(errno));
-    }
-  }
-
-  ttl = 128;
-  if (taosSetSockOpt(sockFd, IPPROTO_IP, IP_TTL, (void *)&ttl, sizeof(ttl)) < 0) {
-    uError("setsockopt IP_TTL failed: %d (%s)", errno, strerror(errno));
-    close(sockFd);
-    return -1;
-  }
-
   if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
     uError("failed to set the send buffer size for UDP socket\n");
     close(sockFd);

src/util/src/ttime.c

@@ -56,8 +56,13 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0,
 		year -= 1;
 	}
 
-  int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
+  //int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
-		  year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
+	//	  year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
+  int64_t res;
+  res  = 367*((int64_t)mon)/12;
+  res += year/4 - year/100 + year/400 + day + year*365 - 719499;
+  res  = res*24;
+  res  = ((res + hour) * 60 + min) * 60 + sec;  
 
 	return (res + timezone);
 }

src/util/src/ttimer.c

@@ -90,7 +90,7 @@ typedef struct time_wheel_t {
 } time_wheel_t;
 
 int32_t tmrDebugFlag = 131;
-uint32_t taosMaxTmrCtrl = 512;
+uint32_t tsMaxTmrCtrl = 512;
 
 static pthread_once_t  tmrModuleInit = PTHREAD_ONCE_INIT;
 static pthread_mutex_t tmrCtrlMutex;
@@ -385,8 +385,8 @@ static void taosTimerLoopFunc(int signo) {
 
         timer = next;
       }
-      pthread_mutex_unlock(&wheel->mutex);
       wheel->nextScanAt += wheel->resolution;
+      pthread_mutex_unlock(&wheel->mutex);
     }
 
     addToExpired(expired);
@@ -492,17 +492,17 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle,
 }
 
 static void taosTmrModuleInit(void) {
-  tmrCtrls = malloc(sizeof(tmr_ctrl_t) * taosMaxTmrCtrl);
+  tmrCtrls = malloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl);
   if (tmrCtrls == NULL) {
     tmrError("failed to allocate memory for timer controllers.");
     return;
   }
 
-  for (int i = 0; i < taosMaxTmrCtrl - 1; ++i) {
+  for (int i = 0; i < tsMaxTmrCtrl - 1; ++i) {
     tmr_ctrl_t* ctrl = tmrCtrls + i;
     ctrl->next = ctrl + 1;
   }
-  (tmrCtrls + taosMaxTmrCtrl - 1)->next = NULL;
+  (tmrCtrls + tsMaxTmrCtrl - 1)->next = NULL;
   unusedTmrCtrl = tmrCtrls;
 
   pthread_mutex_init(&tmrCtrlMutex, NULL);
@@ -514,17 +514,14 @@ static void taosTmrModuleInit(void) {
       tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno));
       return;
     }
-    pthread_mutex_lock(&wheel->mutex);
     wheel->nextScanAt = now + wheel->resolution;
     wheel->index = 0;
     wheel->slots = (tmr_obj_t**)calloc(wheel->size, sizeof(tmr_obj_t*));
     if (wheel->slots == NULL) {
       tmrError("failed to allocate wheel slots");
-      pthread_mutex_unlock(&wheel->mutex);
       return;
     }
     timerMap.size += wheel->size;
-    pthread_mutex_unlock(&wheel->mutex);
   }
 
   timerMap.count = 0;

src/util/src/tutil.c

@@ -55,6 +55,25 @@ uint32_t taosRand(void)
 */
   return rand();
 }
+
+uint32_t trand(void)
+{
+  int fd;
+  int seed;
+  
+  fd = open("/dev/urandom", 0);
+  if (fd < 0) {
+    seed = time(0);
+  } else {
+    int len = read(fd, &seed, sizeof(seed));
+    if (len < 0) {
+      seed = time(0);
+    }  
+    close(fd);
+  }
+
+  return (uint32_t)seed;
+}
 #endif
 
 size_t twcslen(const wchar_t *wcs) {
@@ -295,6 +314,7 @@ int64_t strnatoi(char *num, int32_t len) {
   return ret;
 }
 
+#if 0
 FORCE_INLINE size_t getLen(size_t old, size_t size) {
   if (old == 1) {
     old = 2;
@@ -382,6 +402,7 @@ char *strreplace(const char *str, const char *pattern, const char *rep) {
 
   return dest;
 }
+#endif
 
 char *strbetween(char *string, char *begin, char *end) {
   char *result = NULL;
@@ -772,3 +793,9 @@ void taosRemoveDir(char *rootDir) {
 
   uPrint("dir:%s is removed", rootDir);
 }
+
+int tmkdir(const char *path, mode_t mode) {
+  int code = mkdir(path, 0755);
+  if (code < 0 && errno == EEXIST) code = 0;
+  return code;
+}

src/util/tests/stringTest.cpp

@@ -26,6 +26,7 @@ TEST(testCase, string_dequote_test) {
   EXPECT_EQ(3, lx);
 }
 
+#if 0
 TEST(testCase, string_replace_test) {
   char  t3[] = "abc01abc02abc";
   char* ret = strreplace(t3, "abc", "7");
@@ -87,6 +88,7 @@ TEST(testCase, string_replace_test) {
   EXPECT_STREQ("abcdef", ret);
   free(ret);
 }
+#endif
 
 TEST(testCase, string_tolower_test) {
   char t[1024] = {1};

src/wal/src/walMain.c

@@ -71,31 +71,28 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
   tstrncpy(pWal->path, path, sizeof(pWal->path));
   pthread_mutex_init(&pWal->mutex, NULL);
 
-  if (access(path, F_OK) != 0) {
+  if (tmkdir(path, 0755) != 0) {
-    if (mkdir(path, 0755) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
     pthread_mutex_destroy(&pWal->mutex);
     free(pWal);
     pWal = NULL;
   }
-  }
      
   if (pCfg->keep == 1) return pWal;
 
   if (walHandleExistingFiles(path) == 0) 
     walRenew(pWal);
 
-  if (pWal->fd <0) {
+  if (pWal && pWal->fd <0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     wError("wal:%s, failed to open(%s)", path, strerror(errno));
     pthread_mutex_destroy(&pWal->mutex);
     free(pWal);
     pWal = NULL;
-  } else {
-    wTrace("wal:%s, it is open, level:%d", path, pWal->level);
   } 
 
+  if (pWal) wTrace("wal:%s, it is open, level:%d", path, pWal->level);
   return pWal;
 }
 
@@ -218,10 +215,13 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
   if ( pWal->keep == 0) 
     strcpy(opath+slen, "/old");
 
-  // is there old directory?
-  if (access(opath, F_OK)) return 0; 
-
   DIR *dir = opendir(opath);
+  if (dir == NULL && errno == ENOENT) return 0;
+  if (dir == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return terrno;
+  }
+
   while ((ent = readdir(dir))!= NULL) {
     if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
       index = atol(ent->d_name + plen);
@@ -379,13 +379,11 @@ int walHandleExistingFiles(const char *path) {
       if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
         snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
         snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
-        if (access(opath, F_OK) != 0) {
+        if (tmkdir(opath, 0755) != 0) {
-          if (mkdir(opath, 0755) != 0) {
           wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
           terrno = TAOS_SYSTEM_ERROR(errno);
           break; 
         }
-        }
 
         if (rename(oname, nname) < 0) {
           wError("wal:%s, failed to move to new:%s", oname, nname);
@@ -409,10 +407,14 @@ static int walRemoveWalFiles(const char *path) {
   char   name[TSDB_FILENAME_LEN * 3];
  
   terrno = 0;
-  if (access(path, F_OK) != 0) return 0;
 
   struct dirent *ent;
   DIR   *dir = opendir(path);
+  if (dir == NULL && errno == ENOENT) return 0;
+  if (dir == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return terrno;
+  }  
 
   while ((ent = readdir(dir))!= NULL) {
     if ( strncmp(ent->d_name, walPrefix, plen) == 0) {

tests/examples/c/demo.c

@@ -44,7 +44,9 @@ int main(int argc, char *argv[]) {
 
 
   taos_query(taos, "drop database demo");
-  if (taos_query(taos, "create database demo") != 0) {
+
+  result = taos_query(taos, "create database demo");
+  if (result == NULL) {
     printf("failed to create database, reason:%s\n", taos_errstr(taos));
     exit(1);
   }
@@ -53,7 +55,7 @@ int main(int argc, char *argv[]) {
   taos_query(taos, "use demo");
 
   // create table
-  if (taos_query(taos, "create table m1 (ts timestamp, speed int)") != 0) {
+  if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
     printf("failed to create table, reason:%s\n", taos_errstr(taos));
     exit(1);
   }
@@ -65,9 +67,10 @@ int main(int argc, char *argv[]) {
   // insert 10 records
   int i = 0;
   for (i = 0; i < 10; ++i) {
-    sprintf(qstr, "insert into m1 values (%ld, %d)", 1546300800000 + i * 1000, i * 10);
+    sprintf(qstr, "insert into m1 values (%ld, %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello");
+    printf("qstr: %s\n", qstr);
     if (taos_query(taos, qstr)) {
-      printf("failed to insert row: %i, reason:%s\n", i, taos_errstr(taos));
+      printf("insert row: %i, reason:%s\n", i, taos_errstr(taos));
     }
     //sleep(1);
   }
@@ -83,10 +86,11 @@ int main(int argc, char *argv[]) {
 
   TAOS_ROW    row;
   int         rows = 0;
-  int         num_fields = taos_field_count(taos);
+  int         num_fields = taos_field_count(result);
   TAOS_FIELD *fields = taos_fetch_fields(result);
-  char        temp[256];
+  char        temp[1024];
 
+  printf("num_fields = %d\n", num_fields);
   printf("select * from table, result:\n");
   // fetch the records row by row
   while ((row = taos_fetch_row(result))) {

tests/perftest-scripts/perftest-consistent-inserting-data.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+ulimit -c unlimited
+
+function buildTDengine {
+	cd /root/TDengine
+
+	git remote update
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "repo up-to-date"
+	else
+		echo "repo need to pull"
+		git pull
+
+		LOCAL_COMMIT=`git rev-parse --short @`
+		cd debug
+		rm -rf *
+		cmake ..
+		make > /dev/null
+		make install
+	fi
+}
+
+function restartTaosd {
+	systemctl stop taosd
+	pkill -KILL -x taosd
+	sleep 10
+	
+	rm -rf /var/lib/taos/data/*
+	rm -rf /var/lib/taos/log/*
+	
+	taosd 2>&1 > /dev/null &
+	sleep 10
+}
+
+buildTDengine
+restartTaosd
+cd /root/TDengine/tests/pytest/insert
+python3 writeDBNonStop.py

tests/pytest/alter/alter_table_crash.py

@@ -47,7 +47,6 @@ class TDTestCase:
         tdSql.query("select * from st")
         tdSql.checkRows(1)
 
-
         print("==============Case 2: keep adding columns, restart taosd")
         tdSql.execute(
             "create table dt(ts timestamp, tbcol1 tinyint) tags(tgcol1 tinyint)")
@@ -72,7 +71,7 @@ class TDTestCase:
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
 
-        tdSql.query("select * from st")
+        tdSql.query("select * from dt")
         tdSql.checkRows(0)
 
     def stop(self):

tests/pytest/client/client.py

@@ -0,0 +1,49 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        ret = tdSql.query('select database()')
+        tdSql.checkData(0, 0, "db")
+
+        ret = tdSql.query('select server_version()')
+        tdSql.checkData(0, 0, "2.0.0.0")
+
+        ret = tdSql.query('select client_version()')
+        tdSql.checkData(0, 0, "2.0.0.0")
+
+        ret = tdSql.query('select server_status()')
+        tdSql.checkData(0, 0, 1)
+
+        ret = tdSql.query('select server_status() as result')
+        tdSql.checkData(0, 0, 1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/fulltest.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 ulimit -c unlimited
 
+python3 ./test.py -f client/client.py
 python3 ./test.py -f insert/basic.py
 python3 ./test.py -f insert/int.py
 python3 ./test.py -f insert/float.py
@@ -142,6 +143,7 @@ python3 ./test.py -f query/filterFloatAndDouble.py
 python3 ./test.py -f query/filterOtherTypes.py
 python3 ./test.py -f query/querySort.py
 python3 ./test.py -f query/queryJoin.py
+python3 ./test.py -f query/select_last_crash.py
 
 #stream
 python3 ./test.py -f stream/stream1.py

tests/pytest/insert/writeDBNonStop.py

@@ -0,0 +1,82 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+from datetime import datetime
+import csv
+
+
+class DBWriteNonStop:
+    def __init__(self):
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taos"
+
+    def connectDB(self):
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)
+        self.cursor = self.conn.cursor()
+
+    def createTable(self):
+        self.cursor.execute("drop database if exists dbwrite")
+        self.cursor.execute("create database dbwrite")
+        self.cursor.execute("use dbwrite")
+        self.cursor.execute(
+            "create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
+
+    def writeDataToCSVFile(self, data, duration):
+        csvFile = open('/root/csvFile.csv', 'a', newline='')
+        writer = csv.writer(csvFile)
+        writer.writerow([data[0][0], data[0][1], data[0][2],
+                         data[0][3], data[0][4], data[0][5], duration])
+        csvFile.close()
+
+    def insertData(self):
+        print("===============inserting data===============")
+        i = 1
+        startTime = datetime.now()
+        while True:
+            self.cursor.execute(
+                "insert into st1 using st tags('dev_001') values(now, 'taosdata%d', %d)" %
+                (i %
+                 10000,
+                 i %
+                 100000))
+            i += 1
+            i = i % 32000000
+            endTime = datetime.now()
+            if (endTime - startTime).seconds >= 3600 * 1:
+                startTime = endTime
+                start = datetime.now()
+                self.cursor.execute(
+                    "select first(ts), last(ts), min(speed), max(speed), avg(speed), count(*) from st")
+                data = self.cursor.fetchall()
+                end = datetime.now()                
+                self.writeDataToCSVFile(data, (end - start).seconds)
+            time.sleep(.001)
+
+    def closeConn(self):
+        self.cursor.close()
+        self.conn.close()
+
+test = DBWriteNonStop()
+test.connectDB()
+test.createTable()
+test.insertData()
+test.closeConn()

tests/pytest/query/select_last_crash.py

@@ -0,0 +1,54 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        
+        self.rowNum = 5000
+        self.ts = 1537146000000
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, value nchar(50), speed int) tags(dev nchar(50))")
+        tdSql.execute(
+            "create table t1 using st tags('dev_001')")
+
+        for i in range(self.rowNum):
+            tdSql.execute("insert into t1 values(%d, 'taosdata%d', %d)" % (self.ts + i, i + 1, i + 1))
+
+        tdSql.query("select last(*) from st")
+        tdSql.checkRows(1)
+
+        print(
+            "======= Verify filter for %s type finished =========" %
+            curType)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/random-test/random-test-multi-threading-3.py

@@ -24,19 +24,23 @@ last_tb = ""
 last_stb = ""
 written = 0
 last_timestamp = 0
+colAdded = False
+killed = False
 
 
 class Test (Thread):
-    def __init__(self, threadId, name, events):
+    def __init__(self, threadId, name, events, q):
         Thread.__init__(self)
         self.threadId = threadId
         self.name = name
         self.dataEvent, self.dbEvent, self.queryEvent = events
+        self.q = q
 
     def create_table(self):
         tdLog.info("create_table")
         global last_tb
         global written
+        global killed
 
         current_tb = "tb%d" % int(round(time.time() * 1000))
 
@@ -51,8 +55,14 @@ class Test (Thread):
                     current_tb)
                 last_tb = current_tb
                 written = 0
+                killed = False
             except Exception as e:
-                tdLog.info(repr(e))
+                tdLog.info("killed: %d error: %s" % (killed, e.args[0]))
+                if killed and (e.args[0] == 'network unavailable'):
+                    tdLog.info("database killed, expect failed")
+                    return 0
+                return -1
+        return 0
 
     def insert_data(self):
         tdLog.info("insert_data")
@@ -74,22 +84,33 @@ class Test (Thread):
             for j in range(0, insertRows):
                 if (last_tb == ""):
                     tdLog.info("no table, return")
-                    return
+                    return 0
+                try:
                     tdSql.execute(
                         'insert into %s values (%d + %da, %d, "test")' %
                         (last_tb, start_time, last_timestamp, last_timestamp))
                     written = written + 1
                     last_timestamp = last_timestamp + 1
+                except Exception as e:
+                    if killed:
+                        tdLog.info(
+                            "database killed, expect failed %s" %
+                            e.args[0])
+                        return 0
+                    tdLog.info(repr(e))
+                    return -1
+        return 0
 
     def query_data(self):
         tdLog.info("query_data")
         global last_tb
-        global written
+        global killed
 
-        if (written > 0):
+        if not killed and last_tb != "":
             tdLog.info("query data from table")
             tdSql.query("select * from %s" % last_tb)
             tdSql.checkRows(written)
+        return 0
 
     def create_stable(self):
         tdLog.info("create_stable")
@@ -123,6 +144,7 @@ class Test (Thread):
                 (last_tb, start_time, last_timestamp))
             written = written + 1
             last_timestamp = last_timestamp + 1
+        return 0
 
     def drop_stable(self):
         tdLog.info("drop_stable")
@@ -139,22 +161,57 @@ class Test (Thread):
             last_stb = ""
             last_tb = ""
             written = 0
+        return 0
+
+    def alter_table_to_add_col(self):
+        tdLog.info("alter_table_to_add_col")
+        global last_stb
+        global colAdded
+
+        if last_stb != "" and colAdded == False:
+            tdSql.execute(
+                "alter table %s add column col binary(20)" %
+                last_stb)
+            colAdded = True
+        return 0
+
+    def alter_table_to_drop_col(self):
+        tdLog.info("alter_table_to_drop_col")
+        global last_stb
+        global colAdded
+
+        if last_stb != "" and colAdded:
+            tdSql.execute("alter table %s drop column col" % last_stb)
+            colAdded = False
+        return 0
 
     def restart_database(self):
         tdLog.info("restart_database")
         global last_tb
         global written
+        global killed
 
         tdDnodes.stop(1)
+        killed = True
         tdDnodes.start(1)
+        tdLog.sleep(10)
+        killed = False
+        return 0
 
     def force_restart_database(self):
         tdLog.info("force_restart_database")
         global last_tb
         global written
+        global killed
 
         tdDnodes.forcestop(1)
+        last_tb = ""
+        written = 0
+        killed = True
         tdDnodes.start(1)
+#        tdLog.sleep(10)
+        killed = False
+        return 0
 
     def drop_table(self):
         tdLog.info("drop_table")
@@ -167,6 +224,7 @@ class Test (Thread):
                 tdSql.execute("drop table %s" % last_tb)
                 last_tb = ""
                 written = 0
+        return 0
 
     def query_data_from_stable(self):
         tdLog.info("query_data_from_stable")
@@ -178,6 +236,7 @@ class Test (Thread):
         else:
             tdLog.info("will query data from super table")
             tdSql.execute('select * from %s' % last_stb)
+        return 0
 
     def reset_query_cache(self):
         tdLog.info("reset_query_cache")
@@ -187,39 +246,45 @@ class Test (Thread):
         tdLog.info("reset query cache")
         tdSql.execute("reset query cache")
         tdLog.sleep(1)
+        return 0
 
     def reset_database(self):
         tdLog.info("reset_database")
         global last_tb
         global last_stb
         global written
+        global killed
 
         tdDnodes.forcestop(1)
+        killed = True
         tdDnodes.deploy(1)
         tdDnodes.start(1)
         tdSql.prepare()
-        last_tb = ""
+        killed = False
-        last_stb = ""
+        return 0
-        written = 0
 
     def delete_datafiles(self):
         tdLog.info("delete_data_files")
         global last_tb
         global last_stb
         global written
+        global killed
 
         dnodesDir = tdDnodes.getDnodesRootDir()
         tdDnodes.forcestop(1)
+        killed = True
         dataDir = dnodesDir + '/dnode1/data/*'
         deleteCmd = 'rm -rf %s' % dataDir
         os.system(deleteCmd)
-
-        tdDnodes.start(1)
-        tdSql.prepare()
         last_tb = ""
         last_stb = ""
         written = 0
 
+        tdDnodes.start(1)
+        tdSql.prepare()
+        killed = False
+        return 0
+
     def run(self):
         dataOp = {
             1: self.insert_data,
@@ -235,6 +300,8 @@ class Test (Thread):
             7: self.reset_database,
             8: self.delete_datafiles,
             9: self.drop_stable,
+            10: self.alter_table_to_add_col,
+            11: self.alter_table_to_drop_col,
         }
 
         queryOp = {
@@ -247,16 +314,28 @@ class Test (Thread):
                 self.dataEvent.wait()
                 tdLog.notice("first thread")
                 randDataOp = random.randint(1, 1)
-                dataOp.get(randDataOp, lambda: "ERROR")()
+                ret1 = dataOp.get(randDataOp, lambda: "ERROR")()
+
+                if ret1 == -1:
+                    self.q.put(-1)
+                    tdLog.exit("first thread failed")
+                else:
+                    self.q.put(1)
+
+                if (self.q.get() != -2):
                     self.dataEvent.clear()
                     self.queryEvent.clear()
                     self.dbEvent.set()
+                else:
+                    self.q.put(-1)
+                    tdLog.exit("second thread failed, first thread exit too")
+
 
         elif (self.threadId == 2):
             while True:
                 self.dbEvent.wait()
                 tdLog.notice("second thread")
-                randDbOp = random.randint(1, 9)
+                randDbOp = random.randint(1, 11)
                 dbOp.get(randDbOp, lambda: "ERROR")()
                 self.dbEvent.clear()
                 self.dataEvent.clear()
@@ -298,6 +377,10 @@ class TDTestCase:
         test2.join()
         test3.join()
 
+        while not q.empty():
+            if (q.get() != 0):
+                tdLog.exit("failed to end of test")
+
         tdLog.info("end of test")
 
     def stop(self):

tests/pytest/random-test/random-test-multi-threading.py

@@ -14,6 +14,7 @@
 import sys
 import random
 import threading
+import queue
 
 from util.log import *
 from util.cases import *
@@ -24,13 +25,16 @@ last_tb = ""
 last_stb = ""
 written = 0
 last_timestamp = 0
+colAdded = False
+killed = False
 
 
 class Test (threading.Thread):
-    def __init__(self, threadId, name):
+    def __init__(self, threadId, name, q):
         threading.Thread.__init__(self)
         self.threadId = threadId
         self.name = name
+        self.q = q
 
         self.threadLock = threading.Lock()
 
@@ -38,11 +42,12 @@ class Test (threading.Thread):
         tdLog.info("create_table")
         global last_tb
         global written
+        global killed
 
         current_tb = "tb%d" % int(round(time.time() * 1000))
 
         if (current_tb == last_tb):
-            return
+            return 0
         else:
             tdLog.info("will create table %s" % current_tb)
 
@@ -52,8 +57,14 @@ class Test (threading.Thread):
                     current_tb)
                 last_tb = current_tb
                 written = 0
+                killed = False
             except Exception as e:
-                tdLog.info(repr(e))
+                tdLog.info("killed: %d error: %s" % (killed, e.args[0]))
+                if killed and (e.args[0] == 'network unavailable'):
+                    tdLog.info("database killed, expect failed")
+                    return 0
+                return -1
+        return 0
 
     def insert_data(self):
         tdLog.info("insert_data")
@@ -75,22 +86,34 @@ class Test (threading.Thread):
             for j in range(0, insertRows):
                 if (last_tb == ""):
                     tdLog.info("no table, return")
-                    return
+                    return 0
+
+                try:
                     tdSql.execute(
                         'insert into %s values (%d + %da, %d, "test")' %
                         (last_tb, start_time, last_timestamp, last_timestamp))
                     written = written + 1
                     last_timestamp = last_timestamp + 1
+                except Exception as e:
+                    if killed:
+                        tdLog.info(
+                            "database killed, expect failed %s" %
+                            e.args[0])
+                        return 0
+                    tdLog.info(repr(e))
+                    return -1
+        return 0
 
     def query_data(self):
         tdLog.info("query_data")
         global last_tb
-        global written
+        global killed
 
-        if (written > 0):
+        if not killed and last_tb != "":
             tdLog.info("query data from table")
             tdSql.query("select * from %s" % last_tb)
             tdSql.checkRows(written)
+        return 0
 
     def create_stable(self):
         tdLog.info("create_stable")
@@ -101,9 +124,7 @@ class Test (threading.Thread):
 
         current_stb = "stb%d" % int(round(time.time() * 1000))
 
-        if (current_stb == last_stb):
+        if (current_stb != last_stb):
-            return
-        else:
             tdLog.info("will create stable %s" % current_stb)
             tdLog.info(
                 'create table %s(ts timestamp, c1 int, c2 nchar(10)) tags (t1 int, t2 nchar(10))' %
@@ -131,6 +152,8 @@ class Test (threading.Thread):
             written = written + 1
             last_timestamp = last_timestamp + 1
 
+        return 0
+
     def drop_stable(self):
         tdLog.info("drop_stable")
         global last_stb
@@ -139,31 +162,63 @@ class Test (threading.Thread):
 
         if (last_stb == ""):
             tdLog.info("no super table")
-            return
         else:
-            tdLog.info("will drop last super table")
+            tdLog.info("will drop last super table %s" % last_stb)
             tdSql.execute('drop table %s' % last_stb)
             last_stb = ""
             last_tb = ""
             written = 0
+        return 0
+
+    def alter_table_to_add_col(self):
+        tdLog.info("alter_table_to_add_col")
+        global last_stb
+        global colAdded
+
+        if last_stb != "" and not colAdded:
+            tdSql.execute(
+                "alter table %s add column col binary(20)" %
+                last_stb)
+            colAdded = True
+        return 0
+
+    def alter_table_to_drop_col(self):
+        tdLog.info("alter_table_to_drop_col")
+        global last_stb
+        global colAdded
+
+        if last_stb != "" and colAdded:
+            tdSql.execute("alter table %s drop column col" % last_stb)
+            colAdded = False
+        return 0
 
     def restart_database(self):
         tdLog.info("restart_database")
         global last_tb
         global written
+        global killed
 
         tdDnodes.stop(1)
+        killed = True
         tdDnodes.start(1)
-#        tdLog.sleep(5)
+        tdLog.sleep(10)
+        killed = False
+        return 0
 
     def force_restart_database(self):
         tdLog.info("force_restart_database")
         global last_tb
         global written
+        global killed
 
         tdDnodes.forcestop(1)
+        last_tb = ""
+        written = 0
+        killed = True
         tdDnodes.start(1)
 #        tdLog.sleep(10)
+        killed = False
+        return 0
 
     def drop_table(self):
         tdLog.info("drop_table")
@@ -176,6 +231,7 @@ class Test (threading.Thread):
                 tdSql.execute("drop table %s" % last_tb)
                 last_tb = ""
                 written = 0
+        return 0
 
     def query_data_from_stable(self):
         tdLog.info("query_data_from_stable")
@@ -183,10 +239,10 @@ class Test (threading.Thread):
 
         if (last_stb == ""):
             tdLog.info("no super table")
-            return
         else:
             tdLog.info("will query data from super table")
             tdSql.execute('select * from %s' % last_stb)
+        return 0
 
     def reset_query_cache(self):
         tdLog.info("reset_query_cache")
@@ -196,39 +252,45 @@ class Test (threading.Thread):
         tdLog.info("reset query cache")
         tdSql.execute("reset query cache")
 #        tdLog.sleep(1)
+        return 0
 
     def reset_database(self):
         tdLog.info("reset_database")
         global last_tb
         global last_stb
         global written
+        global killed
 
         tdDnodes.forcestop(1)
+        killed = True
         tdDnodes.deploy(1)
         tdDnodes.start(1)
         tdSql.prepare()
-        last_tb = ""
+        killed = False
-        last_stb = ""
+        return 0
-        written = 0
 
     def delete_datafiles(self):
         tdLog.info("delete_data_files")
         global last_tb
         global last_stb
         global written
+        global killed
 
         dnodesDir = tdDnodes.getDnodesRootDir()
         tdDnodes.forcestop(1)
+        killed = True
         dataDir = dnodesDir + '/dnode1/data/*'
         deleteCmd = 'rm -rf %s' % dataDir
         os.system(deleteCmd)
-
-        tdDnodes.start(1)
-        tdSql.prepare()
         last_tb = ""
         last_stb = ""
         written = 0
 
+        tdDnodes.start(1)
+        tdSql.prepare()
+        killed = False
+        return 0
+
     def run(self):
         dataOp = {
             1: self.insert_data,
@@ -246,6 +308,8 @@ class Test (threading.Thread):
             7: self.reset_database,
             8: self.delete_datafiles,
             9: self.drop_stable,
+            10: self.alter_table_to_add_col,
+            11: self.alter_table_to_drop_col,
         }
 
         if (self.threadId == 1):
@@ -253,16 +317,38 @@ class Test (threading.Thread):
                 self.threadLock.acquire()
                 tdLog.notice("first thread")
                 randDataOp = random.randint(1, 3)
-                dataOp.get(randDataOp, lambda: "ERROR")()
+                ret1 = dataOp.get(randDataOp, lambda: "ERROR")()
+
+                if ret1 == -1:
+                    self.q.put(-1)
+                    tdLog.exit("first thread failed")
+                else:
+                    self.q.put(1)
+
+                if (self.q.get() != -2):
                     self.threadLock.release()
+                else:
+                    self.q.put(-1)
+                    tdLog.exit("second thread failed, first thread exit too")
 
         elif (self.threadId == 2):
             while True:
-                tdLog.notice("second thread")
                 self.threadLock.acquire()
-                randDbOp = random.randint(1, 9)
+                tdLog.notice("second thread")
-                dbOp.get(randDbOp, lambda: "ERROR")()
+                randDbOp = random.randint(1, 11)
+                ret2 = dbOp.get(randDbOp, lambda: "ERROR")()
+
+                if ret2 == -1:
+                    self.q.put(-2)
+                    tdLog.exit("second thread failed")
+                else:
+                    self.q.put(2)
+
+                if (self.q.get() != -1):
                     self.threadLock.release()
+                else:
+                    self.q.put(-2)
+                    tdLog.exit("first thread failed, second exit too")
 
 
 class TDTestCase:
@@ -273,14 +359,19 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        test1 = Test(1, "data operation")
+        q = queue.Queue()
-        test2 = Test(2, "db operation")
+        test1 = Test(1, "data operation", q)
+        test2 = Test(2, "db operation", q)
 
         test1.start()
         test2.start()
         test1.join()
         test2.join()
 
+        while not q.empty():
+            if (q.get() != 0):
+                tdLog.exit("failed to end of test")
+
         tdLog.info("end of test")
 
     def stop(self):