Merge branch 'develop' into feature/tagschema

.gitignore

@@ -41,3 +41,27 @@ pysim/
 
 # Doxygen Generated files
 html/
+/.vs
+/CMakeFiles/3.10.2
+/CMakeCache.txt
+/Makefile
+/*.cmake
+/deps
+/src/cq/test/CMakeFiles/cqtest.dir/*.cmake
+*.cmake
+/src/cq/test/CMakeFiles/cqtest.dir/*.make
+*.make
+link.txt
+*.internal
+*.includecache
+*.marks
+Makefile
+CMakeError.log
+*.log
+/CMakeFiles/CMakeRuleHashes.txt
+/CMakeFiles/Makefile2
+/CMakeFiles/TargetDirectories.txt
+/CMakeFiles/cmake.check_cache
+/out/isenseconfig/WSL-Clang-Debug
+/out/isenseconfig/WSL-GCC-Debug
+/test/cfg

CMakeSettings.json

@@ -0,0 +1,25 @@
+{
+  "configurations": [
+    {
+      "name": "WSL-GCC-Debug",
+      "generator": "Unix Makefiles",
+      "configurationType": "Debug",
+      "buildRoot": "${projectDir}\\build\\",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeExecutable": "/usr/bin/cmake",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "linux_x64" ],
+      "wslPath": "${defaultWSLPath}",
+      "addressSanitizerRuntimeFlags": "detect_leaks=0",
+      "variables": [
+        {
+          "name": "CMAKE_INSTALL_PREFIX",
+          "value": "/mnt/d/TDengine/TDengine/build",
+          "type": "PATH"
+        }
+      ]
+    }
+  ]
+}

src/client/inc/tscLog.h

@@ -30,7 +30,7 @@ extern int32_t cDebugFlag;
   }
 #define tscWarn(...)                                \
   if (cDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  TSC ", cDebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN TSC ", cDebugFlag, __VA_ARGS__); \
   }
 #define tscTrace(...)                         \
   if (cDebugFlag & DEBUG_TRACE) {             \

src/client/inc/tsclient.h

@@ -53,11 +53,7 @@ typedef struct STableComInfo {
 } STableComInfo;
 
 typedef struct STableMeta {
-  // super table if it is created according to super table, otherwise, tableInfo is used
+  STableComInfo tableInfo;
-  union {
-    struct STableMeta *pSTable;
-    STableComInfo      tableInfo;
-  };
   uint8_t       tableType;
   int16_t       sversion;
   SCMVgroupInfo vgroupInfo;
@@ -214,7 +210,7 @@ typedef struct SQueryInfo {
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
   struct STSBuf *  tsBuf;
-  int64_t *        defaultVal;   // default value for interpolation
+  int64_t *        fillVal;      // default value for interpolation
   char *           msg;          // pointer to the pCmd->payload to keep error message temporarily
   int64_t          clauseLimit;  // limit for current sub clause
 
@@ -226,11 +222,8 @@ typedef struct {
   int     command;
   uint8_t msgType;
 
-  union {
+  bool   autoCreated;        // if the table is missing, on-the-fly create it. during getmeterMeta
-    bool   existsCheck;     // check if the table exists or not
+  int8_t dataSourceType;     // load data from file or not
-    bool   autoCreated;     // if the table is missing, on-the-fly create it. during getmeterMeta
-    int8_t dataSourceType;  // load data from file or not
-  };
 
   union {
     int32_t count;

src/client/src/tscAsync.c

@@ -48,7 +48,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
   pSql->param     = param;
   pSql->pTscObj   = pObj;
   pSql->maxRetry  = TSDB_MAX_REPLICA_NUM;
-  pSql->fp = fp;
+  pSql->fp        = fp;
   
   if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     tscError("failed to malloc payload");

src/client/src/tscFunctionImpl.c

@@ -390,16 +390,16 @@ static void function_finalizer(SQLFunctionCtx *pCtx) {
   
   if (pResInfo->hasResult != DATA_SET_FLAG) {
     tscTrace("no result generated, result is set to NULL");
-    setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+    if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+    } else {
+      setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+    }
   }
   
   doFinalizer(pCtx);
 }
 
-static bool usePreVal(SQLFunctionCtx *pCtx) {
-  return pCtx->preAggVals.isSet && pCtx->size == pCtx->preAggVals.size;
-}
-
 /*
  * count function does need the finalize, if data is missing, the default value, which is 0, is used
  * count function does not use the pCtx->interResBuf to keep the intermediate buffer
@@ -412,7 +412,7 @@ static void count_function(SQLFunctionCtx *pCtx) {
    * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true;
    * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false;
    */
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull;
   } else {
     if (pCtx->hasNull) {
@@ -537,7 +537,7 @@ static void do_sum(SQLFunctionCtx *pCtx) {
   int32_t notNullElems = 0;
   
   // Only the pre-computing information loaded and actual data does not loaded
-  if (pCtx->preAggVals.isSet && pCtx->preAggVals.size == pCtx->size) {
+  if (pCtx->preAggVals.isSet) {
     notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull);
     
@@ -768,7 +768,7 @@ static void avg_function(SQLFunctionCtx *pCtx) {
   SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
   double *  pVal = &pAvgInfo->sum;
   
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     // Pre-aggregation
     notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(notNullElems >= 0);
@@ -932,7 +932,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) {
 
 static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) {
   // data in current data block are qualified to the query
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(*notNullElems >= 0);
     
@@ -947,18 +947,21 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
       index = pCtx->preAggVals.statis.maxIndex;
     }
     
-    /**
+    TSKEY key = TSKEY_INITIAL_VAL;
-     * NOTE: work around the bug caused by invalid pre-calculated function.
+    if (pCtx->ptsList != NULL) {
-     * Here the selectivity + ts will not return correct value.
+      /**
-     *
+       * NOTE: work around the bug caused by invalid pre-calculated function.
-     * The following codes of 3 lines will be removed later.
+       * Here the selectivity + ts will not return correct value.
-     */
+       *
-    if (index < 0 || index >= pCtx->size + pCtx->startOffset) {
+       * The following codes of 3 lines will be removed later.
-      index = 0;
+       */
+      if (index < 0 || index >= pCtx->size + pCtx->startOffset) {
+        index = 0;
+      }
+      
+      key = pCtx->ptsList[index];
     }
     
-    TSKEY key = pCtx->ptsList[index];
-    
     if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
       int64_t val = GET_INT64_VAL(tval);
       if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
@@ -1865,12 +1868,22 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) {
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
   if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
     if (pResInfo->hasResult != DATA_SET_FLAG) {
-      setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+      if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+        setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+      } else {
+        setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+      }
+      
       return;
     }
   } else {
     if (pResInfo->hasResult != DATA_SET_FLAG) {
-      setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+      if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+        setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+      } else {
+        setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+      }
+      
       return;
     }
   }
@@ -2886,7 +2899,12 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
   SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
   
   if (pInfo->num == 0) {
-    setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+    if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+    } else {
+      setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+    }
+    
     return;
   }
   
@@ -2913,10 +2931,6 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 static void date_col_output_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->scanFlag == REVERSE_SCAN) {
-    return;
-  }
-  
   SET_VAL(pCtx, pCtx->size, 1);
   *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp;
 }
@@ -3081,7 +3095,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;  // direct previous may be null
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3113,7 +3127,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3144,7 +3158,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].dKey;
           *pTimestamp = pCtx->ptsList[i];
           pOutput += 1;
           pTimestamp += 1;
@@ -3175,7 +3189,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].dKey;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3207,7 +3221,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3239,7 +3253,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3420,7 +3434,7 @@ static void spread_function(SQLFunctionCtx *pCtx) {
   
   // todo : opt with pre-calculated result
   // column missing cause the hasNull to be true
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     
     // all data are null in current data block, ignore current data block
@@ -3446,14 +3460,8 @@ static void spread_function(SQLFunctionCtx *pCtx) {
         pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max));
       }
     }
-  } else {
+    
-//    if (pInfo->min > pCtx->param[1].dKey) {
+    goto _spread_over;
-//      pInfo->min = pCtx->param[1].dKey;
-//    }
-//
-//    if (pInfo->max < pCtx->param[2].dKey) {
-//      pInfo->max = pCtx->param[2].dKey;
-//    }
   }
   
   void *pData = GET_INPUT_CHAR(pCtx);
@@ -3873,7 +3881,11 @@ static void interp_function(SQLFunctionCtx *pCtx) {
       *(TSKEY *)pCtx->aOutputBuf = pInfoDetail->ts;
     } else {
       if (pInfoDetail->type == TSDB_FILL_NULL) {
-        setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+        if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+          setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+        } else {
+          setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+        }
       } else if (pInfoDetail->type == TSDB_FILL_SET_VALUE) {
         tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType);
       } else if (pInfoDetail->type == TSDB_FILL_PREV) {
@@ -3925,7 +3937,11 @@ static void interp_function(SQLFunctionCtx *pCtx) {
           }
           
         } else {
-          setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
+          if (srcType == TSDB_DATA_TYPE_BINARY || srcType == TSDB_DATA_TYPE_NCHAR) {
+            setVardataNull(pCtx->aOutputBuf, pCtx->inputBytes);
+          } else {
+            setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
+          }
         }
       }
     }

src/client/src/tscLocal.c

@@ -56,7 +56,7 @@ static int32_t getToStringLength(const char *pData, int32_t length, int32_t type
     } break;
     case TSDB_DATA_TYPE_TIMESTAMP:
     case TSDB_DATA_TYPE_BIGINT:
-      len = sprintf(buf, "%" PRId64 "", *(int64_t *)pData);
+      len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
       break;
     case TSDB_DATA_TYPE_BOOL:
       len = MAX_BOOL_TYPE_LENGTH;
@@ -384,13 +384,10 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
 
   // keep the code in local variable in order to avoid invalid read in case of async query
   int32_t code = pSql->res.code;
-
+  if (code == TSDB_CODE_SUCCESS) {
-  if (pSql->fp != NULL) {  // callback function
+    (*pSql->fp)(pSql->param, pSql, 0);
-    if (code == 0) {
+  } else {
-      (*pSql->fp)(pSql->param, pSql, 0);
+    tscQueueAsyncRes(pSql);
-    } else {
-      tscQueueAsyncRes(pSql);
-    }
   }
 
   return code;

src/client/src/tscParseInsert.c

@@ -42,35 +42,42 @@ enum {
 static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
 
 static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
-//  int32_t numType = isValidNumber(pToken);
+  if (pToken->n == 0) {
-//  if (TK_ILLEGAL == numType) {
+    return TK_ILLEGAL;
-//    return numType;
+  }
-//  }
+  
-
   int32_t radix = 10;
-  if (pToken->type == TK_HEX) {
+  
-    radix = 16;
+  int32_t radixList[3] = {16, 8, 2};
-  } else if (pToken->type == TK_OCT) {
+  if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) {
-    radix = 8;
+    radix = radixList[pToken->type - TK_HEX];
-  } else if (pToken->type == TK_BIN) {
-    radix = 2;
   }
 
   errno = 0;
   *value = strtoll(pToken->z, endPtr, radix);
+  
+  // not a valid integer number, return error
+  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+    return TK_ILLEGAL;
+  }
 
   return pToken->type;
 }
 
 static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
-//  int32_t numType = isValidNumber(pToken);
+  if (pToken->n == 0) {
-//  if (TK_ILLEGAL == numType) {
+    return TK_ILLEGAL;
-//    return numType;
+  }
-//  }
+  
-
   errno = 0;
   *value = strtod(pToken->z, endPtr);
-  return pToken->type;
+  
+  // not a valid integer number, return error
+  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+    return TK_ILLEGAL;
+  } else {
+    return pToken->type;
+  }
 }
 
 int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@@ -305,8 +312,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
     case TSDB_DATA_TYPE_BINARY:
       // binary data cannot be null-terminated char string, otherwise the last char of the string is lost
       if (pToken->type == TK_NULL) {
-        varDataSetLen(payload, sizeof(int8_t));
+        setVardataNull(payload, TSDB_DATA_TYPE_BINARY);
-        *(uint8_t*) varDataVal(payload) = TSDB_DATA_BINARY_NULL;
       } else { // too long values will return invalid sql, not be truncated automatically
         if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { //todo refactor
           return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z);
@@ -319,8 +325,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
 
     case TSDB_DATA_TYPE_NCHAR:
       if (pToken->type == TK_NULL) {
-        varDataSetLen(payload, sizeof(int32_t));
+        setVardataNull(payload, TSDB_DATA_TYPE_NCHAR);
-        *(uint32_t*) varDataVal(payload) = TSDB_DATA_NCHAR_NULL;
       } else {
         // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
         size_t output = 0;
@@ -422,9 +427,9 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
       return -1;
     }
 
-    if (((sToken.type != TK_NOW) && (sToken.type != TK_INTEGER) && (sToken.type != TK_STRING) &&
+    int16_t type = sToken.type;
-         (sToken.type != TK_FLOAT) && (sToken.type != TK_BOOL) && (sToken.type != TK_NULL)) ||
+    if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL &&
-        (sToken.n == 0) || (sToken.type == TK_RP)) {
+         type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) {
       tscInvalidSQLErrMsg(error, "invalid data or symbol", sToken.z);
       *code = TSDB_CODE_INVALID_SQL;
       return -1;
@@ -779,7 +784,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
 
     STagData *pTag = (STagData *)pCmd->payload;
     memset(pTag, 0, sizeof(STagData));
-    pCmd->payloadLen = sizeof(STagData);
     
     /*
      * the source super table is moved to the secondary position of the pTableMetaInfo list
@@ -928,6 +932,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       }
     }
 
+    // 3. calculate the actual data size of STagData
+    pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen);
+    for (int32_t t = 0; t < numOfTags; ++t) {
+      pTag->dataLen += pTagSchema[t].bytes;
+      pCmd->payloadLen += pTagSchema[t].bytes;
+    }
+    pTag->dataLen = htonl(pTag->dataLen);
+
     if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
     }
@@ -949,7 +961,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     } else {
       sql = sToken.z;
     }
-    code = tscGetTableMeta(pSql, pTableMetaInfo);
+    code = tscGetMeterMetaEx(pSql, pTableMetaInfo, false);
     
     if (pCmd->curSql == NULL) {
       assert(code == TSDB_CODE_ACTION_IN_PROGRESS);
@@ -1299,8 +1311,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
   SQueryInfo *pQueryInfo = NULL;
   tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
 
-  uint16_t type = (sToken.type == TK_INSERT)? TSDB_QUERY_TYPE_INSERT:TSDB_QUERY_TYPE_IMPORT;
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT);
-  TSDB_QUERY_SET_TYPE(pQueryInfo->type, type);
 
   sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL);
   if (sToken.type != TK_INTO) {

src/client/src/tscSQLParser.c

@@ -13,7 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _XOPEN_SOURCE
+#define _BSD_SOURCE
+#define _XOPEN_SOURCE 500
 #define _DEFAULT_SOURCE
 
 #include "os.h"
@@ -3290,14 +3291,15 @@ static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg
 
 static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SCondExpr* pCondExpr, int32_t* type,
                                      int32_t parentOptr) {
-  const char* msg1 = "meter query cannot use tags filter";
+  const char* msg1 = "table query cannot use tags filter";
   const char* msg2 = "illegal column name";
   const char* msg3 = "only one query time range allowed";
   const char* msg4 = "only one join condition allowed";
   const char* msg5 = "not support ordinary column join";
   const char* msg6 = "only one query condition on tbname allowed";
   const char* msg7 = "only in/like allowed in filter table name";
-
+  const char* msg8 = "wildcard string should be less than 20 characters";
+  
   tSQLExpr* pLeft = (*pExpr)->pLeft;
   tSQLExpr* pRight = (*pExpr)->pRight;
 
@@ -3344,7 +3346,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
     // check for like expression
     if ((*pExpr)->nSQLOptr == TK_LIKE) {
       if (pRight->val.nLen > TSDB_PATTERN_STRING_MAX_LEN) {
-        return TSDB_CODE_INVALID_SQL;
+        return invalidSqlErrMsg(pQueryInfo->msg, msg8);
       }
 
       SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
@@ -3360,6 +3362,10 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
       if (!validTableNameOptr(*pExpr)) {
         return invalidSqlErrMsg(pQueryInfo->msg, msg7);
       }
+  
+      if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg1);
+      }
 
       if (pCondExpr->pTableCond == NULL) {
         pCondExpr->pTableCond = *pExpr;
@@ -3808,9 +3814,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
   const char* msg2 = "invalid filter expression";
 
   int32_t ret = TSDB_CODE_SUCCESS;
-
+  pQueryInfo->window = TSWINDOW_INITIALIZER;
-  pQueryInfo->window.skey = 0;
-  pQueryInfo->window.ekey = INT64_MAX;
 
   // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space
   SStringBuilder sb; memset(&sb, 0, sizeof(sb));
@@ -4012,9 +4016,9 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
   
   size_t size = tscSqlExprNumOfExprs(pQueryInfo);
   
-  if (pQueryInfo->defaultVal == NULL) {
+  if (pQueryInfo->fillVal == NULL) {
-    pQueryInfo->defaultVal = calloc(size, sizeof(int64_t));
+    pQueryInfo->fillVal = calloc(size, sizeof(int64_t));
-    if (pQueryInfo->defaultVal == NULL) {
+    if (pQueryInfo->fillVal == NULL) {
       return TSDB_CODE_CLI_OUT_OF_MEMORY;
     }
   }
@@ -4025,7 +4029,11 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
     pQueryInfo->fillType = TSDB_FILL_NULL;
     for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) {
       TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
-      setNull((char*)&pQueryInfo->defaultVal[i], pFields->type, pFields->bytes);
+      if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) {
+        setVardataNull((char*) &pQueryInfo->fillVal[i], pFields->type);
+      } else {
+        setNull((char*)&pQueryInfo->fillVal[i], pFields->type, pFields->bytes);
+      };
     }
   } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) {
     pQueryInfo->fillType = TSDB_FILL_PREV;
@@ -4058,11 +4066,11 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
       TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
 
       if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) {
-        setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes);
+        setVardataNull((char*) &pQueryInfo->fillVal[i], pFields->type);
         continue;
       }
 
-      int32_t ret = tVariantDump(&pFillToken->a[j].pVar, (char*)&pQueryInfo->defaultVal[i], pFields->type);
+      int32_t ret = tVariantDump(&pFillToken->a[j].pVar, (char*)&pQueryInfo->fillVal[i], pFields->type);
       if (ret != TSDB_CODE_SUCCESS) {
         return invalidSqlErrMsg(pQueryInfo->msg, msg);
       }
@@ -4076,9 +4084,9 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
         TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
 
         if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) {
-          setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes);
+          setVardataNull((char*) &pQueryInfo->fillVal[i], pFields->type);
         } else {
-          tVariantDump(&lastItem->pVar, (char*)&pQueryInfo->defaultVal[i], pFields->type);
+          tVariantDump(&lastItem->pVar, (char*)&pQueryInfo->fillVal[i], pFields->type);
         }
       }
     }
@@ -4416,6 +4424,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg13);
     }
+    pAlterSQL->tagData.dataLen = pTagsSchema->bytes;
 
     // validate the length of binary
     if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) &&
@@ -4539,11 +4548,13 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
     return TSDB_CODE_INVALID_SQL;
   }
 
-  const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[14] = {
+  const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 17;
-      {"resetLog", 8},      {"resetQueryCache", 15}, {"dDebugFlag", 10},       {"rpcDebugFlag", 12},
+  const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = {
-      {"tmrDebugFlag", 12}, {"cDebugFlag", 10},      {"uDebugFlag", 10},       {"mDebugFlag", 10},
+      {"resetLog", 8},       {"resetQueryCache", 15},  {"debugFlag", 9},     {"mDebugFlag", 10},
-      {"sdbDebugFlag", 12}, {"httpDebugFlag", 13},   {"monitorDebugFlag", 16}, {"qDebugflag", 10},
+      {"dDebugFlag", 10},    {"sdbDebugFlag", 12},     {"vDebugFlag", 10},   {"cDebugFlag", 10},
-      {"debugFlag", 9},     {"monitor", 7}};
+      {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10},
+      {"tmrDebugFlag", 12},  {"qDebugflag", 10},       {"sDebugflag", 10},   {"tsdbDebugFlag", 13},
+      {"monitor", 7}};
 
   SSQLToken* pOptionToken = &pOptions->a[1];
 
@@ -4555,8 +4566,8 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
         return TSDB_CODE_SUCCESS;
       }
     }
-  } else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[13].name, pOptionToken->z, pOptionToken->n) == 0) &&
+  } else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].name, pOptionToken->z, pOptionToken->n) == 0) &&
-             (DNODE_DYNAMIC_CFG_OPTIONS[13].len == pOptionToken->n)) {
+             (DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].len == pOptionToken->n)) {
     SSQLToken* pValToken = &pOptions->a[2];
     int32_t    val = strtol(pValToken->z, NULL, 10);
     if (val != 0 && val != 1) {
@@ -4572,7 +4583,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
       return TSDB_CODE_INVALID_SQL;
     }
 
-    for (int32_t i = 2; i < tListLen(DNODE_DYNAMIC_CFG_OPTIONS) - 1; ++i) {
+    for (int32_t i = 2; i < DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1; ++i) {
       const SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i];
 
       if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) {
@@ -5331,13 +5342,6 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
     }
   }
 
-  SColumnIndex ind = {0};
-  SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT,
-      tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
-  
-  const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
-  strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
-
   switch (index) {
     case 0:
       pQueryInfo->command = TSDB_SQL_CURRENT_DB;
@@ -5356,6 +5360,13 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
       return TSDB_CODE_SUCCESS;
     default: { return invalidSqlErrMsg(pQueryInfo->msg, msg3); }
   }
+  
+  SColumnIndex ind = {0};
+  SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT,
+                                      tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
+  
+  const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
+  strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
 }
 
 // can only perform the parameters based on the macro definitation
@@ -5550,11 +5561,11 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
   // too long tag values will return invalid sql, not be truncated automatically
   SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta);
 
-  char* tagVal = pCreateTable->usingInfo.tagdata.data;
+  STagData* pTag = &pCreateTable->usingInfo.tagdata;
+  char* tagVal = pTag->data;
   int32_t ret = TSDB_CODE_SUCCESS;
   
   for (int32_t i = 0; i < pList->nExpr; ++i) {
-    
     if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
       // validate the length of binary
       if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pTagSchema[i].bytes) {
@@ -5593,6 +5604,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
     return ret;
   }
 
+  pTag->dataLen = tagVal - pTag->data;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -5602,7 +5614,8 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
   const char* msg3 = "fill only available for interval query";
   const char* msg4 = "fill option not supported in stream computing";
   const char* msg5 = "sql too long";  // todo ADD support
-
+  const char* msg6 = "from missing in subclause";
+  
   SSqlCmd*    pCmd = &pSql->cmd;
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
   assert(pQueryInfo->numOfTables == 1);
@@ -5617,10 +5630,13 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
   if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
-
+  
   tVariantList* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
-  tVariant*     pVar = &pSrcMeterName->a[0].pVar;
+  if (pSrcMeterName == NULL || pSrcMeterName->nExpr == 0) {
-
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
+  }
+  
+  tVariant* pVar = &pSrcMeterName->a[0].pVar;
   SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING};
   if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
     return invalidSqlErrMsg(pQueryInfo->msg, msg1);

src/client/src/tscSecondaryMerge.c

@@ -145,7 +145,7 @@ static SFillColInfo* createFillColInfo(SQueryInfo* pQueryInfo) {
     pFillCol[i].flag       = pExpr->colInfo.flag;
     pFillCol[i].col.offset = offset;
     pFillCol[i].functionId = pExpr->functionId;
-    pFillCol[i].defaultVal.i = pQueryInfo->defaultVal[i];
+    pFillCol[i].fillVal.i  = pQueryInfo->fillVal[i];
     offset += pExpr->resBytes;
   }
   
@@ -638,7 +638,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1);
+  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->numOfSubs);
   if (*pMemBuffer == NULL) {
     tscError("%p failed to allocate memory", pSql);
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@@ -946,8 +946,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
   }
   
   while (1) {
-    int64_t newRows = -1;
+    int64_t newRows = taosGenerateDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
-    taosGenerateDataBlock(pFillInfo, pResPages, &newRows, pLocalReducer->resColModel->capacity);
 
     if (pQueryInfo->limit.offset < newRows) {
       newRows -= pQueryInfo->limit.offset;

src/client/src/tscServer.c

@@ -571,7 +571,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
 
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
-
     pMsg += sizeof(STableIdInfo);
   } else {
     int32_t index = pTableMetaInfo->vgroupIndex;
@@ -601,8 +600,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     }
   }
   
-  tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
+  tscTrace("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
-      pTableMeta->uid);
+      pTableMeta->sid, pTableMeta->uid);
   
   return pMsg;
 }
@@ -782,8 +781,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   if (pQueryInfo->fillType != TSDB_FILL_NONE) {
     for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-      *((int64_t *)pMsg) = htobe64(pQueryInfo->defaultVal[i]);
+      *((int64_t *)pMsg) = htobe64(pQueryInfo->fillVal[i]);
-      pMsg += sizeof(pQueryInfo->defaultVal[0]);
+      pMsg += sizeof(pQueryInfo->fillVal[0]);
     }
   }
   
@@ -1213,8 +1212,13 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   int8_t type = pInfo->pCreateTableInfo->type;
   if (type == TSQL_CREATE_TABLE_FROM_STABLE) {  // create by using super table, tags value
-    memcpy(pMsg, &pInfo->pCreateTableInfo->usingInfo.tagdata, sizeof(STagData));
+    STagData* pTag = &pInfo->pCreateTableInfo->usingInfo.tagdata;
-    pMsg += sizeof(STagData);
+    *(int32_t*)pMsg = htonl(pTag->dataLen);
+    pMsg += sizeof(int32_t);
+    memcpy(pMsg, pTag->name, sizeof(pTag->name));
+    pMsg += sizeof(pTag->name);
+    memcpy(pMsg, pTag->data, pTag->dataLen);
+    pMsg += pTag->dataLen;
   } else {  // create (super) table
     pSchema = (SSchema *)pCreateTableMsg->schema;
 
@@ -1281,9 +1285,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
   pAlterTableMsg->type = htons(pAlterInfo->type);
 
-  pAlterTableMsg->numOfCols = tscNumOfFields(pQueryInfo);
+  pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo));
-  memcpy(pAlterTableMsg->tagVal, pAlterInfo->tagData.data, TSDB_MAX_TAGS_LEN);
-
   SSchema *pSchema = pAlterTableMsg->schema;
   for (int i = 0; i < pAlterTableMsg->numOfCols; ++i) {
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
@@ -1295,6 +1297,9 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   }
 
   pMsg = (char *)pSchema;
+  pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen);
+  memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen);
+  pMsg += pAlterInfo->tagData.dataLen;
 
   msgLen = pMsg - (char*)pAlterTableMsg;
   pCmd->payloadLen = msgLen;
@@ -1863,6 +1868,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   }
 
   free(pTableMeta);
+  tscTrace("%p recv table meta: %"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
   
   return TSDB_CODE_SUCCESS;
 }

src/client/src/tscStream.c

@@ -165,7 +165,7 @@ static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql) {
   if (timestamp != actualTimestamp) {
     // reset the timestamp of each agg point by using start time of each interval
     *((int64_t *)pRes->data) = actualTimestamp;
-    tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64 "", pSql, pStream, timestamp, actualTimestamp);
+    tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64, pSql, pStream, timestamp, actualTimestamp);
   }
 }
 
@@ -223,7 +223,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
           int16_t     offset = tscFieldInfoGetOffset(pQueryInfo, i);
           TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
 
-          assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->defaultVal[i]), pField->bytes, pField->type);
+          assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->fillVal[i]), pField->bytes, pField->type);
           row[i] = pSql->res.data + offset;
         }
 
@@ -287,10 +287,10 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer)
       return;
     }
   
-    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
+    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
              now + timer, timer, delay, pStream->stime, etime);
   } else {
-    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
+    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
              pStream->stime, timer, delay, pStream->stime - pStream->interval, pStream->stime - 1);
   }
 
@@ -380,7 +380,7 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   if (pQueryInfo->intervalTime < minIntervalTime) {
-    tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64, pSql, pStream,
             pQueryInfo->intervalTime, minIntervalTime);
     pQueryInfo->intervalTime = minIntervalTime;
   }
@@ -397,14 +397,14 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
   if (pQueryInfo->slidingTime == -1) {
     pQueryInfo->slidingTime = pQueryInfo->intervalTime;
   } else if (pQueryInfo->slidingTime < minSlidingTime) {
-    tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream,
         pQueryInfo->slidingTime, minSlidingTime);
 
     pQueryInfo->slidingTime = minSlidingTime;
   }
 
   if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) {
-    tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql, pStream,
             pQueryInfo->slidingTime, pQueryInfo->intervalTime);
 
     pQueryInfo->slidingTime = pQueryInfo->intervalTime;
@@ -433,11 +433,11 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
   } else {             // timewindow based aggregation stream
     if (stime == 0) {  // no data in meter till now
       stime = ((int64_t)taosGetTimestamp(pStream->precision) / pStream->interval) * pStream->interval;
-      tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64 "", pSql, pStream, stime);
+      tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime);
     } else {
       int64_t newStime = (stime / pStream->interval) * pStream->interval;
       if (newStime != stime) {
-        tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64 "", pSql, pStream, stime, newStime);
+        tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime);
         stime = newStime;
       }
     }

src/client/src/tscSub.c

@@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
   if (p == NULL) {
     return dflt;
   }
@@ -74,7 +74,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = ts};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
   if (p != NULL) {
     p->key = ts;
   }
@@ -211,7 +211,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
     SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0};
-    SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
       taosArrayPush(pSub->progress, &target);

src/client/src/tscUtil.c

@@ -281,7 +281,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
   }
 
   pQueryInfo->fillType = TSDB_FILL_NONE;
-  tfree(pQueryInfo->defaultVal);
+  tfree(pQueryInfo->fillVal);
 }
 
 int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
@@ -1616,7 +1616,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
   
   pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf);
 
-  tfree(pQueryInfo->defaultVal);
+  tfree(pQueryInfo->fillVal);
 }
 
 void tscClearSubqueryInfo(SSqlCmd* pCmd) {
@@ -1768,7 +1768,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   pNewQueryInfo->order  = pQueryInfo->order;
   pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit;
   pNewQueryInfo->pTableMetaInfo = NULL;
-  pNewQueryInfo->defaultVal  = NULL;
+  pNewQueryInfo->fillVal  = NULL;
   pNewQueryInfo->numOfTables = 0;
   pNewQueryInfo->tsBuf = NULL;
   
@@ -1780,8 +1780,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond);
 
   if (pQueryInfo->fillType != TSDB_FILL_NONE) {
-    pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
+    pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
-    memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
+    memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
   }
 
   if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) {

src/common/inc/tdataformat.h

@@ -154,8 +154,8 @@ typedef struct SDataCol {
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
-void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints);
+void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints);
+void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
@@ -195,7 +195,7 @@ typedef struct {
   int      maxPoints;  // max number of points
   int      bufSize;
 
-  int      numOfPoints;
+  int      numOfRows;
   int      numOfCols;  // Total number of cols
   int      sversion;   // TODO: set sversion
   void *   buf;
@@ -205,7 +205,7 @@ typedef struct {
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
 #define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
 #define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
-#define dataColsKeyLast(pCols) ((pCols->numOfPoints == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfPoints - 1))
+#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
 
 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);

src/common/inc/tglobal.h

@@ -103,6 +103,7 @@ extern int32_t tsOfflineThreshold;
 extern int32_t tsMgmtEqualVnodeNum;
 
 extern int32_t tsEnableHttpModule;
+extern int32_t tsEnableMqttModule;
 extern int32_t tsEnableMonitorModule;
 
 extern int32_t tsRestRowLimit;
@@ -147,6 +148,7 @@ extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
+extern int32_t mqttDebugFlag;
 extern int32_t monitorDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;

src/common/src/tdataformat.c

@@ -319,29 +319,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
 
 }
 
-void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints) {
+void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
   ASSERT(pCol != NULL && value != NULL);
 
   switch (pCol->type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
       // set offset
-      pCol->dataOff[numOfPoints] = pCol->len;
+      pCol->dataOff[numOfRows] = pCol->len;
       // Copy data
       memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
       // Update the length
       pCol->len += varDataTLen(value);
       break;
     default:
-      ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
+      ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
       memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
       pCol->len += pCol->bytes;
       break;
   }
 }
 
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) {
+void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
-  int pointsLeft = numOfPoints - pointsToPop;
+  int pointsLeft = numOfRows - pointsToPop;
 
   ASSERT(pointsLeft > 0);
 
@@ -353,7 +353,7 @@ void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) {
     memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
     dataColSetOffset(pCol, pointsLeft);
   } else {
-    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
+    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
     pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
     memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
   }
@@ -454,7 +454,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 
   pRet->numOfCols = pDataCols->numOfCols;
   pRet->sversion = pDataCols->sversion;
-  if (keepData) pRet->numOfPoints = pDataCols->numOfPoints;
+  if (keepData) pRet->numOfRows = pDataCols->numOfRows;
 
   for (int i = 0; i < pDataCols->numOfCols; i++) {
     pRet->cols[i].type = pDataCols->cols[i].type;
@@ -484,7 +484,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 }
 
 void tdResetDataCols(SDataCols *pCols) {
-  pCols->numOfPoints = 0;
+  pCols->numOfRows = 0;
   for (int i = 0; i < pCols->maxCols; i++) {
     dataColReset(pCols->cols + i);
   }
@@ -497,14 +497,14 @@ void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) {
     SDataCol *pCol = pCols->cols + i;
     void *    value = tdGetRowDataOfCol(row, pCol->type, pCol->offset);
 
-    dataColAppendVal(pCol, value, pCols->numOfPoints, pCols->maxPoints);
+    dataColAppendVal(pCol, value, pCols->numOfRows, pCols->maxPoints);
   }
-  pCols->numOfPoints++;
+  pCols->numOfRows++;
 }
 
 // Pop pointsToPop points from the SDataCols
 void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
-  int pointsLeft = pCols->numOfPoints - pointsToPop;
+  int pointsLeft = pCols->numOfRows - pointsToPop;
   if (pointsLeft <= 0) {
     tdResetDataCols(pCols);
     return;
@@ -512,14 +512,14 @@ void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
 
   for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
     SDataCol *pCol = pCols->cols + iCol;
-    dataColPopPoints(pCol, pointsToPop, pCols->numOfPoints);
+    dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
   }
-  pCols->numOfPoints = pointsLeft;
+  pCols->numOfRows = pointsLeft;
 }
 
 int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
-  ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints);
+  ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
-  ASSERT(target->numOfPoints + rowsToMerge <= target->maxPoints);
+  ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
   ASSERT(target->numOfCols == source->numOfCols);
 
   SDataCols *pTarget = NULL;
@@ -527,10 +527,10 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
   if (dataColsKeyLast(target) < dataColsKeyFirst(source)) {  // No overlap
     for (int i = 0; i < rowsToMerge; i++) {
       for (int j = 0; j < source->numOfCols; j++) {
-        dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfPoints,
+        dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfRows,
                          target->maxPoints);
       }
-      target->numOfPoints++;
+      target->numOfRows++;
     }
   } else {
     pTarget = tdDupDataCols(target, true);
@@ -538,7 +538,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
 
     int iter1 = 0;
     int iter2 = 0;
-    tdMergeTwoDataCols(target, pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
+    tdMergeTwoDataCols(target, pTarget, &iter1, source, &iter2, pTarget->numOfRows + rowsToMerge);
   }
 
   tdFreeDataCols(pTarget);
@@ -553,33 +553,31 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol
   // TODO: add resolve duplicate key here
   tdResetDataCols(target);
 
-  while (target->numOfPoints < tRows) {
+  while (target->numOfRows < tRows) {
-    if (*iter1 >= src1->numOfPoints && *iter2 >= src2->numOfPoints) break;
+    if (*iter1 >= src1->numOfRows && *iter2 >= src2->numOfRows) break;
 
-    TSKEY key1 = (*iter1 >= src1->numOfPoints) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
+    TSKEY key1 = (*iter1 >= src1->numOfRows) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
-    TSKEY key2 = (*iter2 >= src2->numOfPoints) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+    TSKEY key2 = (*iter2 >= src2->numOfRows) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
 
-    if (key1 < key2) {
+    if (key1 <= key2) {
       for (int i = 0; i < src1->numOfCols; i++) {
         ASSERT(target->cols[i].type == src1->cols[i].type);
-        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
+        dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
                          target->maxPoints);
       }
 
-      target->numOfPoints++;
+      target->numOfRows++;
       (*iter1)++;
-    } else if (key1 > key2) {
+      if (key1 == key2) (*iter2)++;
+    } else {
       for (int i = 0; i < src2->numOfCols; i++) {
         ASSERT(target->cols[i].type == src2->cols[i].type);
-        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
+        dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
                          target->maxPoints);
       }
 
-      target->numOfPoints++;
+      target->numOfRows++;
       (*iter2)++;
-    } else {
-      // TODO: deal with duplicate keys
-      ASSERT(false);
     }
   }
 }

src/common/src/tglobal.c

@@ -107,7 +107,6 @@ int32_t tsReplications  = TSDB_DEFAULT_REPLICA_NUM;
 int16_t tsAffectedRowsMod = 0;
 int32_t tsNumOfMPeers = 3;
 int32_t tsMaxShellConns = 2000;
-int32_t tsMaxTables = 100000;
 
 char    tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
 char    tsDefaultUser[64] = "root";
@@ -121,6 +120,7 @@ int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
 int32_t tsMgmtEqualVnodeNum = 4;
 
 int32_t tsEnableHttpModule = 1;
+int32_t tsEnableMqttModule = 0;   // not finished yet, not started it by default
 int32_t tsEnableMonitorModule = 0;
 
 int32_t tsRestRowLimit = 10240;
@@ -135,13 +135,14 @@ int32_t cDebugFlag = 135;
 int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
+int32_t mqttDebugFlag = 131;
 int32_t monitorDebugFlag = 131;
 int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 135;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
 int32_t sDebugFlag = 135;
-int32_t tsdbDebugFlag = 131;
+int32_t tsdbDebugFlag = 135;
 
 // the maximum number of results for projection query on super table that are returned from
 // one virtual node, to order according to timestamp
@@ -201,6 +202,8 @@ char tsTimezone[64] = {0};
 char tsLocale[TSDB_LOCALE_LEN] = {0};
 char tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
 
+int32_t tsMaxBinaryDisplayWidth = 30;
+
 static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
 
 void taosSetAllDebugFlag() {
@@ -213,6 +216,7 @@ void taosSetAllDebugFlag() {
     jniDebugFlag = debugFlag;
     odbcDebugFlag = debugFlag;
     httpDebugFlag = debugFlag;
+    mqttDebugFlag = debugFlag;
     monitorDebugFlag = debugFlag;
     rpcDebugFlag = debugFlag;
     uDebugFlag = debugFlag;
@@ -891,6 +895,17 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  
+  cfg.option = "mqtt";
+  cfg.ptr = &tsEnableMqttModule;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "monitor";
   cfg.ptr = &tsEnableMonitorModule;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1113,6 +1128,17 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "mqttDebugFlag";
+  cfg.ptr = &mqttDebugFlag;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
+  cfg.minValue = 0;
+  cfg.maxValue = 255;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+
   cfg.option = "monitorDebugFlag";
   cfg.ptr = &monitorDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1203,6 +1229,16 @@ static void doInitGlobalConfig() {
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
+
+  cfg.option = "maxBinaryDisplayWidth";
+  cfg.ptr = &tsMaxBinaryDisplayWidth;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 1;
+  cfg.maxValue = 0x7fffffff;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
 }
 
 void taosInitGlobalCfg() {

src/common/src/tname.c

@@ -29,6 +29,7 @@ void extractTableName(const char* tableId, char* name) {
   size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]);
   
   strncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
+  name[TSDB_TABLE_NAME_LEN] = 0;
 }
 
 char* extractDBName(const char* tableId, char* name) {

src/common/src/ttypes.c

@@ -42,9 +42,6 @@ static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t nu
   
   ASSERT(numOfRow <= INT16_MAX);
   
-  //  int64_t lastKey = 0;
-  //  int8_t  lastVal = TSDB_DATA_TINYINT_NULL;
-  
   for (int32_t i = 0; i < numOfRow; ++i) {
     if (isNull((char *)&data[i], TSDB_DATA_TYPE_TINYINT)) {
       (*numOfNull) += 1;
@@ -213,15 +210,6 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num
       fmax = fv;
       *maxIndex = i;
     }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
   }
   
   double csum = 0;
@@ -232,9 +220,9 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num
   SET_DOUBLE_VAL_ALIGN(max, &fmax);
   SET_DOUBLE_VAL_ALIGN(min, &fmin);
 #else
-  *sum = csum;
+  *(double*)sum = csum;
-  *max = fmax;
+  *(double*)max = fmax;
-  *min = fmin;
+  *(double*)min = fmin;
 #endif
 }
 
@@ -267,15 +255,6 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num
       dmax = dv;
       *maxIndex = i;
     }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
   }
   
   double csum = 0;
@@ -285,27 +264,67 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num
 
 #ifdef _TD_ARM_32_
   SET_DOUBLE_VAL_ALIGN(sum, &csum);
-    SET_DOUBLE_VAL_ALIGN(max, &dmax);
+  SET_DOUBLE_VAL_ALIGN(max, &dmax);
-    SET_DOUBLE_VAL_ALIGN(min, &dmin);
+  SET_DOUBLE_VAL_ALIGN(min, &dmin);
 #else
-  *sum = csum;
+  *(double*) sum = csum;
-  *max = dmax;
+  *(double*) max = dmax;
-  *min = dmin;
+  *(double*) min = dmin;
 #endif
 }
 
+static void getStatics_bin(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                         int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  const char* data = pData;
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) varDataVal(data), TSDB_DATA_TYPE_BINARY)) {
+      (*numOfNull) += 1;
+    }
+    
+    data += varDataLen(data);
+  }
+  
+  *sum = 0;
+  *max = 0;
+  *min = 0;
+  *minIndex = 0;
+  *maxIndex = 0;
+}
+
+static void getStatics_nchr(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  const char* data = pData;
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) varDataVal(data), TSDB_DATA_TYPE_NCHAR)) {
+      (*numOfNull) += 1;
+    }
+    
+    data += varDataLen(data);
+  }
+  
+  *sum = 0;
+  *max = 0;
+  *min = 0;
+  *minIndex = 0;
+  *maxIndex = 0;
+}
+
 tDataTypeDescriptor tDataTypeDesc[11] = {
   {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL,                  NULL},
-  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool,      NULL},
+  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool,      getStatics_i8},
   {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
   {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
   {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt,       getStatics_i32},
   {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT",    tsCompressBigint,    tsDecompressBigint,    getStatics_i64},
   {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat,     getStatics_f},
   {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble,    getStatics_d},
-  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString,    NULL},
+  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString,    getStatics_bin},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, NULL},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
-  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString,    NULL},
+  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString,    getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {
@@ -362,6 +381,18 @@ bool isNull(const char *val, int32_t type) {
   };
 }
 
+void setVardataNull(char* val, int32_t type) {
+  if (type == TSDB_DATA_TYPE_BINARY) {
+    varDataSetLen(val, sizeof(int8_t));
+    *(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL;
+  } else if (type == TSDB_DATA_TYPE_NCHAR) {
+    varDataSetLen(val, sizeof(int32_t));
+    *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL;
+  } else {
+    assert(0);
+  }
+}
+
 void setNull(char *val, int32_t type, int32_t bytes) { setNullN(val, type, bytes, 1); }
 
 void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
@@ -464,7 +495,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
       break;
     };
     case TSDB_DATA_TYPE_NCHAR: {
-      wcsncpy((wchar_t*)val, (wchar_t*)src, len / TSDB_NCHAR_SIZE);
+      varDataCopy(val, src);
       break;
     };
     default: {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -188,7 +188,7 @@ public class TSDBDriver implements java.sql.Driver {
 	}
 
 	public boolean acceptsURL(String url) throws SQLException {
-		return true;
+		return StringUtils.isNotBlank(url) && url.startsWith(URL_PREFIX);
 	}
 
 	public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
@@ -355,4 +355,4 @@ public class TSDBDriver implements java.sql.Driver {
 	public String database(Properties props) {
 		return props.getProperty(PROPERTY_KEY_DBNAME);
 	}
-}
+}

src/connector/python/linux/python2/taos/cursor.py

@@ -34,6 +34,7 @@ class TDengineCursor(object):
         self._block_rows = -1
         self._block_iter = 0
         self._affected_rows = 0
+        self._logfile = ""
 
         if connection is not None:
             self._connection = connection
@@ -83,6 +84,9 @@ class TDengineCursor(object):
         """
         pass
 
+    def log(self, logfile):
+        self._logfile = logfile
+
     def close(self):
         """Close the cursor.
         """
@@ -113,6 +117,11 @@ class TDengineCursor(object):
             pass
 
         res = CTaosInterface.query(self._connection._conn, stmt)
+
+        if (self._logfile):
+            with open(self._logfile, "a") as logfile:
+                logfile.write("%s;\n" % operation)
+
         if res == 0:
             if CTaosInterface.fieldsCount(self._connection._conn) == 0:
                 self._affected_rows += CTaosInterface.affectedRows(

src/connector/python/linux/python3/taos/cursor.py

@@ -36,6 +36,7 @@ class TDengineCursor(object):
         self._block_rows = -1
         self._block_iter = 0
         self._affected_rows = 0
+        self._logfile = ""
 
         if connection is not None:
             self._connection = connection
@@ -85,6 +86,9 @@ class TDengineCursor(object):
         """
         pass
 
+    def log(self, logfile):
+        self._logfile = logfile
+
     def close(self):
         """Close the cursor.
         """
@@ -121,6 +125,10 @@ class TDengineCursor(object):
         res = CTaosInterface.query(self._connection._conn, stmt)
         # print("   << Query ({}) Exec Done".format(localSeqNum))
 
+        if (self._logfile):
+            with open(self._logfile, "a") as logfile:
+                logfile.write("%s;\n" % operation)
+
         if res == 0:
             if CTaosInterface.fieldsCount(self._connection._conn) == 0:
                 self._affected_rows += CTaosInterface.affectedRows(

src/cq/src/cqMain.c

@@ -18,6 +18,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <pthread.h>
+#include <errno.h>
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tglobal.h"
@@ -64,7 +65,10 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
 void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   
   SCqContext *pContext = calloc(sizeof(SCqContext), 1);
-  if (pContext == NULL) return NULL;
+  if (pContext == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return NULL;
+  }
 
   strcpy(pContext->user, pCfg->user);
   strcpy(pContext->pass, pCfg->pass);
@@ -82,6 +86,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
 
 void cqClose(void *handle) {
   SCqContext *pContext = handle;
+  if (handle == NULL) return;
 
   // stop all CQs
   cqStop(pContext);
@@ -106,9 +111,9 @@ void cqClose(void *handle) {
 
 void cqStart(void *handle) {
   SCqContext *pContext = handle;
-  cTrace("vgId:%d, start all CQs", pContext->vgId);
   if (pContext->dbConn || pContext->master) return;
 
+  cTrace("vgId:%d, start all CQs", pContext->vgId);
   pthread_mutex_lock(&pContext->mutex);
 
   pContext->master = 1;

src/dnode/CMakeLists.txt

@@ -16,7 +16,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
 
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb twal vnode cJson lz4)
+  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4)
 
   IF (TD_ACCOUNT)
     TARGET_LINK_LIBRARIES(taosd account)

src/dnode/src/dnodeMain.c

@@ -37,6 +37,7 @@ static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
 int32_t dnodeInitSystem() {
   dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
   tscEmbedded  = 1;
+  taosBlockSIGPIPE();
   taosResolveCRC();
   taosInitGlobalCfg();
   taosReadGlobalLogCfg();

src/dnode/src/dnodeModule.c

@@ -20,6 +20,7 @@
 #include "trpc.h"
 #include "mnode.h"
 #include "http.h"
+#include "mqtt.h"
 #include "monitor.h"
 #include "dnodeInt.h"
 #include "dnodeModule.h"
@@ -62,6 +63,16 @@ static void dnodeAllocModules() {
     dnodeSetModuleStatus(TSDB_MOD_HTTP);
   }
 
+  tsModule[TSDB_MOD_MQTT].enable = (tsEnableMqttModule == 1);
+  tsModule[TSDB_MOD_MQTT].name = "mqtt";
+  tsModule[TSDB_MOD_MQTT].initFp = mqttInitSystem;
+  tsModule[TSDB_MOD_MQTT].cleanUpFp = mqttCleanUpSystem;
+  tsModule[TSDB_MOD_MQTT].startFp = mqttStartSystem;
+  tsModule[TSDB_MOD_MQTT].stopFp = mqttStopSystem;
+  if (tsEnableMqttModule) {
+    dnodeSetModuleStatus(TSDB_MOD_MQTT);
+  }
+
   tsModule[TSDB_MOD_MONITOR].enable    = (tsEnableMonitorModule == 1);
   tsModule[TSDB_MOD_MONITOR].name      = "monitor";
   tsModule[TSDB_MOD_MONITOR].initFp    = monitorInitSystem;
@@ -114,18 +125,20 @@ void dnodeStartModules() {
 }
 
 void dnodeProcessModuleStatus(uint32_t moduleStatus) {
-  bool enableMgmtModule = moduleStatus & (1 << TSDB_MOD_MGMT);
+  for (int32_t module = TSDB_MOD_MGMT; module < TSDB_MOD_HTTP; ++module) {
-  if (!tsModule[TSDB_MOD_MGMT].enable && enableMgmtModule) {
+    bool enableModule = moduleStatus & (1 << module);
-    dPrint("module status is received, start mgmt module", tsModuleStatus, moduleStatus);
+    if (!tsModule[module].enable && enableModule) {
-    tsModule[TSDB_MOD_MGMT].enable = true;
+      dPrint("module status:%u is received, start %s module", tsModuleStatus, tsModule[module].name);
-    dnodeSetModuleStatus(TSDB_MOD_MGMT);
+      tsModule[module].enable = true;
-    (*tsModule[TSDB_MOD_MGMT].startFp)();
+      dnodeSetModuleStatus(module);
-  }
+      (*tsModule[module].startFp)();
+    }
 
-  if (tsModule[TSDB_MOD_MGMT].enable && !enableMgmtModule) {
+    if (tsModule[module].enable && !enableModule) {
-    dPrint("module status is received, stop mgmt module", tsModuleStatus, moduleStatus);
+      dPrint("module status:%u is received, stop %s module", tsModuleStatus, tsModule[module].name);
-    tsModule[TSDB_MOD_MGMT].enable = false;
+      tsModule[module].enable = false;
-    dnodeUnSetModuleStatus(TSDB_MOD_MGMT);
+      dnodeUnSetModuleStatus(module);
-    (*tsModule[TSDB_MOD_MGMT].stopFp)();
+      (*tsModule[module].stopFp)();
+    }
   }
 }

src/inc/mqtt.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MQTT_H
+#define TDENGINE_MQTT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+int32_t mqttGetReqCount();
+int32_t mqttInitSystem();
+int32_t mqttStartSystem();
+void mqttStopSystem();
+void mqttCleanUpSystem();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/taosdef.h

@@ -157,6 +157,7 @@ extern tDataTypeDescriptor tDataTypeDesc[11];
 bool isValidDataType(int32_t type, int32_t length);
 bool isNull(const char *val, int32_t type);
 
+void setVardataNull(char* val, int32_t type);
 void setNull(char *val, int32_t type, int32_t bytes);
 void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems);
 
@@ -326,8 +327,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 
 #define TSDB_QUERY_TYPE_TAG_FILTER_QUERY              0x400u
 #define TSDB_QUERY_TYPE_INSERT                        0x100u    // insert type
-#define TSDB_QUERY_TYPE_IMPORT                        0x200u    // import data
+#define TSDB_QUERY_TYPE_MULTITABLE_QUERY              0x200u
-#define TSDB_QUERY_TYPE_MULTITABLE_QUERY              0x800u
 
 #define TSDB_QUERY_HAS_TYPE(x, _type)         (((x) & (_type)) != 0)
 #define TSDB_QUERY_SET_TYPE(x, _type)         ((x) |= (_type))
@@ -340,13 +340,14 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_SESSIONS_PER_VNODE (300)
 #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES)
 
-#define TSDB_MAX_MNODES        5
+#define TSDB_DEFAULT_MNODES_HASH_SIZE   5
-#define TSDB_MAX_DNODES        10
+#define TSDB_DEFAULT_DNODES_HASH_SIZE   10
-#define TSDB_MAX_ACCOUNTS      10
+#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10
-#define TSDB_MAX_USERS         20
+#define TSDB_DEFAULT_USERS_HASH_SIZE    20
-#define TSDB_MAX_DBS           100
+#define TSDB_DEFAULT_DBS_HASH_SIZE      100
-#define TSDB_MAX_VGROUPS       1000
+#define TSDB_DEFAULT_VGROUPS_HASH_SIZE  100
-#define TSDB_MAX_SUPER_TABLES  100
+#define TSDB_DEFAULT_STABLES_HASH_SIZE  100
+#define TSDB_DEFAULT_CTABLES_HASH_SIZE  10000
 
 #define TSDB_PORT_DNODESHELL 0 
 #define TSDB_PORT_DNODEDNODE 5 
@@ -375,6 +376,7 @@ typedef enum {
   TSDB_MOD_MGMT,
   TSDB_MOD_HTTP,
   TSDB_MOD_MONITOR,
+  TSDB_MOD_MQTT,
   TSDB_MOD_MAX
 } EModuleType;
 

src/inc/taoserror.h

@@ -156,6 +156,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY,         0, 405, "server out of m
 TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS,        0, 406, "no disk permissions")
 TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,             0, 407, "file corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,           0, 408, "memory corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUCH_FILE_OR_DIR,       0, 409, "no such file or directory")
 
 // client
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION,     0, 451, "invalid client version")

src/inc/taosmsg.h

@@ -269,9 +269,11 @@ typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN + 1];
   char    db[TSDB_DB_NAME_LEN + 1];
   int16_t type; /* operation type   */
-  char    tagVal[TSDB_MAX_BYTES_PER_ROW];
+  int16_t numOfCols; /* number of schema */
-  int8_t  numOfCols; /* number of schema */
+  int32_t tagValLen;
   SSchema schema[];
+  // tagVal is padded after schema
+  // char    tagVal[];
 } SCMAlterTableMsg;
 
 typedef struct {
@@ -441,7 +443,7 @@ typedef struct {
   int16_t     numOfOutput;  // final output columns numbers
   int16_t     tagNameRelType;   // relation of tag criteria and tbname criteria
   int16_t     fillType;      // interpolate type
-  uint64_t    defaultVal;       // default value array list
+  uint64_t    fillVal;       // default value array list
   int32_t     tsOffset;       // offset value in current msg body, NOTE: ts list is compressed
   int32_t     tsLen;          // total length of ts comp block
   int32_t     tsNumOfBlocks;  // ts comp block numbers
@@ -647,6 +649,7 @@ typedef struct SMultiTableMeta {
 } SMultiTableMeta;
 
 typedef struct {
+  int32_t dataLen;
   char name[TSDB_TABLE_ID_LEN + 1];
   char data[TSDB_MAX_TAGS_LEN];
 } STagData;
@@ -722,6 +725,8 @@ typedef struct {
 typedef struct {
   uint32_t  queryId;
   uint32_t  streamId;
+  uint32_t  totalDnodes;
+  uint32_t  onlineDnodes;
   int8_t    killConnection;
   SRpcIpSet ipList;
 } SCMHeartBeatRsp;

src/inc/tsync.h

@@ -94,8 +94,8 @@ typedef void* tsync_h;
 
 tsync_h syncStart(const SSyncInfo *);
 void    syncStop(tsync_h shandle);
-int     syncReconfig(tsync_h shandle, const SSyncCfg *);
+int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);
-int     syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
+int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
 void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
 void    syncRecover(tsync_h shandle);      // recover from other nodes:
 int     syncGetNodesRole(tsync_h shandle, SNodesRole *);

src/kit/shell/inc/shell.h

@@ -29,18 +29,6 @@
 #define MAX_COMMAND_SIZE       65536
 #define HISTORY_FILE           ".taos_history"
 
-#define BOOL_OUTPUT_LENGTH     6
-#define TINYINT_OUTPUT_LENGTH  6
-#define SMALLINT_OUTPUT_LENGTH 7
-#define INT_OUTPUT_LENGTH      11
-#define BIGINT_OUTPUT_LENGTH   21
-#define FLOAT_OUTPUT_LENGTH    20
-#define DOUBLE_OUTPUT_LENGTH   25
-#define BINARY_OUTPUT_LENGTH   20
-
-// dynamic config timestamp width according to maximum time precision
-extern int32_t TIMESTAMP_OUTPUT_LENGTH;
-
 typedef struct SShellHistory {
   char* hist[MAX_HISTORY_SIZE];
   int   hstart;
@@ -80,7 +68,7 @@ void get_history_path(char* history);
 void cleanup_handler(void* arg);
 void exitShell();
 int shellDumpResult(TAOS* con, char* fname, int* error_no, bool printMode);
-void shellPrintNChar(char* str, int width, bool printMode);
+void shellPrintNChar(const char* str, int length, int width);
 void shellGetGrantInfo(void *con);
 int isCommentLine(char *line);
 

src/kit/shell/src/shellDarwin.c

@@ -352,37 +352,31 @@ void *shellLoopQuery(void *arg) {
   return NULL;
 }
 
-void shellPrintNChar(char *str, int width, bool printMode) {
+void shellPrintNChar(const char *str, int length, int width) {
-  int col_left = width;
+  int pos = 0, cols = 0;
-  wchar_t wc;
+  while (pos < length) {
-  while (col_left > 0) {
+    wchar_t wc;
-    if (*str == '\0') break;
+    pos += mbtowc(&wc, str + pos, MB_CUR_MAX);
-    char *tstr = str;
+    if (pos > length) {
-    int byte_width = mbtowc(&wc, tstr, MB_CUR_MAX);
+      break;
-    if (byte_width <= 0) break;
+    }
-    int col_width = wcwidth(wc);
+
-    if (col_width <= 0) {
+    int w = wcwidth(wc);
-      str += byte_width;
+    if (w > 0) {
-      continue;
+      if (width > 0 && cols + w > width) {
+        break;
+      }
+      printf("%lc", wc);
+      cols += w;
     }
-    if (col_left < col_width) break;
-    printf("%lc", wc);
-    str += byte_width;
-    col_left -= col_width;
   }
 
-  while (col_left > 0) {
+  for (; cols < width; cols++) {
-    printf(" ");
+    putchar(' ');
-    col_left--;
-  }
-
-  if (!printMode) {
-    printf("|");
-  } else {
-    printf("\n");
   }
 }
 
+
 int get_old_terminal_mode(struct termios *tio) {
   /* Make sure stdin is a terminal. */
   if (!isatty(STDIN_FILENO)) {

src/kit/shell/src/shellEngine.c

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _BSD_SOURCE
+#define _GNU_SOURCE
 #define _XOPEN_SOURCE
 #define _DEFAULT_SOURCE
 
@@ -35,6 +37,9 @@ int       prompt_size = 6;
 TAOS_RES *result = NULL;
 SShellHistory   history;
 
+#define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30
+extern int32_t tsMaxBinaryDisplayWidth;
+
 /*
  * FUNCTION: Initialize the shell.
  */
@@ -195,7 +200,15 @@ int32_t shellRunCommand(TAOS *con, char *command) {
   } else if (regex_match(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
     // If clear the screen.
     system("clear");
-    return 0;
+  } else if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
+    strtok(command, " \t");
+    strtok(NULL, " \t");
+    char* p = strtok(NULL, " \t");
+    if (strcasecmp(p, "default") == 0) {
+      tsMaxBinaryDisplayWidth = DEFAULT_MAX_BINARY_DISPLAY_WIDTH;
+    } else {
+      tsMaxBinaryDisplayWidth = atoi(p);
+    }
   } else if (regex_match(command, "^[ \t]*source[\t ]+[^ ]+[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
     /* If source file. */
     char *c_ptr = strtok(command, " ;");
@@ -310,360 +323,347 @@ int regex_match(const char *s, const char *reg, int cflags) {
   return 0;
 }
 
-int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) {
-  TAOS_ROW   row = NULL;
-  int        numOfRows = 0;
-  time_t     tt;
-  char       buf[25] = "\0";
-  struct tm *ptm;
-  int        output_bytes = 0;
-  FILE *     fp = NULL;
-  int        num_fields = taos_field_count(con);
-  wordexp_t  full_path;
 
-  assert(num_fields != 0);
+static char* formatTimestamp(char* buf, int64_t val, int precision) {
+  if (args.is_raw_time) {
+    sprintf(buf, "%" PRId64, val);
+    return buf;
+  }
 
-  result = taos_use_result(con);
+  time_t tt;
+  if (precision == TSDB_TIME_PRECISION_MICRO) {
+    tt = (time_t)(val / 1000000);
+  } else {
+    tt = (time_t)(val / 1000);
+  }
+
+  struct tm* ptm = localtime(&tt);
+  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);
+
+  if (precision == TSDB_TIME_PRECISION_MICRO) {
+    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
+  } else {
+    sprintf(buf + pos, ".%03d", (int)(val % 1000));
+  }
+
+  return buf;
+}
+
+
+static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) {
+  if (val == NULL) {
+    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
+    return;
+  }
+
+  char buf[TSDB_MAX_BYTES_PER_ROW];
+  switch (field->type) {
+    case TSDB_DATA_TYPE_BOOL:
+      fprintf(fp, "%d", ((((int)(*((char *)val))) == 1) ? 1 : 0));
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      fprintf(fp, "%d", (int)(*((char *)val)));
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      fprintf(fp, "%d", (int)(*((short *)val)));
+      break;
+    case TSDB_DATA_TYPE_INT:
+      fprintf(fp, "%d", *((int *)val));
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      fprintf(fp, "%" PRId64, *((int64_t *)val));
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
+      break;
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      memcpy(buf, val, length);
+      buf[length] = 0;
+      fprintf(fp, "\'%s\'", buf);
+      break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      formatTimestamp(buf, *(int64_t*)val, precision);
+      fprintf(fp, "'%s'", buf);
+      break;
+    default:
+      break;
+  }
+}
+
+static int dumpResultToFile(const char* fname, TAOS_RES* result) {
+  TAOS_ROW row = taos_fetch_row(result);
+  if (row == NULL) {
+    return 0;
+  }
+
+  wordexp_t full_path;
+
+  if (wordexp(fname, &full_path, 0) != 0) {
+    fprintf(stderr, "ERROR: invalid file name: %s\n", fname);
+    return -1;
+  }
+
+  FILE* fp = fopen(full_path.we_wordv[0], "w");
+  if (fp == NULL) {
+    fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]);
+    wordfree(&full_path);
+    return -1;
+  }
+
+  wordfree(&full_path);
+
+  int num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  int32_t* length = taos_fetch_lengths(result);
+  int precision = taos_result_precision(result);
+
+  for (int col = 0; col < num_fields; col++) {
+    if (col > 0) {
+      fprintf(fp, ",");
+    }
+    fprintf(fp, "%s", fields[col].name);
+  }
+  fputc('\n', fp);
+  
+  int numOfRows = 0;
+  do {
+    for (int i = 0; i < num_fields; i++) {
+      if (i > 0) {
+        fputc(',', fp);
+      }
+      dumpFieldToFile(fp, row[i], fields +i, length[i], precision);
+    }
+    fputc('\n', fp);
+
+    numOfRows++;
+    row = taos_fetch_row(result);
+  } while( row != NULL);
+
+  fclose(fp);
+  return numOfRows;
+}
+
+
+static void printField(const char* val, TAOS_FIELD* field, int width, int32_t length, int precision) {
+  if (val == NULL) {
+    int w = width;
+    if (field->type < TSDB_DATA_TYPE_TINYINT || field->type > TSDB_DATA_TYPE_DOUBLE) {
+      w = 0;
+    }
+    w = printf("%*s", w, TSDB_DATA_NULL_STR);
+    for (; w < width; w++) {
+      putchar(' ');
+    }
+    return;
+  }
+
+  char buf[TSDB_MAX_BYTES_PER_ROW];
+  switch (field->type) {
+    case TSDB_DATA_TYPE_BOOL:
+      printf("%*s", width, ((((int)(*((char *)val))) == 1) ? "true" : "false"));
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      printf("%*d", width, (int)(*((char *)val)));
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      printf("%*d", width, (int)(*((short *)val)));
+      break;
+    case TSDB_DATA_TYPE_INT:
+      printf("%*d", width, *((int *)val));
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      printf("%*" PRId64, width, *((int64_t *)val));
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      printf("%*.5f", width, GET_FLOAT_VAL(val));
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      printf("%*.9f", width, GET_DOUBLE_VAL(val));
+      break;
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      shellPrintNChar(val, length, width);
+      break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      formatTimestamp(buf, *(int64_t*)val, precision);
+      printf("%s", buf);
+      break;
+    default:
+      break;
+  }
+}
+
+
+static int verticalPrintResult(TAOS_RES* result) {
+  TAOS_ROW row = taos_fetch_row(result);
+  if (row == NULL) {
+    return 0;
+  }
+
+  int num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  int32_t* length = taos_fetch_lengths(result);
+  int precision = taos_result_precision(result);
+
+  int maxColNameLen = 0;
+  for (int col = 0; col < num_fields; col++) {
+    int len = strlen(fields[col].name);
+    if (len > maxColNameLen) {
+      maxColNameLen = len;
+    }
+  }
+
+  int numOfRows = 0;
+  do {
+    printf("*************************** %d.row ***************************\n", numOfRows + 1);
+    for (int i = 0; i < num_fields; i++) {
+      TAOS_FIELD* field = fields + i;
+
+      int padding = (int)(maxColNameLen - strlen(field->name));
+      printf("%*.s%s: ", padding, " ", field->name);
+
+      printField(row[i], field, 0, length[i], precision);
+      putchar('\n');
+    }
+
+    numOfRows++;
+    row = taos_fetch_row(result);
+  } while(row != NULL);
+
+  return numOfRows;
+}
+
+
+static int calcColWidth(TAOS_FIELD* field, int precision) {
+  int width = strlen(field->name);
+
+  switch (field->type) {
+    case TSDB_DATA_TYPE_BOOL:
+      return MAX(5, width); // 'false'
+
+    case TSDB_DATA_TYPE_TINYINT:
+      return MAX(4, width); // '-127'
+
+    case TSDB_DATA_TYPE_SMALLINT:
+      return MAX(6, width); // '-32767'
+
+    case TSDB_DATA_TYPE_INT:
+      return MAX(11, width); // '-2147483648'
+
+    case TSDB_DATA_TYPE_BIGINT:
+      return MAX(21, width); // '-9223372036854775807'
+
+    case TSDB_DATA_TYPE_FLOAT:
+      return MAX(20, width);
+
+    case TSDB_DATA_TYPE_DOUBLE:
+      return MAX(25, width);
+
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      if (field->bytes > tsMaxBinaryDisplayWidth) {
+        return MAX(tsMaxBinaryDisplayWidth, width);
+      } else {
+        return MAX(field->bytes, width);
+      }
+
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      if (args.is_raw_time) {
+        return MAX(14, width);
+      } else if (precision == TSDB_TIME_PRECISION_MICRO) {
+        return MAX(26, width); // '2020-01-01 00:00:00.000000'
+      } else {
+        return MAX(23, width); // '2020-01-01 00:00:00.000'
+      }
+
+    default:
+      assert(false);
+  }
+
+  return 0;
+}
+
+
+static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) {
+  int rowWidth = 0;
+  for (int col = 0; col < num_fields; col++) {
+    TAOS_FIELD* field = fields + col;
+    int padding = (int)(width[col] - strlen(field->name));
+    int left = padding / 2;
+    printf(" %*.s%s%*.s |", left, " ", field->name, padding - left, " ");
+    rowWidth += width[col] + 3;
+  }
+
+  putchar('\n');
+  for (int i = 0; i < rowWidth; i++) {
+    putchar('=');
+  }
+  putchar('\n');
+}
+
+
+static int horizontalPrintResult(TAOS_RES* result) {
+  TAOS_ROW row = taos_fetch_row(result);
+  if (row == NULL) {
+    return 0;
+  }
+
+  int num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  int32_t* length = taos_fetch_lengths(result);
+  int precision = taos_result_precision(result);
+
+  int width[TSDB_MAX_COLUMNS];
+  for (int col = 0; col < num_fields; col++) {
+    width[col] = calcColWidth(fields + col, precision);
+  }
+
+  printHeader(fields, width, num_fields);
+
+  int numOfRows = 0;
+  do {
+    for (int i = 0; i < num_fields; i++) {
+      putchar(' ');
+      printField(row[i], fields + i, width[i], length[i], precision);
+      putchar(' ');
+      putchar('|');
+    }
+    putchar('\n');
+    numOfRows++;
+    row = taos_fetch_row(result);
+  } while(row != NULL);
+
+  return numOfRows;
+}
+
+
+int shellDumpResult(TAOS *con, char *fname, int *error_no, bool vertical) {
+  int numOfRows = 0;
+
+  TAOS_RES* result = taos_use_result(con);
   if (result == NULL) {
     taos_error(con);
     return -1;
   }
 
   if (fname != NULL) {
-    if (wordexp(fname, &full_path, 0) != 0) {
+    numOfRows = dumpResultToFile(fname, result);
-      fprintf(stderr, "ERROR: invalid file name: %s\n", fname);
+  } else if(vertical) {
-      return -1;
+    numOfRows = verticalPrintResult(result);
-    }
+  } else {
-
+    numOfRows = horizontalPrintResult(result);
-    fp = fopen(full_path.we_wordv[0], "w");
-    if (fp == NULL) {
-      fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]);
-      wordfree(&full_path);
-      return -1;
-    }
-
-    wordfree(&full_path);
-  }
-
-  TAOS_FIELD *fields = taos_fetch_fields(result);
-
-  row = taos_fetch_row(result);
-  int32_t* length = taos_fetch_lengths(result);
-  
-  char t_str[TSDB_MAX_BYTES_PER_ROW] = "\0";
-  int  l[TSDB_MAX_COLUMNS] = {0};
-  int  maxLenColumnName = 0;
-
-  if (row) {
-    // Print the header indicator
-    if (fname == NULL) {  // print to standard output
-      if (!printMode) {
-        for (int col = 0; col < num_fields; col++) {
-          switch (fields[col].type) {
-            case TSDB_DATA_TYPE_BOOL:
-              l[col] = MAX(BOOL_OUTPUT_LENGTH, strlen(fields[col].name));
-              break;
-            case TSDB_DATA_TYPE_TINYINT:
-              l[col] = MAX(TINYINT_OUTPUT_LENGTH, strlen(fields[col].name));
-              break;
-            case TSDB_DATA_TYPE_SMALLINT:
-              l[col] = MAX(SMALLINT_OUTPUT_LENGTH, strlen(fields[col].name));
-              break;
-            case TSDB_DATA_TYPE_INT:
-              l[col] = MAX(INT_OUTPUT_LENGTH, strlen(fields[col].name));
-              break;
-            case TSDB_DATA_TYPE_BIGINT:
-              l[col] = MAX(BIGINT_OUTPUT_LENGTH, strlen(fields[col].name));
-              break;
-            case TSDB_DATA_TYPE_FLOAT:
-              l[col] = MAX(FLOAT_OUTPUT_LENGTH, strlen(fields[col].name));
-              break;
-            case TSDB_DATA_TYPE_DOUBLE:
-              l[col] = MAX(DOUBLE_OUTPUT_LENGTH, strlen(fields[col].name));
-              break;
-            case TSDB_DATA_TYPE_BINARY:
-            case TSDB_DATA_TYPE_NCHAR:
-              l[col] = MAX(fields[col].bytes, strlen(fields[col].name));
-              /* l[col] = max(BINARY_OUTPUT_LENGTH, strlen(fields[col].name)); */
-              break;
-            case TSDB_DATA_TYPE_TIMESTAMP: {
-              int32_t defaultWidth = TIMESTAMP_OUTPUT_LENGTH;
-              if (args.is_raw_time) {
-                defaultWidth = 14;
-              }
-              if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) {
-                defaultWidth += 3;
-              }
-              l[col] = MAX(defaultWidth, strlen(fields[col].name));
-
-              break;
-            }
-            default:
-              break;
-          }
-
-          int spaces = (int)(l[col] - strlen(fields[col].name));
-          int left_space = spaces / 2;
-          int right_space = (spaces % 2 ? left_space + 1 : left_space);
-          printf("%*.s%s%*.s|", left_space, " ", fields[col].name, right_space, " ");
-          output_bytes += (l[col] + 1);
-        }
-        printf("\n");
-        for (int k = 0; k < output_bytes; k++) printf("=");
-        printf("\n");
-      } else {
-        for (int col = 0; col < num_fields; col++) {
-          if (strlen(fields[col].name) > maxLenColumnName) maxLenColumnName = strlen(fields[col].name);
-        }
-      }
-
-      // print the elements
-      do {
-        if (!printMode) {
-          for (int i = 0; i < num_fields; i++) {
-            if (row[i] == NULL) {
-              printf("%*s|", l[i], TSDB_DATA_NULL_STR);
-              continue;
-            }
-
-            switch (fields[i].type) {
-              case TSDB_DATA_TYPE_BOOL:
-                printf("%*s|", l[i], ((((int)(*((char *)row[i]))) == 1) ? "true" : "false"));
-                break;
-              case TSDB_DATA_TYPE_TINYINT:
-                printf("%*d|", l[i], (int)(*((char *)row[i])));
-                break;
-              case TSDB_DATA_TYPE_SMALLINT:
-                printf("%*d|", l[i], (int)(*((short *)row[i])));
-                break;
-              case TSDB_DATA_TYPE_INT:
-                printf("%*d|", l[i], *((int *)row[i]));
-                break;
-              case TSDB_DATA_TYPE_BIGINT:
-                printf("%*" PRId64 "|", l[i], *((int64_t *)row[i]));
-                break;
-              case TSDB_DATA_TYPE_FLOAT: {
-                float fv = 0;
-                fv = GET_FLOAT_VAL(row[i]);
-                printf("%*.5f|", l[i], fv);
-              }              
-                break;
-              case TSDB_DATA_TYPE_DOUBLE: {
-                double dv = 0;
-                dv = GET_DOUBLE_VAL(row[i]);
-                printf("%*.9f|", l[i], dv);
-              }
-                break;
-              case TSDB_DATA_TYPE_BINARY:
-              case TSDB_DATA_TYPE_NCHAR:
-                memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW);
-                memcpy(t_str, row[i], length[i]);
-                /* printf("%-*s|",max(fields[i].bytes, strlen(fields[i].name)),
-                 * t_str); */
-                /* printf("%-*s|", l[i], t_str); */
-                shellPrintNChar(t_str, l[i], printMode);
-                break;
-              case TSDB_DATA_TYPE_TIMESTAMP:
-                if (args.is_raw_time) {
-                  printf(" %" PRId64 "|", *(int64_t *)row[i]);
-                } else {
-                  if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) {
-                    tt = (time_t)((*(int64_t *)row[i]) / 1000000);
-                  } else {
-                    tt = (time_t)((*(int64_t *)row[i]) / 1000);
-                  }
-
-                  ptm = localtime(&tt);
-                  strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
-
-                  if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) {
-                    printf(" %s.%06d|", buf, (int)(*(int64_t *)row[i] % 1000000));
-                  } else {
-                    printf(" %s.%03d|", buf, (int)(*(int64_t *)row[i] % 1000));
-                  }
-                }
-                break;
-              default:
-                break;
-            }
-          }
-          printf("\n");
-        } else {
-          printf("*************************** %d.row ***************************\n", numOfRows + 1);
-          for (int i = 0; i < num_fields; i++) {
-            // 1. print column name
-            int left_space = (int)(maxLenColumnName - strlen(fields[i].name));
-            printf("%*.s%s: ", left_space, " ", fields[i].name);
-
-            // 2. print column value
-            if (row[i] == NULL) {
-              printf("%s\n", TSDB_DATA_NULL_STR);
-              continue;
-            }
-
-            switch (fields[i].type) {
-              case TSDB_DATA_TYPE_BOOL:
-                printf("%s\n", ((((int)(*((char *)row[i]))) == 1) ? "true" : "false"));
-                break;
-              case TSDB_DATA_TYPE_TINYINT:
-                printf("%d\n", (int)(*((char *)row[i])));
-                break;
-              case TSDB_DATA_TYPE_SMALLINT:
-                printf("%d\n", (int)(*((short *)row[i])));
-                break;
-              case TSDB_DATA_TYPE_INT:
-                printf("%d\n", *((int *)row[i]));
-                break;
-              case TSDB_DATA_TYPE_BIGINT:
-                printf("%" PRId64 "\n", *((int64_t *)row[i]));
-                break;
-              case TSDB_DATA_TYPE_FLOAT: {
-                float fv = 0;
-                fv = GET_FLOAT_VAL(row[i]);
-                printf("%.5f\n", fv);
-              }              
-                break;
-              case TSDB_DATA_TYPE_DOUBLE: {
-                double dv = 0;
-                dv = GET_DOUBLE_VAL(row[i]);
-                printf("%.9f\n", dv);
-              }
-                break;
-              case TSDB_DATA_TYPE_BINARY:
-              case TSDB_DATA_TYPE_NCHAR:
-                memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW);
-                memcpy(t_str, row[i], length[i]);
-                
-                l[i] = MAX(fields[i].bytes, strlen(fields[i].name));
-                shellPrintNChar(t_str, l[i], printMode);
-                break;
-              case TSDB_DATA_TYPE_TIMESTAMP:
-                if (args.is_raw_time) {
-                  printf("%" PRId64 "\n", *(int64_t *)row[i]);
-                } else {
-                  if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) {
-                    tt = (time_t)((*(int64_t *)row[i]) / 1000000);
-                  } else {
-                    tt = (time_t)((*(int64_t *)row[i]) / 1000);
-                  }
-
-                  ptm = localtime(&tt);
-                  strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
-
-                  if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) {
-                    printf("%s.%06d\n", buf, (int)(*(int64_t *)row[i] % 1000000));
-                  } else {
-                    printf("%s.%03d\n", buf, (int)(*(int64_t *)row[i] % 1000));
-                  }
-                }
-                break;
-              default:
-                break;
-            }
-          }
-        }
-
-        numOfRows++;
-      } while ((row = taos_fetch_row(result)));
-
-    } else {  // dump to file
-      // first write column
-      for (int col = 0; col < num_fields; col++) {
-        fprintf(fp, "%s", fields[col].name);
-        if (col < num_fields - 1) {
-          fprintf(fp, ",");
-        } else {
-          fprintf(fp, "\n");
-        }
-      }
-      
-      do {
-        for (int i = 0; i < num_fields; i++) {
-          if (row[i]) {
-            switch (fields[i].type) {
-              case TSDB_DATA_TYPE_BOOL:
-                fprintf(fp, "%d", ((((int)(*((char *)row[i]))) == 1) ? 1 : 0));
-                break;
-              case TSDB_DATA_TYPE_TINYINT:
-                fprintf(fp, "%d", (int)(*((char *)row[i])));
-                break;
-              case TSDB_DATA_TYPE_SMALLINT:
-                fprintf(fp, "%d", (int)(*((short *)row[i])));
-                break;
-              case TSDB_DATA_TYPE_INT:
-                fprintf(fp, "%d", *((int *)row[i]));
-                break;
-              case TSDB_DATA_TYPE_BIGINT:
-                fprintf(fp, "%" PRId64, *((int64_t *)row[i]));
-                break;
-              case TSDB_DATA_TYPE_FLOAT: {
-                float fv = 0;
-                fv = GET_FLOAT_VAL(row[i]);
-                fprintf(fp, "%.5f", fv);
-              }              
-                break;
-              case TSDB_DATA_TYPE_DOUBLE: {
-                double dv = 0;
-                dv = GET_DOUBLE_VAL(row[i]);
-                fprintf(fp, "%.9f", dv);
-              }
-                break;
-              case TSDB_DATA_TYPE_BINARY:
-              case TSDB_DATA_TYPE_NCHAR:
-                memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW);
-                memcpy(t_str, row[i], length[i]);
-                fprintf(fp, "\'%s\'", t_str);
-                break;
-              case TSDB_DATA_TYPE_TIMESTAMP:
-                if (args.is_raw_time) {
-                  fprintf(fp, "%" PRId64, *(int64_t *)row[i]);
-                } else {
-                  if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) {
-                    tt = (time_t)((*(int64_t *)row[i]) / 1000000);
-                  } else {
-                    tt = (time_t)((*(int64_t *)row[i]) / 1000);
-                  }
-
-                  ptm = localtime(&tt);
-                  strftime(buf, 64, "%Y-%m-%d %H:%M:%S", ptm);
-
-                  if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) {
-                    fprintf(fp, "\'%s.%06d\'", buf, (int)(*(int64_t *)row[i] % 1000000));
-                  } else {
-                    fprintf(fp, "\'%s.%03d\'", buf, (int)(*(int64_t *)row[i] % 1000));
-                  }
-                }
-                break;
-              default:
-                break;
-            }
-          } else {
-            fprintf(fp, "%s", TSDB_DATA_NULL_STR);
-          }
-          if (i < num_fields - 1) {
-            fprintf(fp, ",");
-          } else {
-            fprintf(fp, "\n");
-          }
-        }
-
-        numOfRows++;
-      } while ((row = taos_fetch_row(result)));
-    }
   }
 
   *error_no = taos_errno(con);
-
   taos_free_result(result);
-  result = NULL;
-
-  if (fname != NULL) {
-    fclose(fp);
-  }
-
   return numOfRows;
 }
 
+
 void read_history() {
   // Initialize history
   memset(history.hist, 0, sizeof(char *) * MAX_HISTORY_SIZE);

src/kit/shell/src/shellImport.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _GNU_SOURCE
 #define _XOPEN_SOURCE
 #define _DEFAULT_SOURCE
 

src/kit/shell/src/shellLinux.c

@@ -329,34 +329,27 @@ void *shellLoopQuery(void *arg) {
   return NULL;
 }
 
-void shellPrintNChar(char *str, int width, bool printMode) {
+void shellPrintNChar(const char *str, int length, int width) {
-  int col_left = width;
+  int pos = 0, cols = 0;
-  wchar_t wc;
+  while (pos < length) {
-  while (col_left > 0) {
+    wchar_t wc;
-    if (*str == '\0') break;
+    pos += mbtowc(&wc, str + pos, MB_CUR_MAX);
-    char *tstr = str;
+    if (pos > length) {
-    int byte_width = mbtowc(&wc, tstr, MB_CUR_MAX);
+      break;
-    if (byte_width <= 0) break;
+    }
-    int col_width = wcwidth(wc);
+
-    if (col_width <= 0) {
+    int w = wcwidth(wc);
-      str += byte_width;
+    if (w > 0) {
-      continue;
+      if (width > 0 && cols + w > width) {
+        break;
+      }
+      printf("%lc", wc);
+      cols += w;
     }
-    if (col_left < col_width) break;
-    printf("%lc", wc);
-    str += byte_width;
-    col_left -= col_width;
   }
 
-  while (col_left > 0) {
+  for (; cols < width; cols++) {
-    printf(" ");
+    putchar(' ');
-    col_left--;
-  }
-
-  if (!printMode) {
-    printf("|");
-  } else {
-    printf("\n");
   }
 }
 

src/kit/shell/src/shellMain.c

@@ -20,7 +20,6 @@
 
 TAOS*     con;
 pthread_t pid;
-int32_t   TIMESTAMP_OUTPUT_LENGTH = 22;
 
 // TODO: IMPLEMENT INTERRUPT HANDLER.
 void interruptHandler(int signum) {

src/kit/shell/src/shellWindows.c

@@ -217,32 +217,32 @@ void *shellLoopQuery(void *arg) {
   return NULL;
 }
 
-void shellPrintNChar(char *str, int width, bool printMode) {
+void shellPrintNChar(const char *str, int length, int width) {
-  int     col_left = width;
+  int pos = 0, cols = 0;
-  wchar_t wc;
+  while (pos < length) {
-  while (col_left > 0) {
+    wchar_t wc;
-    if (*str == '\0') break;
+    int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX);
-    char *tstr = str;
+    pos += bytes;
-    int   byte_width = mbtowc(&wc, tstr, MB_CUR_MAX);
+    if (pos > length) {
-    int   col_width = byte_width;
+      break;
-    if (col_left < col_width) break;
+    }
-    printf("%lc", wc);
+
-    str += byte_width;
+    int w = bytes;
-    col_left -= col_width;
+    if (w > 0) {
+      if (width > 0 && cols + w > width) {
+        break;
+      }
+      printf("%lc", wc);
+      cols += w;
+    }
   }
 
-  while (col_left > 0) {
+  for (; cols < width; cols++) {
-    printf(" ");
+    putchar(' ');
-    col_left--;
-  }
-  
-  if (!printMode) {
-    printf("|");
-  } else {
-    printf("\n");
   }
 }
 
+
 void get_history_path(char *history) { sprintf(history, "%s/%s", ".", HISTORY_FILE); }
 
 void exitShell() { exit(EXIT_SUCCESS); }

src/kit/taosdemo/taosdemo.c

@@ -362,6 +362,26 @@ int main(int argc, char *argv[]) {
   
   time_t tTime = time(NULL);
   struct tm tm = *localtime(&tTime);
+  printf("###################################################################\n");
+  printf("# Server IP:                         %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);
+  printf("# User:                              %s\n", user);
+  printf("# Password:                          %s\n", pass);
+  printf("# Use metric:                        %s\n", use_metric ? "true" : "false");
+  printf("# Datatype of Columns:               %s\n", dataString);
+  printf("# Binary Length(If applicable):      %d\n",
+          (strcasestr(dataString, "BINARY") != NULL) ? len_of_binary : -1);
+  printf("# Number of Columns per record:      %d\n", ncols_per_record);
+  printf("# Number of Connections:             %d\n", nconnections);
+  printf("# Number of Tables:                  %d\n", ntables);
+  printf("# Number of Data per Table:          %d\n", nrecords_per_table);
+  printf("# Records/Request:                   %d\n", nrecords_per_request);
+  printf("# Database name:                     %s\n", db_name);
+  printf("# Table prefix:                      %s\n", tb_prefix);
+  printf("# Test time:                         %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
+          tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
+  printf("###################################################################\n\n");
+  printf("Press enter key to continue");
+  getchar();
 
   fprintf(fp, "###################################################################\n");
   fprintf(fp, "# Server IP:                         %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);
@@ -858,15 +878,16 @@ void generateData(char *res, char **data_type, int num_of_cols, int64_t timestam
   pstr += sprintf(pstr, ")");
 }
 
+static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890";
 void rand_string(char *str, int size) {
-  memset(str, 0, size);
+  str[0] = 0;
-  const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890";
+  if (size > 0) {
-  char *sptr = str;
-  if (size) {
     --size;
-    for (size_t n = 0; n < size; n++) {
+    int n;
+    for (n = 0; n < size; n++) {
       int key = rand() % (int)(sizeof charset - 1);
-      sptr += sprintf(sptr, "%c", charset[key]);
+      str[n] = charset[key];
     }
+    str[n] = 0;
   }
 }

src/kit/taosdump/taosdump.c

@@ -37,6 +37,8 @@
 #define COMMAND_SIZE 65536
 #define DEFAULT_DUMP_FILE "taosdump.sql"
 
+#define MAX_DBS  100
+
 int  converStringToReadable(char *str, int size, char *buf, int bufsize);
 int  convertNCharToReadable(char *str, int size, char *buf, int bufsize);
 void taosDumpCharset(FILE *fp);
@@ -359,7 +361,7 @@ int main(int argc, char *argv[]) {
 
 void taosFreeDbInfos() {
   if (dbInfos == NULL) return;
-  for (int i = 0; i < TSDB_MAX_DBS; i++) tfree(dbInfos[i]);
+  for (int i = 0; i < MAX_DBS; i++) tfree(dbInfos[i]);
   tfree(dbInfos);
 }
 
@@ -437,7 +439,7 @@ int taosDumpOut(SDumpArguments *arguments) {
     return -1;
   }
 
-  dbInfos = (SDbInfo **)calloc(TSDB_MAX_DBS, sizeof(SDbInfo *));
+  dbInfos = (SDbInfo **)calloc(MAX_DBS, sizeof(SDbInfo *));
   if (dbInfos == NULL) {
     fprintf(stderr, "failed to allocate memory\n");
     goto _exit_failure;
@@ -531,7 +533,7 @@ int taosDumpOut(SDumpArguments *arguments) {
     }
   }
 
-  taos_free_result(result);
+  // taos_free_result(result);
 
   if (count == 0) {
     fprintf(stderr, "No databases valid to dump\n");
@@ -720,6 +722,57 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
   count_temp = counter;
 
   for (; counter < numOfCols; counter++) {
+    TAOS_ROW row = NULL;
+
+    sprintf(command, "select %s from %s limit 1", tableDes->cols[counter].field, tableDes->name);
+    if (taos_query(taos, command) != 0) {
+      fprintf(stderr, "failed to run command %s\n", command);
+      return;
+    }
+
+    result = taos_use_result(taos);
+    if (result == NULL) {
+      fprintf(stderr, "failed to use result\n");
+      return;
+    }
+
+    TAOS_FIELD *fields = taos_fetch_fields(result);
+
+    row = taos_fetch_row(result);
+    switch (fields[0].type) {
+      case TSDB_DATA_TYPE_BOOL:
+        sprintf(tableDes->cols[counter].note, "%d", ((((int)(*((char *)row[0]))) == 1) ? 1 : 0));
+        break;
+      case TSDB_DATA_TYPE_TINYINT:
+        sprintf(tableDes->cols[counter].note, "%d", (int)(*((char *)row[0])));
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        sprintf(tableDes->cols[counter].note, "%d", (int)(*((short *)row[0])));
+        break;
+      case TSDB_DATA_TYPE_INT:
+        sprintf(tableDes->cols[counter].note, "%d", *((int *)row[0]));
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        sprintf(tableDes->cols[counter].note, "%" PRId64 "", *((int64_t *)row[0]));
+        break;
+      case TSDB_DATA_TYPE_FLOAT:
+        sprintf(tableDes->cols[counter].note, "%f", GET_FLOAT_VAL(row[0]));
+        break;
+      case TSDB_DATA_TYPE_DOUBLE:
+        sprintf(tableDes->cols[counter].note, "%f", GET_DOUBLE_VAL(row[0]));
+        break;
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        sprintf(tableDes->cols[counter].note, "%" PRId64 "", *(int64_t *)row[0]);
+        break;
+      case TSDB_DATA_TYPE_BINARY:
+      case TSDB_DATA_TYPE_NCHAR:
+      default:
+        strncpy(tableDes->cols[counter].note, (char *)row[0], fields[0].bytes);
+        break;
+    }
+
+    taos_free_result(result);
+
     if (counter != count_temp) {
       if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
           strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
@@ -941,7 +994,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
           pstr += sprintf(pstr, "%d", *((int *)row[col]));
           break;
         case TSDB_DATA_TYPE_BIGINT:
-          pstr += sprintf(pstr, "%" PRId64 "", *((int64_t *)row[col]));
+          pstr += sprintf(pstr, "%" PRId64, *((int64_t *)row[col]));
           break;
         case TSDB_DATA_TYPE_FLOAT:
           pstr += sprintf(pstr, "%f", GET_FLOAT_VAL(row[col]));
@@ -960,7 +1013,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
           pstr += sprintf(pstr, "\'%s\'", tbuf);
           break;
         case TSDB_DATA_TYPE_TIMESTAMP:
-          pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]);
+          pstr += sprintf(pstr, "%" PRId64, *(int64_t *)row[col]);
           break;
         default:
           break;

src/mnode/inc/mgmtDef.h

@@ -63,10 +63,8 @@ typedef struct SMnodeObj {
   int8_t     updateEnd[1];
   int32_t    refCount;
   int8_t     role;
-  SDnodeObj *pDnode;
 } SMnodeObj;
 
-// todo use dynamic length string
 typedef struct {
   char  *tableId;
   int8_t type;

src/mnode/inc/mgmtDnode.h

@@ -34,6 +34,7 @@ char*   mgmtGetDnodeStatusStr(int32_t dnodeStatus);
 void    mgmtMonitorDnodeModule();
 
 int32_t mgmtGetDnodesNum();
+int32_t mgmtGetOnlinDnodesNum();
 void *  mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode);
 void    mgmtIncDnodeRef(SDnodeObj *pDnode);
 void    mgmtDecDnodeRef(SDnodeObj *pDnode);

src/mnode/inc/mgmtMnode.h

@@ -44,7 +44,7 @@ void    mgmtDecMnodeRef(struct SMnodeObj *pMnode);
 char *  mgmtGetMnodeRoleStr();
 void    mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
 void    mgmtGetMnodeInfos(void *mnodes);
-
+void    mgmtUpdateMnodeIpSet();
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtTable.h

@@ -31,6 +31,7 @@ void *  mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable);
 void *  mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
 void    mgmtDropAllChildTables(SDbObj *pDropDb);
 void    mgmtDropAllSuperTables(SDbObj *pDropDb);
+void    mgmtDropAllChildTablesInVgroups(SVgObj *pVgroup);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtVgroup.h

@@ -29,6 +29,7 @@ void    mgmtIncVgroupRef(SVgObj *pVgroup);
 void    mgmtDecVgroupRef(SVgObj *pVgroup);
 void    mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
 void    mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
+void    mgmtUpdateAllDbVgroups(SDbObj *pAlterDb);
 
 void *  mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup);
 void    mgmtUpdateVgroup(SVgObj *pVgroup);

src/mnode/src/mgmtAcct.c

@@ -94,7 +94,7 @@ int32_t mgmtInitAccts() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_ACCOUNT,
     .tableName    = "accounts",
-    .hashSessions = TSDB_MAX_ACCOUNTS,
+    .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE,
     .maxRowSize   = tsAcctUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,

src/mnode/src/mgmtDb.c

@@ -96,6 +96,7 @@ static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pDb, pOper->rowSize);
     free(pDb);
   }
+  mgmtUpdateAllDbVgroups(pSaved);
   mgmtDecDbRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
@@ -127,7 +128,7 @@ int32_t mgmtInitDbs() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_DB,
     .tableName    = "dbs",
-    .hashSessions = TSDB_MAX_DBS,
+    .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE,
     .maxRowSize   = tsDbUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,

src/mnode/src/mgmtDnode.c

@@ -74,7 +74,6 @@ static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
  
 #ifndef _SYNC 
-  //TODO: drop dnode local
   mgmtDropAllDnodeVgroups(pDnode);
 #endif  
   mgmtDropMnodeLocal(pDnode->dnodeId);
@@ -130,7 +129,7 @@ int32_t mgmtInitDnodes() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_DNODE,
     .tableName    = "dnodes",
-    .hashSessions = TSDB_MAX_DNODES,
+    .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE,
     .maxRowSize   = tsDnodeUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_AUTO,
@@ -179,6 +178,23 @@ int32_t mgmtGetDnodesNum() {
   return sdbGetNumOfRows(tsDnodeSdb);
 }
 
+int32_t mgmtGetOnlinDnodesNum(char *ep) {
+  SDnodeObj *pDnode = NULL;
+  void *     pIter = NULL;
+  int32_t    onlineDnodes = 0;
+
+  while (1) {
+    pIter = mgmtGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status != TAOS_DN_STATUS_OFFLINE) onlineDnodes++;
+    mgmtDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  return onlineDnodes;
+}
+
 void *mgmtGetDnode(int32_t dnodeId) {
   return sdbGetRow(tsDnodeSdb, &dnodeId);
 }
@@ -397,7 +413,6 @@ static int32_t mgmtCreateDnode(char *ep) {
   return code;
 }
 
-//TODO drop others tables
 int32_t mgmtDropDnode(SDnodeObj *pDnode) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
@@ -410,7 +425,7 @@ int32_t mgmtDropDnode(SDnodeObj *pDnode) {
     code = TSDB_CODE_SDB_ERROR;
   }
 
-  mLPrint("dnode:%d is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
+  mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
   return code;
 }
 

src/mnode/src/mgmtMnode.c

@@ -36,6 +36,25 @@ static int32_t tsMnodeUpdateSize = 0;
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
+static SRpcIpSet tsMnodeRpcIpSet;
+static SDMMnodeInfos tsMnodeInfos;
+
+#if defined(LINUX)
+  static pthread_rwlock_t        tsMnodeLock;
+  #define mgmtMnodeWrLock()      pthread_rwlock_wrlock(&tsMnodeLock)
+  #define mgmtMnodeRdLock()      pthread_rwlock_rdlock(&tsMnodeLock)
+  #define mgmtMnodeUnLock()      pthread_rwlock_unlock(&tsMnodeLock)
+  #define mgmtMnodeInitLock()    pthread_rwlock_init(&tsMnodeLock, NULL)
+  #define mgmtMnodeDestroyLock() pthread_rwlock_destroy(&tsMnodeLock)
+#else
+  static pthread_mutex_t         tsMnodeLock;
+  #define mgmtMnodeWrLock()      pthread_mutex_lock(&tsMnodeLock)
+  #define mgmtMnodeRdLock()      pthread_mutex_lock(&tsMnodeLock)
+  #define mgmtMnodeUnLock()      pthread_mutex_unlock(&tsMnodeLock)
+  #define mgmtMnodeInitLock()    pthread_mutex_init(&tsMnodeLock, NULL)
+  #define mgmtMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock)
+#endif
+
 static int32_t mgmtMnodeActionDestroy(SSdbOper *pOper) {
   tfree(pOper->pObj);
   return TSDB_CODE_SUCCESS;
@@ -46,7 +65,6 @@ static int32_t mgmtMnodeActionInsert(SSdbOper *pOper) {
   SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
   if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
 
-  pMnode->pDnode = pDnode;
   pDnode->isMgmt = true;
   mgmtDecDnodeRef(pDnode);
   
@@ -102,17 +120,22 @@ static int32_t mgmtMnodeActionRestored() {
     }
     sdbFreeIter(pIter);
   }
+
+  mgmtUpdateMnodeIpSet();
+
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t mgmtInitMnodes() {
+  mgmtMnodeInitLock();
+
   SMnodeObj tObj;
   tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_MNODE,
     .tableName    = "mnodes",
-    .hashSessions = TSDB_MAX_MNODES,
+    .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE,
     .maxRowSize   = tsMnodeUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_INT,
@@ -140,6 +163,7 @@ int32_t mgmtInitMnodes() {
 
 void mgmtCleanupMnodes() {
   sdbCloseTable(tsMnodeSdb);
+  mgmtMnodeDestroyLock();
 }
 
 int32_t mgmtGetMnodesNum() { 
@@ -177,50 +201,65 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
   }
 }
 
-void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
+void mgmtUpdateMnodeIpSet() {
-  void *pIter = NULL;
+  SRpcIpSet *ipSet = &tsMnodeRpcIpSet;
-  while (1) {
+  SDMMnodeInfos *mnodes = &tsMnodeInfos;
-    SMnodeObj *pMnode = NULL;
-    pIter = mgmtGetNextMnode(pIter, &pMnode);
-    if (pMnode == NULL) break;
 
-    strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
+  mPrint("update mnodes ipset, numOfIps:%d ", mgmtGetMnodesNum());
-    ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
 
-    if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
+  mgmtMnodeWrLock();
-      ipSet->inUse = ipSet->numOfIps;
-    }
 
-    ipSet->numOfIps++;
+  memset(ipSet, 0, sizeof(tsMnodeRpcIpSet));
-    
+  memset(mnodes, 0, sizeof(SDMMnodeInfos));
-    mgmtDecMnodeRef(pMnode);
-  }
-  sdbFreeIter(pIter);
-}
 
-void mgmtGetMnodeInfos(void *param) {
-  SDMMnodeInfos *mnodes = param;
-  mnodes->inUse = 0;
-  
   int32_t index = 0;
-  void *pIter = NULL;
+  void *  pIter = NULL;
   while (1) {
     SMnodeObj *pMnode = NULL;
     pIter = mgmtGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
-    mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
+    SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
-    strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp);
+    if (pDnode != NULL) {
-    if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
+      strcpy(ipSet->fqdn[ipSet->numOfIps], pDnode->dnodeFqdn);
-      mnodes->inUse = index;
+      ipSet->port[ipSet->numOfIps] = htons(pDnode->dnodePort);
+
+      mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
+      strcpy(mnodes->nodeInfos[index].nodeEp, pDnode->dnodeEp);
+
+      if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
+        ipSet->inUse = ipSet->numOfIps;
+        mnodes->inUse = index;
+      }
+
+      mPrint("mnode:%d, ep:%s %s", index, pDnode->dnodeEp,
+             pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
+
+      ipSet->numOfIps++;
+      index++;
     }
 
-    index++;
+    mgmtDecDnodeRef(pDnode);
     mgmtDecMnodeRef(pMnode);
   }
-  sdbFreeIter(pIter);
 
   mnodes->nodeNum = index;
+
+  sdbFreeIter(pIter);
+
+  mgmtMnodeUnLock();
+}
+
+void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
+  mgmtMnodeRdLock();
+  *ipSet = tsMnodeRpcIpSet;
+  mgmtMnodeUnLock();
+}
+
+void mgmtGetMnodeInfos(void *mnodeInfos) {
+  mgmtMnodeRdLock();
+  *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;
+  mgmtMnodeUnLock();
 }
 
 int32_t mgmtAddMnode(int32_t dnodeId) {
@@ -240,6 +279,8 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
     code = TSDB_CODE_SDB_ERROR;
   }
 
+  mgmtUpdateMnodeIpSet();
+
   return code;
 }
 
@@ -250,6 +291,8 @@ void mgmtDropMnodeLocal(int32_t dnodeId) {
     sdbDeleteRow(&oper);
     mgmtDecMnodeRef(pMnode);
   }
+
+  mgmtUpdateMnodeIpSet();
 }
 
 int32_t mgmtDropMnode(int32_t dnodeId) {
@@ -270,6 +313,9 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
   }
 
   sdbDecRef(tsMnodeSdb, pMnode);
+
+  mgmtUpdateMnodeIpSet();
+
   return code;
 }
 
@@ -343,7 +389,15 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pMnode->pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+    
+    SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+    if (pDnode != NULL) {
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+    } else {
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+    }
+    mgmtDecDnodeRef(pDnode);
+
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

src/mnode/src/mgmtSdb.c

@@ -28,6 +28,7 @@
 #include "mgmtDef.h"
 #include "mgmtInt.h"
 #include "mgmtMnode.h"
+#include "mgmtDnode.h"
 #include "mgmtSdb.h"
 
 typedef enum {
@@ -196,6 +197,8 @@ void sdbUpdateMnodeRoles() {
       mgmtDecMnodeRef(pMnode);
     }
   }
+
+  mgmtUpdateMnodeIpSet();
 }
 
 static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) {
@@ -257,10 +260,15 @@ void sdbUpdateSync() {
       if (pMnode == NULL) break;
 
       syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
-      syncCfg.nodeInfo[index].nodePort = pMnode->pDnode->dnodePort + TSDB_PORT_SYNC;
-      strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
-      index++;
 
+      SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+      if (pDnode != NULL) {
+        syncCfg.nodeInfo[index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC;
+        strcpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeEp);
+        index++;
+      }
+
+      mgmtDecDnodeRef(pDnode);
       mgmtDecMnodeRef(pMnode);
     }
     sdbFreeIter(pIter);
@@ -352,7 +360,7 @@ void sdbIncRef(void *handle, void *pObj) {
   int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
   atomic_add_fetch_32(pRefCount, 1);
   if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
-    sdbTrace("table:%s, add ref to record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
+    sdbTrace("add ref to table:%s record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
   }
 }
 
@@ -363,7 +371,7 @@ void sdbDecRef(void *handle, void *pObj) {
   int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
   int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
   if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) {
-    sdbTrace("table:%s, def ref of record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
+    sdbTrace("def ref of table:%s record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount);
   }
 
   int8_t *updateEnd = pObj + pTable->refCountPos - 1;
@@ -442,8 +450,8 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
 
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
+  sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%d version:%" PRIu64, pTable->tableName,
-           sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
+           sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
 
   (*pTable->insertFp)(pOper);
   return TSDB_CODE_SUCCESS;

src/mnode/src/mgmtShell.c

@@ -119,7 +119,7 @@ static void mgmtDoDealyedAddToShellQueue(void *param, void *tmrId) {
 
 void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) {
   void *unUsed = NULL;
-  taosTmrReset(mgmtDoDealyedAddToShellQueue, 1000, queuedMsg, tsMgmtTmr, &unUsed);
+  taosTmrReset(mgmtDoDealyedAddToShellQueue, 300, queuedMsg, tsMgmtTmr, &unUsed);
 }
 
 void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
@@ -325,6 +325,8 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
     return;
   }
 
+  pHBRsp->onlineDnodes = htonl(mgmtGetOnlinDnodesNum());
+  pHBRsp->totalDnodes = htonl(mgmtGetDnodesNum());
   mgmtGetMnodeIpSet(&pHBRsp->ipList);
   
   /*

src/mnode/src/mgmtTable.c

@@ -340,7 +340,7 @@ static int32_t mgmtInitChildTables() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_CTABLE,
     .tableName    = "ctables",
-    .hashSessions = tsMaxTables,
+    .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
     .maxRowSize   = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_VAR_STRING,
@@ -507,7 +507,7 @@ static int32_t mgmtInitSuperTables() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_STABLE,
     .tableName    = "stables",
-    .hashSessions = TSDB_MAX_SUPER_TABLES,
+    .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
     .maxRowSize   = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_VAR_STRING,
@@ -1139,7 +1139,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
   prefixLen = strlen(prefix);
 
   SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
-  char stableName[TSDB_TABLE_NAME_LEN] = {0};
+  char stableName[TSDB_TABLE_NAME_LEN + 1] = {0};
 
   while (numOfRows < rows) {    
     pShow->pIter = mgmtGetNextSuperTable(pShow->pIter, &pTable);
@@ -1305,7 +1305,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
         if (pDnode == NULL) break;
 
         strncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
-        pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(tsDnodeShellPort);
+        pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(pDnode->dnodePort);
 
         pVgroupInfo->vgroups[vgSize].numOfIps++;
       }
@@ -1334,13 +1334,13 @@ static void mgmtProcessDropSuperTableRsp(SRpcMsg *rpcMsg) {
 }
 
 static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) {
-  char *  pTagData = NULL;
+  STagData *  pTagData = NULL;
   int32_t tagDataLen = 0;
   int32_t totalCols = 0;
   int32_t contLen = 0;
   if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) {
-    pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1;
+    pTagData = (STagData*)pMsg->schema;
-    tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1;
+    tagDataLen = ntohl(pTagData->dataLen);
     totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags;
     contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen + pTable->sqlLen;
   } else {
@@ -1393,7 +1393,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb
   }
 
   if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) {
-    memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen);
+    memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData->data, tagDataLen);
     memcpy(pCreate->data + totalCols * sizeof(SSchema) + tagDataLen, pTable->sql, pTable->sqlLen);
   }
 
@@ -1420,10 +1420,10 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
   pTable->vgId         = pVgroup->vgId;
     
   if (pTable->info.type == TSDB_CHILD_TABLE) {
-    char *pTagData = (char *) pCreate->schema;  // it is a tag key
+    STagData *pTagData = (STagData *) pCreate->schema;  // it is a tag key
-    SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData);
+    SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData->name);
     if (pSuperTable == NULL) {
-      mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData);
+      mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name);
       free(pTable);
       terrno = TSDB_CODE_INVALID_TABLE;
       return NULL;
@@ -1538,7 +1538,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
   SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
   newMsg->ahandle = pMsg->pTable;
-  newMsg->maxRetry = 5;
+  newMsg->maxRetry = 10;
   SRpcMsg rpcMsg = {
       .handle  = newMsg,
       .pCont   = pMDCreate,
@@ -1742,7 +1742,9 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
 
 static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
   SCMTableInfoMsg *pInfo = pMsg->pCont;
-  int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
+  STagData* pTag = (STagData*)pInfo->tags;
+
+  int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + ntohl(pTag->dataLen);
   SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
   if (pCreateMsg == NULL) {
     mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
@@ -1756,14 +1758,9 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
   pCreateMsg->getMeta = 1;
   pCreateMsg->contLen = htonl(contLen);
 
-  contLen = sizeof(STagData);
+  memcpy(pCreateMsg->schema, pInfo->tags, contLen - sizeof(SCMCreateTableMsg));
-  if (contLen > pMsg->contLen - sizeof(SCMTableInfoMsg)) {
-    contLen = pMsg->contLen - sizeof(SCMTableInfoMsg);
-  }
-  memcpy(pCreateMsg->schema, pInfo->tags, contLen);
 
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
-  pMsg->pCont = newMsg->pCont;
   newMsg->msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
   newMsg->pCont = pCreateMsg;
 
@@ -1790,6 +1787,34 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
+void mgmtDropAllChildTablesInVgroups(SVgObj *pVgroup) {
+  void *  pIter = NULL;
+  int32_t numOfTables = 0;
+  SChildTableObj *pTable = NULL;
+
+  mPrint("vgId:%d, all child tables will be dropped from sdb", pVgroup->vgId);
+
+  while (1) {
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
+    if (pTable == NULL) break;
+
+    if (pTable->vgId == pVgroup->vgId) {
+      SSdbOper oper = {
+        .type = SDB_OPER_LOCAL,
+        .table = tsChildTableSdb,
+        .pObj = pTable,
+      };
+      sdbDeleteRow(&oper);
+      numOfTables++;
+    }
+    mgmtDecTableRef(pTable);
+  }
+
+  sdbFreeIter(pIter);
+
+  mPrint("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId);
+}
+
 void mgmtDropAllChildTables(SDbObj *pDropDb) {
   void *  pIter = NULL;
   int32_t numOfTables = 0;
@@ -1999,7 +2024,7 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
   SCMMultiTableInfoMsg *pInfo = pMsg->pCont;
   pInfo->numOfTables = htonl(pInfo->numOfTables);
 
-  int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice
+  int32_t totalMallocLen = 4 * 1024 * 1024;  // first malloc 4 MB, subsequent reallocation as twice
   SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen);
   if (pMultiMeta == NULL) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
@@ -2009,26 +2034,30 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
   pMultiMeta->contLen = sizeof(SMultiTableMeta);
   pMultiMeta->numOfTables = 0;
 
-  for (int t = 0; t < pInfo->numOfTables; ++t) {
+  for (int32_t t = 0; t < pInfo->numOfTables; ++t) {
-    char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN);
+    char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN + 1);
     SChildTableObj *pTable = mgmtGetChildTable(tableId);
     if (pTable == NULL) continue;
 
     if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(tableId);
-    if (pMsg->pDb == NULL) continue;
+    if (pMsg->pDb == NULL) {
+      mgmtDecTableRef(pTable);
+      continue;
+    }
 
     int availLen = totalMallocLen - pMultiMeta->contLen;
     if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)) {
-      //TODO realloc
+      totalMallocLen *= 2;
-      //totalMallocLen *= 2;
+      pMultiMeta = rpcReallocCont(pMultiMeta, totalMallocLen);
-      //pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen);
+      if (pMultiMeta == NULL) {
-      //if (pMultiMeta == NULL) {
+        mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
-      ///  rpcSendResponse(ahandle, TSDB_CODE_SERV_OUT_OF_MEMORY, NULL, 0);
+        mgmtDecTableRef(pTable);
-      //  return TSDB_CODE_SERV_OUT_OF_MEMORY;
+        return;
-      //} else {
+      } else {
-      //  t--;
+        t--;
-      //  continue;
+        mgmtDecTableRef(pTable);
-      //}
+        continue;
+      }
     }
 
     STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen);
@@ -2037,6 +2066,8 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
       pMultiMeta->numOfTables ++;
       pMultiMeta->contLen += pMeta->contLen;
     }
+
+    mgmtDecTableRef(pTable);
   }
 
   SRpcMsg rpcRsp = {0};
@@ -2123,7 +2154,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
       continue;
     }
 
-    char tableName[TSDB_TABLE_NAME_LEN] = {0};
+    char tableName[TSDB_TABLE_NAME_LEN + 1] = {0};
     
     // pattern compare for table name
     mgmtExtractTableName(pTable->info.tableId, tableName);
@@ -2201,6 +2232,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
   }
 
   pAlter->type = htons(pAlter->type);
+  pAlter->numOfCols = htons(pAlter->numOfCols);
+  pAlter->tagValLen = htonl(pAlter->tagValLen);
 
   if (pAlter->numOfCols > 2) {
     mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
@@ -2232,7 +2265,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
     mTrace("table:%s, start to alter ctable", pAlter->tableId);
     SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
     if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
-      code = mgmtModifyChildTableTagValue(pTable, pAlter->schema[0].name, pAlter->tagVal);
+      char *tagVal = (char*)(pAlter->schema + pAlter->numOfCols);
+      code = mgmtModifyChildTableTagValue(pTable, pAlter->schema[0].name, tagVal);
     } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
       code = mgmtAddNormalTableColumn(pMsg->pDb, pTable, pAlter->schema, 1);
     } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) {

src/mnode/src/mgmtUser.c

@@ -117,7 +117,7 @@ int32_t mgmtInitUsers() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_USER,
     .tableName    = "users",
-    .hashSessions = TSDB_MAX_USERS,
+    .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
     .maxRowSize   = tsUserUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,

src/mnode/src/mgmtVgroup.c

@@ -121,6 +121,20 @@ static int32_t mgmtVgroupActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
+static void mgmtVgroupUpdateIdPool(SVgObj *pVgroup) {
+  int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
+  SDbObj *pDb = pVgroup->pDb;
+  if (pDb != NULL) {
+    if (pDb->cfg.maxTables != oldTables) {
+      mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
+      taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
+      int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
+      pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
+      memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj **));
+    }
+  }
+}
+
 static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
   SVgObj *pNew = pOper->pObj;
   SVgObj *pVgroup = mgmtGetVgroup(pNew->vgId);
@@ -146,20 +160,11 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
     }
   }
 
-  int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
+  mgmtVgroupUpdateIdPool(pVgroup);
-  SDbObj *pDb = pVgroup->pDb;
-  if (pDb != NULL) {
-    if (pDb->cfg.maxTables != oldTables) {
-      mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
-      taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
-      int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
-      pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
-    }
-  }
 
   mgmtDecVgroupRef(pVgroup);
 
-  mTrace("vgId:%d, is updated, numOfVnode:%d tables:%d", pVgroup->vgId, pVgroup->numOfVnodes, pDb == NULL ? 0 : pDb->cfg.maxTables);
+  mTrace("vgId:%d, is updated, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -196,7 +201,7 @@ int32_t mgmtInitVgroups() {
   SSdbTableDesc tableDesc = {
     .tableId      = SDB_TABLE_VGROUP,
     .tableName    = "vgroups",
-    .hashSessions = TSDB_MAX_VGROUPS,
+    .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE,
     .maxRowSize   = tsVgUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_AUTO,
@@ -742,11 +747,14 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
   SVgObj *pVgroup = NULL;
   int32_t numOfVgroups = 0;
 
+  mPrint("dnode:%d, all vgroups will be dropped from sdb", pDropDnode->dnodeId);
+
   while (1) {
     pIter = mgmtGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
+      mgmtDropAllChildTablesInVgroups(pVgroup);
       SSdbOper oper = {
         .type = SDB_OPER_LOCAL,
         .table = tsVgroupSdb,
@@ -754,12 +762,35 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
       };
       sdbDeleteRow(&oper);
       numOfVgroups++;
-      continue;
     }
     mgmtDecVgroupRef(pVgroup);
   }
 
   sdbFreeIter(pIter);
+
+  mPrint("dnode:%d, all vgroups is dropped from sdb", pDropDnode->dnodeId);
+}
+
+void mgmtUpdateAllDbVgroups(SDbObj *pAlterDb) {
+  void *  pIter = NULL;
+  SVgObj *pVgroup = NULL;
+
+  mPrint("db:%s, all vgroups will be update in sdb", pAlterDb->name);
+
+  while (1) {
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    if (pVgroup->pDb == pAlterDb) {
+      mgmtVgroupUpdateIdPool(pVgroup);
+    }
+
+    mgmtDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+
+  mPrint("db:%s, all vgroups is updated in sdb", pAlterDb->name);
 }
 
 void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {

src/os/linux/src/linuxPlatform.c

@@ -148,6 +148,10 @@ static void taosDeleteTimer(void *tharg) {
   timer_delete(*pTimer);
 }
 
+static pthread_t timerThread;
+static timer_t         timerId;
+static volatile bool stopTimer = false;
+
 void *taosProcessAlarmSignal(void *tharg) {
   // Block the signal
   sigset_t sigset;
@@ -156,7 +160,6 @@ void *taosProcessAlarmSignal(void *tharg) {
   sigprocmask(SIG_BLOCK, &sigset, NULL);
   void (*callback)(int) = tharg;
 
-  static timer_t         timerId;
   struct sigevent sevent = {{0}};
 
   #ifdef _ALPINE
@@ -187,7 +190,7 @@ void *taosProcessAlarmSignal(void *tharg) {
   }
 
   int signo;
-  while (1) {
+  while (!stopTimer) {
     if (sigwait(&sigset, &signo)) {
       uError("Failed to wait signal: number %d", signo);
       continue;
@@ -202,7 +205,6 @@ void *taosProcessAlarmSignal(void *tharg) {
   return NULL;
 }
 
-static pthread_t timerThread;
 
 int taosInitTimer(void (*callback)(int), int ms) {
   pthread_attr_t tattr;
@@ -217,7 +219,7 @@ int taosInitTimer(void (*callback)(int), int ms) {
 }
 
 void taosUninitTimer() {
-  pthread_cancel(timerThread);
+  stopTimer = true;
   pthread_join(timerThread, NULL);
 }
 

src/plugins/CMakeLists.txt

@@ -3,3 +3,4 @@ PROJECT(TDengine)
 
 ADD_SUBDIRECTORY(monitor)
 ADD_SUBDIRECTORY(http)
+ADD_SUBDIRECTORY(mqtt)

src/plugins/http/src/httpServer.c

@@ -267,8 +267,10 @@ static void httpStopThread(HttpThread* pThread) {
   struct epoll_event event = { .events = EPOLLIN };
   eventfd_t fd = eventfd(1, 0);
   if (fd == -1) {
+    httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno));
     pthread_cancel(pThread->thread);
   } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno));
     pthread_cancel(pThread->thread);
   }
 

src/plugins/mqtt/CMakeLists.txt

@@ -0,0 +1,22 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
+  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  ADD_LIBRARY(mqtt ${SRC})
+  TARGET_LINK_LIBRARIES(mqtt taos_static z)
+
+  IF (TD_ADMIN)
+    TARGET_LINK_LIBRARIES(mqtt admin)
+  ENDIF ()
+ENDIF ()

src/plugins/mqtt/inc/mqttLog.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MQTT_LOG_H
+#define TDENGINE_MQTT_LOG_H
+
+#include "tlog.h"
+
+extern int32_t mqttDebugFlag;
+
+#define mqttError(...)                       \
+  if (mqttDebugFlag & DEBUG_ERROR) {         \
+    taosPrintLog("ERROR MQT ", 255, __VA_ARGS__); \
+  }
+#define mqttWarn(...)                                  \
+  if ( mqttDebugFlag & DEBUG_WARN) {                    \
+    taosPrintLog("WARN  MQT ",  mqttDebugFlag, __VA_ARGS__); \
+  }
+#define  mqttTrace(...)                           \
+  if ( mqttDebugFlag & DEBUG_TRACE) {             \
+    taosPrintLog("MQT ",  mqttDebugFlag, __VA_ARGS__); \
+  }
+#define  mqttDump(...)                                        \
+  if ( mqttDebugFlag & DEBUG_TRACE) {                         \
+    taosPrintLongString("MQT ",  mqttDebugFlag, __VA_ARGS__); \
+  }
+#define  mqttPrint(...) \
+  { taosPrintLog("MQT ", 255, __VA_ARGS__); }
+
+#endif

src/plugins/mqtt/inc/mqttSystem.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MQTT_SYSTEM_H
+#define TDENGINE_MQTT_SYSTEM_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+int32_t mqttGetReqCount();
+int32_t mqttInitSystem();
+int32_t mqttStartSystem();
+void    mqttStopSystem();
+void    mqttCleanUpSystem();
+ 
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/plugins/mqtt/src/mqttSystem.c

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mqttSystem.h"
+#include "mqtt.h"
+#include "mqttLog.h"
+#include "os.h"
+#include "taos.h"
+#include "tglobal.h"
+#include "tsocket.h"
+#include "ttimer.h"
+
+int32_t mqttGetReqCount() { return 0; }
+int     mqttInitSystem() {
+  mqttPrint("mqttInitSystem");
+  return 0;
+}
+
+int mqttStartSystem() {
+  mqttPrint("mqttStartSystem");
+  return 0;
+}
+
+void mqttStopSystem() {
+	mqttPrint("mqttStopSystem");
+}
+
+void mqttCleanUpSystem() { 
+	mqttPrint("mqttCleanUpSystem");
+}

src/query/inc/qExecutor.h

@@ -138,7 +138,7 @@ typedef struct SQuery {
   SColumnInfo*      colList;
   SColumnInfo*      tagColList;
   int32_t           numOfFilterCols;
-  int64_t*          defaultVal;
+  int64_t*          fillVal;
   uint32_t          status;  // query status
   SResultRec        rec;
   int32_t           pos;

src/query/inc/qfill.h

@@ -28,7 +28,7 @@ typedef struct {
   STColumn col;             // column info
   int16_t  functionId;      // sql function id
   int16_t  flag;            // column flag: TAG COLUMN|NORMAL COLUMN
-  union {int64_t i; double d;} defaultVal;
+  union {int64_t i; double d;} fillVal;
 } SFillColInfo;
   
 typedef struct SFillInfo {
@@ -75,15 +75,13 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu
 
 TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision);
 
-int32_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows);
+int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows);
 
 int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
 
-int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData);
-  
 int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
 
-void taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int64_t* outputRows, int32_t capacity);
+int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
 
 #ifdef __cplusplus
 }

src/query/inc/tsqlfunction.h

@@ -126,7 +126,6 @@ typedef struct SArithmeticSupport {
 
 typedef struct SQLPreAggVal {
   bool    isSet;
-  int32_t size;
   SDataStatis statis;
 } SQLPreAggVal;
 
@@ -174,7 +173,6 @@ typedef struct SQLFunctionCtx {
   int16_t  outputBytes;  // size of results, determined by function and input column data type
   bool     hasNull;      // null value exist in current block
   int16_t  functionId;   // function id
-  int32_t  blockStatus;  // Indicate if data is loaded, it is first/last/internal block. Only for file blocks
   void *   aInputElemBuf;
   char *   aOutputBuf;            // final result output buffer, point to sdata->data
   uint8_t  currentStage;          // record current running step, default: 0

src/query/src/qExecutor.c

@@ -110,7 +110,7 @@ static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInf
 static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId);
 static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow);
 
-static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
+static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* pData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
                           int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag);
 static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols);
@@ -200,7 +200,7 @@ bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *
       return false;
     } else {  // prev has been located
       if (pQuery->fileId >= 0) {
-        pQuery->pos = pQuery->pBlock[pQuery->slot].numOfPoints - 1;
+        pQuery->pos = pQuery->pBlock[pQuery->slot].numOfRows - 1;
         getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos);
 
         qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery),
@@ -210,11 +210,11 @@ bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *
         assert(vnodeIsDatablockLoaded(pRuntimeEnv, pMeterObj, -1, true) == DISK_BLOCK_NO_NEED_TO_LOAD);
         pBlock = &pRuntimeEnv->cacheBlock;
 
-        pQuery->pos = pBlock->numOfPoints - 1;
+        pQuery->pos = pBlock->numOfRows - 1;
         getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos);
 
         qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery),
-               pQuery->fileId, pQuery->slot, pBlock->numOfPoints - 1, pQuery->pos);
+               pQuery->fileId, pQuery->slot, pBlock->numOfRows - 1, pQuery->pos);
       }
     }
   }
@@ -376,11 +376,16 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) {
 
 bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; }
 
-static bool limitResults(SQuery *pQuery) {
+static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) {
+  SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+  
   if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) {
     pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total;
-    assert(pQuery->rec.rows > 0);
+    
-
+    qTrace("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%d, total:%"PRId64,
+        pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
+    assert(pQuery->rec.rows >= 0);
     setQueryStatus(pQuery, QUERY_COMPLETED);
     return true;
   }
@@ -403,23 +408,24 @@ static bool isTopBottomQuery(SQuery *pQuery) {
   return false;
 }
 
-static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) {
+static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, int32_t numOfCols, int32_t index) {
   // for a tag column, no corresponding field info
-  SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].base.colInfo;
+  SColIndex *pColIndex = &pQuery->pSelectExpr[index].base.colInfo;
-  if (TSDB_COL_IS_TAG(pColIndexEx->flag)) {
+  if (TSDB_COL_IS_TAG(pColIndex->flag)) {
     return NULL;
   }
-
+  
   /*
    * Choose the right column field info by field id, since the file block may be out of date,
    * which means the newest table schema is not equalled to the schema of this block.
+   * TODO: speedup by using bsearch
    */
-  for (int32_t i = 0; i < pDataBlockInfo->numOfCols; ++i) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
-    if (pColIndexEx->colId == pStatis[i].colId) {
+    if (pColIndex->colId == pStatis[i].colId) {
       return &pStatis[i];
     }
   }
-
+  
   return NULL;
 }
 
@@ -431,8 +437,7 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo
  * @param pColStatis
  * @return
  */
-static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis,
+static bool hasNullValue(SQuery *pQuery, int32_t col, int32_t numOfCols, SDataStatis *pStatis, SDataStatis **pColStatis) {
-                         SDataStatis **pColStatis) {
   SColIndex *pColIndex = &pQuery->pSelectExpr[col].base.colInfo;
   if (TSDB_COL_IS_TAG(pColIndex->flag)) {
     return false;
@@ -444,7 +449,7 @@ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlock
   }
 
   if (pStatis != NULL) {
-    *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col);
+    *pColStatis = getStatisInfo(pQuery, pStatis, numOfCols, col);
   } else {
     *pColStatis = NULL;
   }
@@ -603,9 +608,9 @@ static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int
   return &pWindowResInfo->pResult[slot].status;
 }
 
-static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos,
+static int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos,
                                       int16_t order, int64_t *pData) {
-  int32_t endPos = searchFn((char *)pData, numOfPoints, ekey, order);
+  int32_t endPos = searchFn((char *)pData, numOfRows, ekey, order);
   int32_t forwardStep = 0;
 
   if (endPos >= 0) {
@@ -624,15 +629,17 @@ static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t s
 /**
  * NOTE: the query status only set for the first scan of master scan.
  */
-static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) {
+static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
   if (pRuntimeEnv->scanFlag != MASTER_SCAN || (!isIntervalQuery(pQuery))) {
-    return;
+    return pWindowResInfo->size;
   }
 
   // no qualified results exist, abort check
+  int32_t numOfClosed = 0;
+  
   if (pWindowResInfo->size == 0) {
-    return;
+    return pWindowResInfo->size;
   }
 
   // query completed
@@ -646,10 +653,10 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
     int32_t i = 0;
     int64_t skey = TSKEY_INITIAL_VAL;
 
-    // TODO opt performance: get the closed time window here
     for (i = 0; i < pWindowResInfo->size; ++i) {
       SWindowResult *pResult = &pWindowResInfo->pResult[i];
       if (pResult->status.closed) {
+        numOfClosed += 1;
         continue;
       }
 
@@ -672,16 +679,26 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
 
     pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey;
 
-    // the number of completed slots are larger than the threshold, dump to client immediately.
+    // the number of completed slots are larger than the threshold, return current generated results to client.
-    int32_t n = numOfClosedTimeWindow(pWindowResInfo);
+    if (numOfClosed > pWindowResInfo->threshold) {
-    if (n > pWindowResInfo->threshold) {
+      qTrace("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return",
+          GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold);
+      
       setQueryStatus(pQuery, QUERY_RESBUF_FULL);
+    } else {
+      qTrace("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size,
+             numOfClosed);
     }
-
-    qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, n);
   }
-
+  
+  // output has reached the limitation, set query completed
+  if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed &&
+      pRuntimeEnv->scanFlag == MASTER_SCAN) {
+    setQueryStatus(pQuery, QUERY_COMPLETED);
+  }
+  
   assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL);
+  return numOfClosed;
 }
 
 static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn,
@@ -733,7 +750,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
 }
 
 static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin,
-                                      int32_t startPos, int32_t forwardStep, TSKEY *tsBuf) {
+                                      int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) {
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
@@ -743,12 +760,17 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat
 
       pCtx[k].nStartQueryTimestamp = pWin->skey;
       pCtx[k].size = forwardStep;
-      pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1);
+      pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
 
       if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
-        pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset];
+        pCtx[k].ptsList = tsBuf;
       }
 
+      // not a whole block involved in query processing, statistics data can not be used
+      if (forwardStep != numOfTotal) {
+        pCtx[k].preAggVals.isSet = false;
+      }
+      
       if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
         aAggs[functionId].xFunction(&pCtx[k]);
       }
@@ -890,7 +912,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
       }
       
       assert(dataBlock != NULL);
-      sas->data[i] = dataBlock + pCtx->startOffset * pQuery->colList[i].bytes;  // start from the offset
+      sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/;  // start from the offset
     }
 
   } else {  // other type of query function
@@ -933,14 +955,15 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
   
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
+    int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
 
     SDataStatis *tpField = NULL;
 
-    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField);
+    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &tpField);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
-    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull,
+    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, tpField, hasNull,
-                  &sasArray[k], pRuntimeEnv->scanFlag);
+                  &sasArray[k], colId);
   }
 
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
@@ -958,7 +981,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
         getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true);
 
     SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
-    doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol);
+    doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
 
     int32_t     index = pWindowResInfo->curIndex;
     STimeWindow nextWin = win;
@@ -978,7 +1001,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
       forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true);
 
       pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
-      doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol);
+      doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
     }
 
     pWindowResInfo->curIndex = index;
@@ -1154,14 +1177,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
-
+    int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
+    
     SDataStatis *pColStatis = NULL;
 
-    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis);
+    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &pColStatis);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
-    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull,
+    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, pColStatis, hasNull,
-                  &sasArray[k], pRuntimeEnv->scanFlag);
+                  &sasArray[k], colId);
   }
 
   // set the input column data
@@ -1214,7 +1238,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
       }
 
       // all startOffset are identical
-      offset -= pCtx[0].startOffset;
+//      offset -= pCtx[0].startOffset;
 
       SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
       doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset);
@@ -1255,9 +1279,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         }
       }
 
-      // all startOffset are identical
-      offset -= pCtx[0].startOffset;
-
       for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
         int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
         if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
@@ -1305,55 +1326,55 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
   TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey;
   pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
 
-  doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo);
 
   // interval query with limit applied
-  if (isIntervalQuery(pQuery) && pQuery->limit.limit > 0 &&
+  int32_t numOfRes = 0;
-      (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosedTimeWindow(pWindowResInfo) &&
+  
-      pRuntimeEnv->scanFlag == MASTER_SCAN) {
+  if (isIntervalQuery(pQuery)) {
-    setQueryStatus(pQuery, QUERY_COMPLETED);
+    numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo);
-  }
+  } else {
+    numOfRes = getNumOfResult(pRuntimeEnv);
 
-  int32_t numOfRes = getNumOfResult(pRuntimeEnv);
+    // update the number of output result
+    if (numOfRes > 0 && pQuery->checkBuffer == 1) {
+      assert(numOfRes >= pQuery->rec.rows);
+      pQuery->rec.rows = numOfRes;
 
-  // update the number of output result
+      if (numOfRes >= pQuery->rec.threshold) {
-  if (numOfRes > 0 && pQuery->checkBuffer == 1) {
+        setQueryStatus(pQuery, QUERY_RESBUF_FULL);
-    assert(numOfRes >= pQuery->rec.rows);
+      }
-    pQuery->rec.rows = numOfRes;
 
-    if (numOfRes >= pQuery->rec.threshold) {
+      if ((pQuery->limit.limit >= 0) && (pQuery->limit.limit + pQuery->limit.offset) <= numOfRes) {
-      setQueryStatus(pQuery, QUERY_RESBUF_FULL);
+        setQueryStatus(pQuery, QUERY_COMPLETED);
-    }
+      }
-    
-    if ((pQuery->limit.limit >= 0) && numOfRes >= (pQuery->limit.limit + pQuery->limit.offset)) {
-      setQueryStatus(pQuery, QUERY_COMPLETED);
     }
   }
 
   return numOfRes;
 }
 
-void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
+void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
-                   int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) {
+                   int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t colId) {
-  pCtx->scanFlag = scanFlag;
+  pCtx->hasNull  = hasNull;
-
   pCtx->aInputElemBuf = inputData;
-  pCtx->hasNull = hasNull;
 
   if (pStatis != NULL) {
-    pCtx->preAggVals.isSet = true;
+    pCtx->preAggVals.isSet  = true;
-    pCtx->preAggVals.size = size;
     pCtx->preAggVals.statis = *pStatis;
+    if (pCtx->preAggVals.statis.numOfNull == -1) {
+      pCtx->preAggVals.statis.numOfNull = pBlockInfo->rows;  // todo :can not be -1
+    }
   } else {
     pCtx->preAggVals.isSet = false;
   }
 
-  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0;
+  // limit/offset query will affect this value
-  pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? size - pQuery->pos : pQuery->pos + 1;
+  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0;
+  pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1;
 
   uint32_t status = aAggs[functionId].nStatus;
   if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {
-    pCtx->ptsList = &tsCol[pCtx->startOffset];
+    pCtx->ptsList = tsCol;
   }
 
   if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) {
@@ -1363,7 +1384,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
   } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA ||
              functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) {
     /*
-     * leastsquares function needs two columns of input, currently, the x value of linear equation is set to
+     * least squares function needs two columns of input, currently, the x value of linear equation is set to
      * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer
      *
      * top/bottom function needs timestamp to indicate when the
@@ -1377,6 +1398,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
 
   } else if (functionId == TSDB_FUNC_ARITHM) {
     pCtx->param[1].pz = param;
+  } else if (functionId == TSDB_FUNC_SPREAD) {  // set the statistics data for primary time stamp column
+    if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+      pCtx->preAggVals.isSet  = true;
+      pCtx->preAggVals.statis.min = pBlockInfo->window.skey;
+      pCtx->preAggVals.statis.max = pBlockInfo->window.ekey;
+    }
   }
 
 #if defined(_DEBUG_VIEW)
@@ -2015,10 +2042,10 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI
 
         tVariantCreateFromBinary(&pCtx->param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT);
 
-        if (isNull((char *)&pQuery->defaultVal[i], pCtx->inputType)) {
+        if (isNull((char *)&pQuery->fillVal[i], pCtx->inputType)) {
           pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
         } else {
-          tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->defaultVal[i], pCtx->inputBytes, pCtx->inputType);
+          tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->fillVal[i], pCtx->inputBytes, pCtx->inputType);
         }
 
         pInterpDetail->ts = pQuery->window.skey;
@@ -2318,7 +2345,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle,
 
 int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) {
   int32_t midPos = -1;
-  int32_t numOfPoints;
+  int32_t numOfRows;
 
   if (num <= 0) {
     return -1;
@@ -2337,8 +2364,8 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) {
       if (key == keyList[firstPos]) return firstPos;
       if (key < keyList[firstPos]) return firstPos - 1;
 
-      numOfPoints = lastPos - firstPos + 1;
+      numOfRows = lastPos - firstPos + 1;
-      midPos = (numOfPoints >> 1) + firstPos;
+      midPos = (numOfRows >> 1) + firstPos;
 
       if (key < keyList[midPos]) {
         lastPos = midPos - 1;
@@ -2363,8 +2390,8 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) {
           return lastPos;
       }
 
-      numOfPoints = lastPos - firstPos + 1;
+      numOfRows = lastPos - firstPos + 1;
-      midPos = (numOfPoints >> 1) + firstPos;
+      midPos = (numOfRows >> 1) + firstPos;
 
       if (key < keyList[midPos]) {
         lastPos = midPos - 1;
@@ -2455,13 +2482,13 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     }
 
     SDataStatis *pStatis = NULL;
-    SArray *     pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
-
     pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1;
+    
+    SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
     int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock);
 
-    qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d", GET_QINFO_ADDR(pRuntimeEnv),
+    qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv),
-           blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes);
+           blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey);
 
     // while the output buffer is full or limit/offset is applied, query may be paused here
     if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL | QUERY_COMPLETED)) {
@@ -3363,7 +3390,9 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus
   // during reverse scan
   pTableQueryInfo->lastKey = pStatus->lastKey;
   pQuery->status = pStatus->status;
+  
   pTableQueryInfo->win = pStatus->w;
+  pQuery->window = pTableQueryInfo->win;
 }
 
 void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
@@ -3385,6 +3414,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
     if (pRuntimeEnv->scanFlag == MASTER_SCAN) {
       qstatus.status = pQuery->status;
       qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step;
+      qstatus.lastKey = pTableQueryInfo->lastKey;
     }
 
     if (!needScanDataBlocksAgain(pRuntimeEnv)) {
@@ -3412,6 +3442,9 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
 
     setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
     pRuntimeEnv->scanFlag = REPEAT_SCAN;
+    
+    qTrace("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo,
+        cond.twindow.skey, cond.twindow.ekey);
 
     // check if query is killed or not
     if (isQueryKilled(pQInfo)) {
@@ -3633,12 +3666,14 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
     STimeWindow     w = {0};
     SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo;
 
-    getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w);
+    TSKEY sk = MIN(win.skey, win.ekey);
+    TSKEY ek = MAX(win.skey, win.ekey);
+    getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &skey1, &ekey1, &w);
     pWindowResInfo->startTime = pTableQueryInfo->win.skey;  // windowSKey may be 0 in case of 1970 timestamp
 
     if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
       if (!QUERY_IS_ASC_QUERY(pQuery)) {
-        assert(win.ekey == pQuery->window.skey);
+        assert(win.ekey == pQuery->window.ekey);
       }
       
       pWindowResInfo->prevSKey = w.skey;
@@ -3673,10 +3708,6 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) {
   return loadPrimaryTS;
 }
 
-bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) {
-  return (pQuery->intervalTime == 0) || ((queryRangeSet == 1) && (isIntervalQuery(pQuery)));
-}
-
 static int32_t getNumOfSubset(SQInfo *pQInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
@@ -3698,7 +3729,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde
   int32_t startIdx = 0;
   int32_t step = -1;
 
-  qTrace("QInfo:%p start to copy data from windowResInfo to query buf", GET_QINFO_ADDR(pQuery));
+  qTrace("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo);
   int32_t totalSubset = getNumOfSubset(pQInfo);
 
   if (orderType == TSDB_ORDER_ASC) {
@@ -3827,7 +3858,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
   }
   
   /*
-   * There are no results returned to client now.
+   * While the code reaches here, there are no results returned to client now.
    * If query is not completed yet, the gaps between two results blocks need to be handled after next data block
    * is retrieved from TSDB.
    *
@@ -3881,18 +3912,24 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
 }
 
 int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t numOfRows, int32_t *numOfInterpo) {
+  SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
   SQuery *pQuery = pRuntimeEnv->pQuery;
+  SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo;
+  
   while (1) {
-    taosGenerateDataBlock(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata, &pQuery->rec.rows, pQuery->rec.capacity);
+    int32_t ret = taosGenerateDataBlock(pFillInfo, (tFilePage**) pQuery->sdata, pQuery->rec.capacity);
-    int32_t ret = pQuery->rec.rows;
     
     // todo apply limit output function
     /* reached the start position of according to offset value, return immediately */
     if (pQuery->limit.offset == 0) {
+      qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows", pQInfo, pFillInfo->numOfRows, ret);
       return ret;
     }
 
     if (pQuery->limit.offset < ret) {
+      qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%d. Discard due to offset, remain:%d, new offset:%d",
+             pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0);
+      
       ret -= pQuery->limit.offset;
       // todo !!!!there exactly number of interpo is not valid.
       // todo refactor move to the beginning of buffer
@@ -3900,10 +3937,16 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
         memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset,
                 ret * pQuery->pSelectExpr[i].bytes);
       }
+      
       pQuery->limit.offset = 0;
       return ret;
     } else {
+      qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%d. Discard due to offset, "
+             "remain:%d, new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0,
+          pQuery->limit.offset - ret);
+      
       pQuery->limit.offset -= ret;
+      pQuery->rec.rows = 0;
       ret = 0;
     }
 
@@ -3911,8 +3954,6 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
       return ret;
     }
   }
-  
-  return 0;
 }
 
 void vnodePrintQueryStatistics(SQInfo *pQInfo) {
@@ -3993,8 +4034,8 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBloc
 
   int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
 
-  qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d", GET_QINFO_ADDR(pRuntimeEnv),
+  qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv),
-         pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes);
+         pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey);
 }
 
 void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
@@ -4022,7 +4063,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
       pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey;
       pTableQueryInfo->lastKey += step;
 
-      qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows,
+      qTrace("QInfo:%p skip rows:%d, offset:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows,
              pQuery->limit.offset);
     } else {  // find the appropriated start position in current block
       updateOffsetVal(pRuntimeEnv, &blockInfo);
@@ -4111,8 +4152,9 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
           int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock);
           pRuntimeEnv->windowResInfo.curIndex = index;  // restore the window index
           
-          qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d",
+          qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64,
-                 GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes);
+                 GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey);
+          
           return true;
         } else { // do nothing
           *start = tw.skey;
@@ -4206,7 +4248,7 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
     pFillCol[i].col.offset = offset;
     pFillCol[i].flag       = TSDB_COL_NORMAL;    // always be ta normal column for table query
     pFillCol[i].functionId = pExprInfo->base.functionId;
-    pFillCol[i].defaultVal.i = pQuery->defaultVal[i];
+    pFillCol[i].fillVal.i = pQuery->fillVal[i];
     
     offset += pExprInfo->bytes;
   }
@@ -4582,7 +4624,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       skipResults(pRuntimeEnv);
 
       // the limitation of output result is reached, set the query completed
-      if (limitResults(pQuery)) {
+      if (limitResults(pRuntimeEnv)) {
         pQInfo->tableIndex = pQInfo->groupInfo.numOfTables;
         break;
       }
@@ -4837,7 +4879,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
 
   skipResults(pRuntimeEnv);
-  limitResults(pQuery);
+  limitResults(pRuntimeEnv);
 }
 
 static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
@@ -4885,7 +4927,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
     resetCtxOutputBuf(pRuntimeEnv);
   }
 
-  limitResults(pQuery);
+  limitResults(pRuntimeEnv);
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) {
     qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo,
         pQuery->current->lastKey, pQuery->window.ekey);
@@ -4963,7 +5005,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
 
     // the offset is handled at prepare stage if no interpolation involved
     if (pQuery->fillType == TSDB_FILL_NONE || pQuery->rec.rows == 0) {
-      limitResults(pQuery);
+      limitResults(pRuntimeEnv);
       break;
     } else {
       TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
@@ -4971,11 +5013,10 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
       taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, ekey);
       taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
       numOfInterpo = 0;
+      
       pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, pQuery->rec.rows, &numOfInterpo);
-
-      qTrace("QInfo: %p fill results completed, final:%d", pQInfo, pQuery->rec.rows);
       if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-        limitResults(pQuery);
+        limitResults(pRuntimeEnv);
         break;
       }
 
@@ -5008,9 +5049,8 @@ static void tableQueryImpl(SQInfo *pQInfo) {
     int32_t remain = taosNumOfRemainRows(pRuntimeEnv->pFillInfo);
     pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, remain, &numOfInterpo);
   
-    qTrace("QInfo: %p fill results completed, final:%d", pQInfo, pQuery->rec.rows);
     if (pQuery->rec.rows > 0) {
-      limitResults(pQuery);
+      limitResults(pRuntimeEnv);
     }
     
     qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
@@ -5134,7 +5174,7 @@ bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SC
 
 static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) {
   if (pQueryMsg->intervalTime < 0) {
-    qError("qmsg:%p illegal value of interval time %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime);
+    qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->intervalTime);
     return false;
   }
 
@@ -5342,7 +5382,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
 
   pQueryMsg->fillType = htons(pQueryMsg->fillType);
   if (pQueryMsg->fillType != TSDB_FILL_NONE) {
-    pQueryMsg->defaultVal = (uint64_t)(pMsg);
+    pQueryMsg->fillVal = (uint64_t)(pMsg);
 
     int64_t *v = (int64_t *)pMsg;
     for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) {
@@ -5610,18 +5650,18 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) {
       continue;
     }
 
-    SColIndex *pColIndexEx = &pSqlExprMsg->colInfo;
+    SColIndex *pColIndex = &pSqlExprMsg->colInfo;
-    if (!TSDB_COL_IS_TAG(pColIndexEx->flag)) {
+    if (!TSDB_COL_IS_TAG(pColIndex->flag)) {
       for (int32_t f = 0; f < pQuery->numOfCols; ++f) {
-        if (pColIndexEx->colId == pQuery->colList[f].colId) {
+        if (pColIndex->colId == pQuery->colList[f].colId) {
-          pColIndexEx->colIndex = f;
+          pColIndex->colIndex = f;
           break;
         }
       }
     } else {
       for (int32_t f = 0; f < pQuery->numOfTags; ++f) {
-        if (pColIndexEx->colId == pQuery->tagColList[f].colId) {
+        if (pColIndex->colId == pQuery->tagColList[f].colId) {
-          pColIndexEx->colIndex = f;
+          pColIndex->colIndex = f;
           break;
         }
       }
@@ -5713,13 +5753,13 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   }
 
   if (pQuery->fillType != TSDB_FILL_NONE) {
-    pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutput);
+    pQuery->fillVal = malloc(sizeof(int64_t) * pQuery->numOfOutput);
-    if (pQuery->defaultVal == NULL) {
+    if (pQuery->fillVal == NULL) {
       goto _cleanup;
     }
 
     // the first column is the timestamp
-    memcpy(pQuery->defaultVal, (char *)pQueryMsg->defaultVal, pQuery->numOfOutput * sizeof(int64_t));
+    memcpy(pQuery->fillVal, (char *)pQueryMsg->fillVal, pQuery->numOfOutput * sizeof(int64_t));
   }
 
   // to make sure third party won't overwrite this structure
@@ -5745,7 +5785,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
       SGroupItem item = { .id = id };
       // NOTE: compare STableIdInfo with STableId
       // not a problem at present because we only use their 1st int64_t field
-      STableIdInfo* pTableId = taosArraySearch( pTableIdList, compareTableIdInfo, &id );
+      STableIdInfo* pTableId = taosArraySearch( pTableIdList, &id, compareTableIdInfo);
       if (pTableId != NULL ) {
         window.skey = pTableId->key;
       } else {
@@ -5776,7 +5816,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   return pQInfo;
 
 _cleanup:
-  tfree(pQuery->defaultVal);
+  tfree(pQuery->fillVal);
 
   if (pQuery->sdata != NULL) {
     for (int16_t col = 0; col < pQuery->numOfOutput; ++col) {
@@ -5884,8 +5924,8 @@ static void freeQInfo(SQInfo *pQInfo) {
     tfree(pQuery->pSelectExpr);
   }
 
-  if (pQuery->defaultVal != NULL) {
+  if (pQuery->fillVal != NULL) {
-    tfree(pQuery->defaultVal);
+    tfree(pQuery->fillVal);
   }
 
   // todo refactor, extract method to destroytableDataInfo
@@ -5988,8 +6028,13 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
   }
 
   pQuery->rec.total += pQuery->rec.rows;
-  qTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+  qTrace("QInfo:%p current numOfRes rows:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
 
+  if (pQuery->limit.limit > 0 && pQuery->limit.limit == pQuery->rec.total) {
+    qTrace("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit);
+    setQueryStatus(pQuery, QUERY_OVER);
+  }
+  
   return TSDB_CODE_SUCCESS;
 
   // todo if interpolation exists, the result may be dump to client by several rounds
@@ -6041,6 +6086,8 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
     isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
     
     STableIdInfo *id = taosArrayGet(pTableIdList, 0);
+    qTrace("qmsg:%p query table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
+    
     if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }

src/query/src/qast.c

@@ -773,9 +773,6 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
 
     // todo refactor:
     tstr *name = ((STableIndexElem *)pData)->pTable->name;
-    //    char* name = NULL;
-//        tsdbGetTableName(pQueryInfo->, pTable, &name);
-
     // todo speed up by using hash
     if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
       if (pQueryInfo->optr == TSDB_RELATION_IN) {
@@ -1051,7 +1048,7 @@ static void* exception_malloc(size_t size) {
   return p;
 }
 
-static char* exception_strdup(const char* str) {
+static UNUSED_FUNC char* exception_strdup(const char* str) {
   char* p = strdup(str);
   if (p == NULL) {
     THROW(TSDB_CODE_SERV_OUT_OF_MEMORY);
@@ -1154,28 +1151,33 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
     tVariant* pVal = exception_calloc(1, sizeof(tVariant));
     right->pVal = pVal;
     pVal->nType = TSDB_DATA_TYPE_ARRAY;
-    pVal->arr = taosArrayInit(2, sizeof(char*));
+    pVal->arr = taosArrayInit(2, POINTER_BYTES);
 
     const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN;
     for (const char *e = cond; *e != 0; e++) {
       if (*e == TS_PATH_DELIMITER[0]) {
         cond = e + 1;
       } else if (*e == ',') {
-        size_t len = e - cond + 1;
+        size_t len = e - cond + VARSTR_HEADER_SIZE;
-        char* p = exception_malloc( len );
+        char* p = exception_malloc(len);
-        memcpy(p, cond, len);
+        varDataSetLen(p, len - VARSTR_HEADER_SIZE);
-        p[len - 1] = 0;
+        memcpy(varDataVal(p), cond, len);
         cond += len;
         taosArrayPush(pVal->arr, &p);
       }
     }
 
     if (*cond != 0) {
-        char* p = exception_strdup( cond );
+      size_t len = strlen(cond) + VARSTR_HEADER_SIZE;
-        taosArrayPush(pVal->arr, &p);
+      
+      char* p = exception_malloc(len);
+      varDataSetLen(p, len - VARSTR_HEADER_SIZE);
+      memcpy(varDataVal(p), cond, len);
+      
+      taosArrayPush(pVal->arr, &p);
     }
 
-    taosArraySortString(pVal->arr);
+    taosArraySortString(pVal->arr, taosArrayCompareString);
   }
 
   CLEANUP_EXECUTE_TO(anchor, false);

src/query/src/qfill.c

@@ -34,7 +34,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, ch
      * here we revised the start time of day according to the local time zone,
      * but in case of DST, the start time of one day need to be dynamically decided.
      *
-     * TODO dynamically decide the start time of a day
+     * TODO dynamically decide the start time of a day, move to common module
      */
 
     // todo refactor to extract function that is available for Linux/Windows/Mac platform
@@ -116,10 +116,9 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
     return;
   }
 
-  pFillInfo->rowIdx = 0;
+  pFillInfo->rowIdx    = 0;
+  pFillInfo->endKey    = endKey;
   pFillInfo->numOfRows = numOfRows;
-  
-  pFillInfo->endKey = endKey;
 }
 
 void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput) {
@@ -131,6 +130,8 @@ void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput)
 
 void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput) {
   assert(pFillInfo->numOfRows == pInput->num);
+  int32_t t = 0;
+  
   for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
     SFillColInfo* pCol = &pFillInfo->pFillCol[i];
     
@@ -138,7 +139,7 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu
     memcpy(pFillInfo->pData[i], s, pInput->num * pCol->col.bytes);
     
     if (pCol->flag == TSDB_COL_TAG) {  // copy the tag value
-      memcpy(pFillInfo->pTags[i], pFillInfo->pData[i], pCol->col.bytes);
+      memcpy(pFillInfo->pTags[t++], pFillInfo->pData[i], pCol->col.bytes);
     }
   }
 }
@@ -170,7 +171,7 @@ static int32_t taosGetTotalNumOfFilledRes(SFillInfo* pFillInfo, const TSKEY* tsA
   }
 }
 
-int32_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows) {
+int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows) {
   int32_t numOfRes = taosGetTotalNumOfFilledRes(pFillInfo, (int64_t*) pFillInfo->pData[0], numOfRows,
                                                 pFillInfo->slidingTime, ekey);
   return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
@@ -193,7 +194,7 @@ static double linearInterpolationImpl(double v1, double v2, double k1, double k2
 int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
   switch (type) {
     case TSDB_DATA_TYPE_INT: {
-      *(int32_t*)point->val = linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, point1->key,
+      *(int32_t*)point->val = (int32_t) linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, point1->key,
                                                         point2->key, point->key);
       break;
     }
@@ -209,17 +210,17 @@ int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoi
     };
     case TSDB_DATA_TYPE_TIMESTAMP:
     case TSDB_DATA_TYPE_BIGINT: {
-      *(int64_t*)point->val = linearInterpolationImpl(*(int64_t*)point1->val, *(int64_t*)point2->val, point1->key,
+      *(int64_t*)point->val = (int64_t) linearInterpolationImpl(*(int64_t*)point1->val, *(int64_t*)point2->val, point1->key,
                                                         point2->key, point->key);
       break;
     };
     case TSDB_DATA_TYPE_SMALLINT: {
-      *(int16_t*)point->val = linearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, point1->key,
+      *(int16_t*)point->val = (int16_t) linearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, point1->key,
                                                         point2->key, point->key);
       break;
     };
     case TSDB_DATA_TYPE_TINYINT: {
-      *(int8_t*)point->val =
+      *(int8_t*) point->val = (int8_t)
           linearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, point1->key, point2->key, point->key);
       break;
     };
@@ -243,8 +244,8 @@ static void setTagsValue(SFillInfo* pColInfo, tFilePage** data, char** pTags, in
 
 static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* num, char** srcData,
                                 int64_t ts, char** pTags, bool outOfBound) {
-  char** prevValues = &pFillInfo->prevValues;
+  char* prevValues = pFillInfo->prevValues;
-  char** nextValues = &pFillInfo->nextValues;
+  char* nextValues = pFillInfo->nextValues;
 
   SPoint point1, point2, point;
 
@@ -257,16 +258,21 @@ static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t*
 
   // set the other values
   if (pFillInfo->fillType == TSDB_FILL_PREV) {
-    char* pInterpolationData = FILL_IS_ASC_FILL(pFillInfo) ? *prevValues : *nextValues;
+    char* p = FILL_IS_ASC_FILL(pFillInfo) ? prevValues : nextValues;
-    if (pInterpolationData != NULL) {
+    
+    if (p != NULL) {
       for (int32_t i = 1; i < numOfValCols; ++i) {
         SFillColInfo* pCol = &pFillInfo->pFillCol[i];
         
         char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-        if (isNull(pInterpolationData + pCol->col.offset, pCol->col.type)) {
+        if (isNull(p + pCol->col.offset, pCol->col.type)) {
-          setNull(val1, pCol->col.type, pCol->col.bytes);
+          if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
+            setVardataNull(val1, pCol->col.type);
+          } else {
+            setNull(val1, pCol->col.type, pCol->col.bytes);
+          }
         } else {
-          assignVal(val1, pInterpolationData + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+          assignVal(val1, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
         }
       }
     } else { // no prev value yet, set the value for NULL
@@ -274,14 +280,18 @@ static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t*
         SFillColInfo* pCol = &pFillInfo->pFillCol[i];
 
         char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-        setNull(val1, pCol->col.type, pCol->col.bytes);
+        if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
+          setVardataNull(val1, pCol->col.type);
+        } else {
+          setNull(val1, pCol->col.type, pCol->col.bytes);
+        }
       }
     }
 
     setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
   } else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
     // TODO : linear interpolation supports NULL value
-    if (*prevValues != NULL && !outOfBound) {
+    if (prevValues != NULL && !outOfBound) {
       for (int32_t i = 1; i < numOfValCols; ++i) {
         SFillColInfo* pCol = &pFillInfo->pFillCol[i];
         
@@ -289,14 +299,17 @@ static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t*
         int16_t bytes = pCol->col.bytes;
         
         char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-        if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
+        if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR) {
+          setVardataNull(val1, pCol->col.type);
+          continue;
+        } else if (type == TSDB_DATA_TYPE_BOOL) {
           setNull(val1, pCol->col.type, bytes);
           continue;
         }
 
-        point1 = (SPoint){.key = *(TSKEY*)(*prevValues), .val = *prevValues + pCol->col.offset};
+        point1 = (SPoint){.key = *(TSKEY*)(prevValues), .val = prevValues + pCol->col.offset};
         point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes};
-        point = (SPoint){.key = pFillInfo->start, .val = val1};
+        point  = (SPoint){.key = pFillInfo->start, .val = val1};
         taosDoLinearInterpolation(type, &point1, &point2, &point);
       }
 
@@ -307,7 +320,12 @@ static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t*
         SFillColInfo* pCol = &pFillInfo->pFillCol[i];
   
         char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-        setNull(val1, pCol->col.type, pCol->col.bytes);
+  
+        if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
+          setVardataNull(val1, pCol->col.type);
+        } else {
+          setNull(val1, pCol->col.type, pCol->col.bytes);
+        }
       }
 
       setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
@@ -318,7 +336,7 @@ static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t*
       SFillColInfo* pCol = &pFillInfo->pFillCol[i];
       
       char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-      assignVal(val1, (char*)&pCol->defaultVal.i, pCol->col.bytes, pCol->col.type);
+      assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
     }
 
     setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
@@ -338,11 +356,16 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** nextValues) {
   *nextValues = calloc(1, pFillInfo->rowSize);
   for (int i = 1; i < pFillInfo->numOfCols; i++) {
     SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-    setNull(*nextValues + pCol->col.offset, pCol->col.type, pCol->col.bytes);
+  
+    if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(*nextValues + pCol->col.offset, pCol->col.type);
+    } else {
+      setNull(*nextValues + pCol->col.offset, pCol->col.type, pCol->col.bytes);
+    }
   }
 }
 
-int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData) {
+int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData) {
   int32_t num = 0;
   pFillInfo->numOfCurrent = 0;
 
@@ -356,8 +379,8 @@ int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numO
 
   if (numOfRows == 0) {
     /*
-     * we need to rebuild whole result set
+     * These data are generated according to fill strategy, since the current timestamp is out of time window of
-     * NOTE:we need to keep the last saved data, to generated the filled data
+     * real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
      */
     while (num < outputRows) {
       doInterpoResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, pTags, true);
@@ -387,7 +410,7 @@ int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numO
         
         while (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
                 (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
-          doInterpoResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, pTags, false);
+          doInterpoResultImpl(pFillInfo, data, &num, srcData, ts, pTags, false);
         }
 
         /* output buffer is full, abort */
@@ -420,7 +443,7 @@ int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numO
               assignVal(val1, src, pCol->col.bytes, pCol->col.type);
               memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
             } else {
-              assignVal(val1, (char*) &pCol->defaultVal.i, pCol->col.bytes, pCol->col.type);
+              assignVal(val1, (char*) &pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
             }
           }
         }
@@ -450,21 +473,12 @@ int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numO
   }
 }
 
-void taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int64_t* outputRows, int32_t capacity) {
+int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
     int32_t remain = taosNumOfRemainRows(pFillInfo);  // todo use iterator?
-    
+    int32_t rows   = taosGetNumOfResultWithFill(pFillInfo, remain, pFillInfo->endKey, capacity);
-//    TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
-//                                      pQuery->slidingTimeUnit, pQuery->precision);
-//    if (QUERY_IS_ASC_QUERY(pQuery)) {
-//      assert(ekey >= pQuery->window.ekey);
-//    } else {
-//      assert(ekey <= pQuery->window.ekey);
-//    }
-    
-    int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, pFillInfo->endKey, capacity);
   
-    int32_t numOfRes = taosDoInterpoResult(pFillInfo, output, remain, rows, pFillInfo->pData);
+    int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData);
-    *outputRows = rows;
-    
     assert(numOfRes == rows);
+    
+    return numOfRes;
 }

src/query/src/tvariant.c

@@ -101,11 +101,12 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
       break;
     }
     case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length
-      pVar->nLen = len / TSDB_NCHAR_SIZE;
+      int32_t lenInwchar = len / TSDB_NCHAR_SIZE;
-      pVar->wpz = calloc(1, (pVar->nLen + 1) * TSDB_NCHAR_SIZE);
+      pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
       
-      wcsncpy(pVar->wpz, (wchar_t *)pz, pVar->nLen);
+      wcsncpy(pVar->wpz, (wchar_t *)pz, lenInwchar);
-      pVar->wpz[pVar->nLen] = 0;
+      pVar->wpz[lenInwchar] = 0;
+      pVar->nLen = len;
       
       break;
     }

src/rpc/src/rpcMain.c

@@ -867,9 +867,8 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   // underlying UDP layer does not know it is server or client
   pRecv->connType = pRecv->connType | pRpc->connType;  
 
-  if (pRecv->ip==0 && pConn) {
+  if (pRecv->ip == 0 && pConn) {
-    rpcProcessBrokenLink(pConn); 
+    rpcProcessBrokenLink(pConn);
-    rpcFreeMsg(pRecv->msg);
     return NULL;
   }
 

src/rpc/src/rpcTcp.c

@@ -147,8 +147,10 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   struct epoll_event event = { .events = EPOLLIN };
   eventfd_t fd = eventfd(1, 0);
   if (fd == -1) {
+    tError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno));
     pthread_cancel(pThreadObj->thread);
   } else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    tError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno));
     pthread_cancel(pThreadObj->thread);
   }
 
@@ -213,7 +215,6 @@ static void* taosAcceptTcpConnection(void *arg) {
       continue;
     }
 
-    tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port);
     taosKeepTcpAlive(connFd);
 
     // pick up the thread to handle this connection
@@ -227,7 +228,8 @@ static void* taosAcceptTcpConnection(void *arg) {
               inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
     } else {
       close(connFd);
-      tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno));
+      tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
+             inet_ntoa(caddr.sin_addr), caddr.sin_port);
     }  
 
     // pick up next thread for next connection
@@ -339,7 +341,9 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
     recvInfo.chandle = NULL;
     recvInfo.connType = RPC_CONN_TCP;
     (*(pThreadObj->processData))(&recvInfo);
-  } 
+  } else {
+    taosFreeFdObj(pFdObj);
+  }
 }
 
 #define maxEvents 10
@@ -350,7 +354,7 @@ static void *taosProcessTcpData(void *param) {
   struct epoll_event events[maxEvents];
   SRecvInfo          recvInfo;
   SRpcHead           rpcHead;
-
+ 
   while (1) {
     int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
     if (pThreadObj->stop) {
@@ -464,7 +468,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
 
   pFdObj->signature = NULL;
   epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
-  close(pFdObj->fd);
+  taosCloseSocket(pFdObj->fd);
 
   pThreadObj->numOfFds--;
 

src/rpc/test/rserver.c

@@ -127,6 +127,8 @@ int main(int argc, char *argv[]) {
   SRpcInit rpcInit;
   char     dataName[20] = "server.data";
 
+  taosBlockSIGPIPE();
+
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort    = 7000;
   rpcInit.label        = "SER";

src/tsdb/inc/tsdbMain.h

@@ -74,7 +74,7 @@ void       tsdbCloseMetaFile(SMetaFile *mfh);
 typedef struct {
   TSKEY   keyFirst;
   TSKEY   keyLast;
-  int32_t numOfPoints;
+  int32_t numOfRows;
   void *  pData;
 } SMemTable;
 
@@ -173,7 +173,7 @@ typedef struct {
 typedef struct {
   TSKEY   keyFirst;
   TSKEY   keyLast;
-  int64_t numOfPoints;
+  int64_t numOfRows;
   SList * list;
 } SCacheMem;
 
@@ -294,7 +294,7 @@ typedef struct {
   int64_t last : 1;          // If the block in data file or last file
   int64_t offset : 63;       // Offset of data block or sub-block index depending on numOfSubBlocks
   int32_t algorithm : 8;     // Compression algorithm
-  int32_t numOfPoints : 24;  // Number of total points
+  int32_t numOfRows : 24;    // Number of total points
   int32_t sversion;          // Schema version
   int32_t len;               // Data block length or nothing
   int16_t numOfSubBlocks;    // Number of sub-blocks;
@@ -495,11 +495,12 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
 int  tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
 
 // --------- For read operations
-int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
+int  tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
-int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
+int  tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
-int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
+int  tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
-int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
+int  tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
-int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
+int  tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
+void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols);
 
 // --------- For write operations
 int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols);

src/tsdb/src/tsdbCache.c

@@ -82,7 +82,7 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) {
   memset(ptr, 0, bytes);
   if (key < pCache->mem->keyFirst) pCache->mem->keyFirst = key;
   if (key > pCache->mem->keyLast) pCache->mem->keyLast = key;
-  pCache->mem->numOfPoints++;
+  pCache->mem->numOfRows++;
 
   return ptr;
 }
@@ -127,7 +127,7 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache) {
     if (pCache->mem == NULL) return -1;
     pCache->mem->keyFirst = INT64_MAX;
     pCache->mem->keyLast = 0;
-    pCache->mem->numOfPoints = 0;
+    pCache->mem->numOfRows = 0;
     pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *));
   }
 
@@ -162,6 +162,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
     pCache->totalCacheBlocks = totalBlocks;
     tsdbAdjustCacheBlocks(pCache);
   }
+  pRepo->config.totalBlocks = totalBlocks;
 
   tsdbUnLockRepo((TsdbRepoT *)pRepo);
   tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);

src/tsdb/src/tsdbFile.c

@@ -233,10 +233,10 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
 //   SCompBlock *pBlock = pStartBlock;
 //   for (int i = 0; i < numOfBlocks; i++) {
 //     if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1;
-//     pCols->numOfPoints += (pCompData->cols[0].len / 8);
+//     pCols->numOfRows += (pCompData->cols[0].len / 8);
 //     for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) {
 //       SCompCol *pCompCol = &(pCompData->cols[iCol]);
-//       // pCols->numOfPoints += pBlock->numOfPoints;
+//       // pCols->numOfRows += pBlock->numOfRows;
 //       int k = 0;
 //       for (; k < pCols->numOfCols; k++) {
 //         if (pCompCol->colId == pCols->cols[k].colId) break;

src/tsdb/src/tsdbMain.c

@@ -37,7 +37,8 @@ static TSKEY   tsdbNextIterKey(SSkipListIterator *pIter);
 static int     tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey);
 static void    tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
 static void    tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
-static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
+static void    tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables);
+static int32_t tsdbSaveConfig(STsdbRepo *pRepo);
 
 #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
 #define TSDB_GET_TABLE_BY_NAME(pRepo, name)
@@ -319,10 +320,25 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
   ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock);
   ASSERT(pRCfg->precision == pCfg->precision);
 
-  if (pRCfg->compression != pCfg->compression) tsdbAlterCompression(pRepo, pCfg->compression);
+  bool configChanged = false;
-  if (pRCfg->keep != pCfg->keep) tsdbAlterKeep(pRepo, pCfg->keep);
+  if (pRCfg->compression != pCfg->compression) {
-  if (pRCfg->totalBlocks != pCfg->totalBlocks) tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
+    configChanged = true;
-  if (pRCfg->maxTables != pCfg->maxTables) tsdbAlterMaxTables(pRepo, pCfg->maxTables);
+    tsdbAlterCompression(pRepo, pCfg->compression);
+  }
+  if (pRCfg->keep != pCfg->keep) {
+    configChanged = true;
+    tsdbAlterKeep(pRepo, pCfg->keep);
+  }
+  if (pRCfg->totalBlocks != pCfg->totalBlocks) {
+    configChanged = true;
+    tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
+  }
+  if (pRCfg->maxTables != pCfg->maxTables) {
+    configChanged = true;
+    tsdbAlterMaxTables(pRepo, pCfg->maxTables);
+  }
+
+  if (configChanged) tsdbSaveConfig(pRepo);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -814,7 +830,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   tSkipListNewNodeInfo(pTable->mem->pData, &level, &headSize);
 
   TSKEY key = dataRowKey(row);
-  // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfPoints);
+  // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfRows);
   
   // Copy row into the memory
   SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key);
@@ -838,7 +854,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key;
   if (key > pTable->lastKey) pTable->lastKey = key;
   
-  pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData);
+  pTable->mem->numOfRows = tSkipListGetSize(pTable->mem->pData);
 
   tsdbTrace("vgId:%d, tid:%d, uid:%" PRId64 ", table:%s a row is inserted to table! key:%" PRId64, pRepo->config.tsdbId,
             pTable->tableId.tid, pTable->tableId.uid, varDataVal(pTable->name), dataRowKey(row));
@@ -1047,7 +1063,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
     while (true) {
       int rowsRead = tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pDataCols);
       assert(rowsRead >= 0);
-      if (pDataCols->numOfPoints == 0) break;
+      if (pDataCols->numOfRows == 0) break;
       nLoop++;
 
       ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey);
@@ -1056,13 +1072,13 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
       int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols);
       ASSERT(rowsWritten != 0);
       if (rowsWritten < 0) goto _err;
-      ASSERT(rowsWritten <= pDataCols->numOfPoints);
+      ASSERT(rowsWritten <= pDataCols->numOfRows);
 
       tdPopDataColsPoints(pDataCols, rowsWritten);
-      maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfPoints;
+      maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows;
     }
 
-    ASSERT(pDataCols->numOfPoints == 0);
+    ASSERT(pDataCols->numOfRows == 0);
 
     // Move the last block to the new .l file if neccessary
     if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
@@ -1134,8 +1150,10 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
 
   int maxFiles = keep / pCfg->maxTables + 3;
   if (pRepo->config.keep > keep) {
+    pRepo->config.keep = keep;
     pRepo->tsdbFileH->maxFGroups = maxFiles;
   } else {
+    pRepo->config.keep = keep;
     pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup));
     if (pRepo->tsdbFileH->fGroup == NULL) {
       // TODO: deal with the error
@@ -1155,6 +1173,8 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
 
   pMeta->maxTables = maxTables;
   pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *));
+  memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables-oldMaxTables));
+  pRepo->config.maxTables = maxTables;
 
   tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
 }
@@ -1176,7 +1196,7 @@ uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *
     // Map index to the file name
     int fid = (*index) / 3;
 
-    if (fid > pFileH->numOfFGroups) {
+    if (fid >= pFileH->numOfFGroups) {
       // return meta data file
       if ((*index) % 3 > 0) { // it is finished
         tfree(spath);

src/tsdb/src/tsdbMeta.c

@@ -129,7 +129,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
   if (pTable->type == TSDB_SUPER_TABLE) {
     STColumn* pColSchema = schemaColAt(pTable->tagSchema, 0);
     pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes,
-                                    1, 0, 0, getTagIndexKey);
+                                    1, 0, 1, getTagIndexKey);
   }
 
   tsdbAddTableToMeta(pMeta, pTable, false);
@@ -292,6 +292,13 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
 
   if (tsdbCheckTableCfg(pCfg) < 0) return -1;
 
+  STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid);
+  if (pTable != NULL) {
+    tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name),
+              pTable->tableId.tid, pTable->tableId.uid);
+    return TSDB_CODE_TABLE_ALREADY_EXIST;
+  }
+
   STable *super = NULL;
   int newSuper = 0;
 
@@ -319,7 +326,7 @@ int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
       // index the first tag column
       STColumn* pColSchema = schemaColAt(super->tagSchema, 0);
       super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes,
-          1, 0, 0, getTagIndexKey);  // Allow duplicate key, no lock
+          1, 0, 1, getTagIndexKey);  // Allow duplicate key, no lock
 
       if (super->pIndex == NULL) {
         tdFreeSchema(super->schema);
@@ -443,6 +450,7 @@ static int tsdbFreeTable(STable *pTable) {
 
   // Free content
   if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) {
+    tdFreeSchema(pTable->tagSchema);
     tSkipListDestroy(pTable->pIndex);
   }
 

src/tsdb/src/tsdbRWHelper.c

@@ -307,7 +307,7 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
  */
 int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
   ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
-  ASSERT(pDataCols->numOfPoints > 0);
+  ASSERT(pDataCols->numOfRows > 0);
 
   SCompBlock compBlock;
   int        rowsToWrite = 0;
@@ -322,7 +322,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
 
   if (pIdx->offset == 0 || (!pIdx->hasLast && keyFirst > pIdx->maxKey)) {  // Just append as a super block
     ASSERT(pHelper->hasOldLastBlock == false);
-    rowsToWrite = pDataCols->numOfPoints;
+    rowsToWrite = pDataCols->numOfRows;
     SFile *pWFile = NULL;
     bool   isLast = false;
 
@@ -380,10 +380,10 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
 
     if (pCompBlock->numOfSubBlocks > 1) {
       if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1;
-      ASSERT(pHelper->pDataCols[0]->numOfPoints > 0 &&
+      ASSERT(pHelper->pDataCols[0]->numOfRows > 0 &&
-             pHelper->pDataCols[0]->numOfPoints < pHelper->config.minRowsPerFileBlock);
+             pHelper->pDataCols[0]->numOfRows < pHelper->config.minRowsPerFileBlock);
       if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0],
-                               pHelper->pDataCols[0]->numOfPoints, &compBlock, true, true) < 0)
+                               pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0)
         return -1;
 
       if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
@@ -409,7 +409,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
     if (pIdx->offset > 0) {
       pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
       if (pIdx->offset < 0) return -1;
-      ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx));
+      ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
       if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1;
     }
@@ -489,6 +489,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
       }
 
       ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM)));
+      if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
     }
 
   }
@@ -542,6 +543,34 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
   return 0;
 }
 
+void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) {
+  SCompData *pCompData = pHelper->pCompData;
+
+  for (int i = 0, j = 0; i < numOfCols;) {
+    if (j >= pCompData->numOfCols) {
+      pStatis[i].numOfNull = -1;
+      i++;
+      continue;
+    }
+
+    if (pStatis[i].colId == pCompData->cols[j].colId) {
+      pStatis[i].sum = pCompData->cols[j].sum;
+      pStatis[i].max = pCompData->cols[j].max;
+      pStatis[i].min = pCompData->cols[j].min;
+      pStatis[i].maxIndex = pCompData->cols[j].maxIndex;
+      pStatis[i].minIndex = pCompData->cols[j].minIndex;
+      pStatis[i].numOfNull = pCompData->cols[j].numOfNull;
+      i++;
+      j++;
+    } else if (pStatis[i].colId < pCompData->cols[j].colId) {
+      pStatis[i].numOfNull = -1;
+      i++;
+    } else {
+      j++;
+    }
+  }
+}
+
 static int comparColIdCompCol(const void *arg1, const void *arg2) {
   return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId;
 }
@@ -596,13 +625,13 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx,
   for (int i = 1; i < numOfSubBlocks; i++) {
     pStartBlock++;
     if (tsdbLoadSingleBlockDataCols(pHelper, pStartBlock, colIds, numOfColIds, pHelper->pDataCols[1]) < 0) return -1;
-    tdMergeDataCols(pDataCols, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints);
+    tdMergeDataCols(pDataCols, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows);
   }
 
   return 0;
 }
 
-static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfPoints,
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows,
                                         int maxPoints, char *buffer, int bufferSize) {
   // Verify by checksum
   if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1;
@@ -611,16 +640,16 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32
   if (comp) {
     // // Need to decompress
     pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))(
-        content, len - sizeof(TSCKSUM), numOfPoints, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
+        content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
     if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
-      dataColSetOffset(pDataCol, numOfPoints);
+      dataColSetOffset(pDataCol, numOfRows);
     }
   } else {
     // No need to decompress, just memcpy it
     pDataCol->len = len - sizeof(TSCKSUM);
     memcpy(pDataCol->pData, content, pDataCol->len);
     if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
-      dataColSetOffset(pDataCol, numOfPoints);
+      dataColSetOffset(pDataCol, numOfRows);
     }
   }
   return 0;
@@ -644,7 +673,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
   int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
   if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err;
 
-  pDataCols->numOfPoints = pCompBlock->numOfPoints;
+  pDataCols->numOfRows = pCompBlock->numOfRows;
 
   // Recover the data
   int ccol = 0;
@@ -653,7 +682,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
     SDataCol *pDataCol = &(pDataCols->cols[dcol]);
     if (ccol >= pCompData->numOfCols) {
       // Set current column as NULL and forward
-      dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
+      dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
       dcol++;
       continue;
     }
@@ -662,15 +691,15 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
 
     if (pCompCol->colId == pDataCol->colId) {
       if (pCompBlock->algorithm == TWO_STAGE_COMP) {
-        int zsize = pDataCol->bytes * pCompBlock->numOfPoints + COMP_OVERFLOW_BYTES;
+        int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES;
         if (pCompCol->type == TSDB_DATA_TYPE_BINARY || pCompCol->type == TSDB_DATA_TYPE_NCHAR) {
-          zsize += (sizeof(VarDataLenT) * pCompBlock->numOfPoints);
+          zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows);
         }
         pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize);
         if (pHelper->compBuffer == NULL) goto _err;
       }
       if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len,
-                                       pCompBlock->algorithm, pCompBlock->numOfPoints, pDataCols->maxPoints,
+                                       pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints,
                                        pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0)
         goto _err;
       dcol++;
@@ -679,7 +708,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
       ccol++;
     } else {
       // Set current column as NULL and forward
-      dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
+      dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
       dcol++;
     }
   }
@@ -703,7 +732,7 @@ int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *tar
     tdResetDataCols(pHelper->pDataCols[1]);
     pCompBlock++;
     if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[1]) < 0) goto _err;
-    if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints) < 0) goto _err;
+    if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err;
   }
 
   // if (target) TODO
@@ -724,7 +753,7 @@ static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
 
 static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock,
                                 bool isLast, bool isSuperBlock) {
-  ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints &&
+  ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows &&
          rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
 
   SCompData *pCompData = (SCompData *)(pHelper->pBuffer);
@@ -747,7 +776,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
 
     pCompCol->colId = pDataCol->colId;
     pCompCol->type = pDataCol->type;
-    if (tDataTypeDesc[pDataCol->type].getStatisFunc) {
+    if (tDataTypeDesc[pDataCol->type].getStatisFunc && ncol != 0) {
       (*tDataTypeDesc[pDataCol->type].getStatisFunc)(
           (TSKEY *)(pDataCols->cols[0].pData), pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max),
           &(pCompCol->sum), &(pCompCol->minIndex), &(pCompCol->maxIndex), &(pCompCol->numOfNull));
@@ -811,7 +840,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
   pCompBlock->last = isLast;
   pCompBlock->offset = offset;
   pCompBlock->algorithm = pHelper->config.compress;
-  pCompBlock->numOfPoints = rowsToWrite;
+  pCompBlock->numOfRows = rowsToWrite;
   pCompBlock->sversion = pHelper->tableInfo.sversion;
   pCompBlock->len = (int32_t)lsize;
   pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0;
@@ -848,7 +877,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
   int        rowsWritten = 0;
   SCompBlock compBlock = {0};
 
-  ASSERT(pDataCols->numOfPoints > 0);
+  ASSERT(pDataCols->numOfRows > 0);
   TSKEY keyFirst = dataColsKeyFirst(pDataCols);
 
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
@@ -860,32 +889,32 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
   // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
 
   if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
-    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks-1);
+    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks-1);
     int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5;  // TODO: make a interface
 
-    rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfPoints), pDataCols->numOfPoints);
+    rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfRows), pDataCols->numOfRows);
     if ((blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) &&
-        (blockAtIdx(pHelper, blkIdx)->numOfPoints + rowsWritten < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd) > 0) {
+        (blockAtIdx(pHelper, blkIdx)->numOfRows + rowsWritten < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd) > 0) {
       if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, rowsWritten, &compBlock, true, false) < 0)
         goto _err;
       if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err;
     } else {
       // Load
       if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err;
-      ASSERT(pHelper->pDataCols[0]->numOfPoints == blockAtIdx(pHelper, blkIdx)->numOfPoints);
+      ASSERT(pHelper->pDataCols[0]->numOfRows == blockAtIdx(pHelper, blkIdx)->numOfRows);
       // Merge
       if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err;
       // Write
       SFile *pWFile = NULL;
       bool isLast = false;
-      if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.minRowsPerFileBlock) {
+      if (pHelper->pDataCols[0]->numOfRows >= pHelper->config.minRowsPerFileBlock) {
         pWFile = &(pHelper->files.dataF);
       } else {
         isLast = true;
         pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF);
       }
       if (tsdbWriteBlockToFile(pHelper, pWFile, pHelper->pDataCols[0],
-                               pHelper->pDataCols[0]->numOfPoints, &compBlock, isLast, true) < 0)
+                               pHelper->pDataCols[0]->numOfRows, &compBlock, isLast, true) < 0)
         goto _err;
       if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err;
     }
@@ -902,7 +931,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
     // rows1: number of rows must merge in this block
     int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast);
     // rows2: max nuber of rows the block can have more
-    int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfPoints;
+    int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfRows;
     // rows3: number of rows between this block and the next block
     int rows3 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, keyLimit);
 
@@ -910,7 +939,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
 
     if ((rows2 >= rows1) &&
         (( blockAtIdx(pHelper, blkIdx)->last) ||
-         ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd < 0)))) {
+         ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd < 0)))) {
       rowsWritten = rows1;
       bool   isLast = false;
       SFile *pFile = NULL;
@@ -936,11 +965,11 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
       int round = 0;
       // tdResetDataCols(pHelper->pDataCols[1]);
       while (true) {
-        if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) break;
+        if (iter1 >= pHelper->pDataCols[0]->numOfRows && iter2 >= rows3) break;
         tdMergeTwoDataCols(pHelper->pDataCols[1], pHelper->pDataCols[0], &iter1, pDataCols, &iter2, pHelper->config.maxRowsPerFileBlock * 4 / 5);
-        ASSERT(pHelper->pDataCols[1]->numOfPoints > 0);
+        ASSERT(pHelper->pDataCols[1]->numOfRows > 0);
         if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pHelper->pDataCols[1],
-                                 pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0)
+                                 pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0)
           goto _err;
         if (round == 0) {
           tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx);
@@ -951,17 +980,17 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
         blkIdx++;
         // TODO: the blkIdx here is not correct
 
-        // if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) {
+        // if (iter1 >= pHelper->pDataCols[0]->numOfRows && iter2 >= rows3) {
-        //   if (pHelper->pDataCols[1]->numOfPoints > 0) {
+        //   if (pHelper->pDataCols[1]->numOfRows > 0) {
         //     if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1],
-        //                              pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0)
+        //                              pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0)
         //       goto _err;
         //     // TODO: the blkIdx here is not correct
-        //     tsdbAddSubBlock(pHelper, &compBlock, blkIdx, pHelper->pDataCols[1]->numOfPoints);
+        //     tsdbAddSubBlock(pHelper, &compBlock, blkIdx, pHelper->pDataCols[1]->numOfRows);
         //   }
         // }
 
-        // TSKEY key1 = iter1 >= pHelper->pDataCols[0]->numOfPoints
+        // TSKEY key1 = iter1 >= pHelper->pDataCols[0]->numOfRows
         //                  ? INT64_MAX
         //                  : ((int64_t *)(pHelper->pDataCols[0]->cols[0].pData))[iter1];
         // TSKEY key2 = iter2 >= rowsWritten ? INT64_MAX : ((int64_t *)(pDataCols->cols[0].pData))[iter2];
@@ -969,11 +998,11 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
         // if (key1 < key2) {
         //   for (int i = 0; i < pDataCols->numOfCols; i++) {
         //     SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i;
-        //     memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints),
+        //     memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfRows),
         //            ((char *)pHelper->pDataCols[0]->cols[i].pData + TYPE_BYTES[pDataCol->type] * iter1),
         //            TYPE_BYTES[pDataCol->type]);
         //   }
-        //   pHelper->pDataCols[1]->numOfPoints++;
+        //   pHelper->pDataCols[1]->numOfRows++;
         //   iter1++;
         // } else if (key1 == key2) {
         //   // TODO: think about duplicate key cases
@@ -981,17 +1010,17 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
         // } else {
         //   for (int i = 0; i < pDataCols->numOfCols; i++) {
         //     SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i;
-        //     memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints),
+        //     memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfRows),
         //            ((char *)pDataCols->cols[i].pData +
         //             TYPE_BYTES[pDataCol->type] * iter2),
         //            TYPE_BYTES[pDataCol->type]);
         //   }
-        //   pHelper->pDataCols[1]->numOfPoints++;
+        //   pHelper->pDataCols[1]->numOfRows++;
         //   iter2++;
         // }
 
-        // if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.maxRowsPerFileBlock * 4 / 5) {
+        // if (pHelper->pDataCols[0]->numOfRows >= pHelper->config.maxRowsPerFileBlock * 4 / 5) {
-        //   if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) goto _err;
+        //   if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0) goto _err;
         //   // TODO: blkIdx here is not correct, fix it
         //   tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx);
 
@@ -1104,7 +1133,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
     pSCompBlock->numOfSubBlocks++;
     ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
     pSCompBlock->len += sizeof(SCompBlock);
-    pSCompBlock->numOfPoints += rowsAdded;
+    pSCompBlock->numOfRows += rowsAdded;
     pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst);
     pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast);
     pIdx->len += sizeof(SCompBlock);
@@ -1135,7 +1164,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
     ((SCompBlock *)ptr)[1] = *pCompBlock;
 
     pSCompBlock->numOfSubBlocks = 2;
-    pSCompBlock->numOfPoints += rowsAdded;
+    pSCompBlock->numOfRows += rowsAdded;
     pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
     pSCompBlock->len = sizeof(SCompBlock) * 2;
     pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst);
@@ -1190,7 +1219,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
 
 // Get the number of rows in range [minKey, maxKey]
 static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey) {
-  if (pDataCols->numOfPoints == 0) return 0;
+  if (pDataCols->numOfRows == 0) return 0;
 
   ASSERT(minKey <= maxKey);
   TSKEY keyFirst = dataColsKeyFirst(pDataCols);
@@ -1199,11 +1228,11 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey)
 
   if (minKey > keyLast || maxKey < keyFirst) return 0;
 
-  void *ptr1 = taosbsearch((void *)&minKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY),
+  void *ptr1 = taosbsearch((void *)&minKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfRows, sizeof(TSKEY),
                            compTSKEY, TD_GE);
   ASSERT(ptr1 != NULL);
 
-  void *ptr2 = taosbsearch((void *)&maxKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY),
+  void *ptr2 = taosbsearch((void *)&maxKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfRows, sizeof(TSKEY),
                            compTSKEY, TD_LE);
   ASSERT(ptr2 != NULL);
 

src/util/inc/tarray.h

@@ -129,7 +129,7 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*));
  * sort string array
  * @param pArray
  */
-void taosArraySortString(SArray* pArray);
+void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
 
 /**
  * search the array
@@ -137,14 +137,14 @@ void taosArraySortString(SArray* pArray);
  * @param compar
  * @param key
  */
-void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key);
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn);
 
 /**
  * search the array
  * @param pArray
  * @param key
  */
-char* taosArraySearchString(const SArray* pArray, const char* key);
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn);
 
 #ifdef __cplusplus
 }

src/util/inc/tcompare.h

@@ -34,16 +34,18 @@ typedef struct SPatternCompareInfo {
   char matchOne;  // symbol for match one wildcard, default: '_'
 } SPatternCompareInfo;
 
-int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo);
+int patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo);
+int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size);
+int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
 
 __compar_fn_t getKeyComparFunc(int32_t keyType);
 
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 
+int32_t taosArrayCompareString(const void* a, const void* b);
+
 #ifdef __cplusplus
 }
 #endif

src/util/inc/tsocket.h

@@ -31,7 +31,6 @@ int  taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
 int  taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
 int  taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
 int  taosKeepTcpAlive(int sockFd);
-void taosCloseTcpSocket(int sockFd);
 
 int      taosGetFqdn(char *);
 uint32_t taosGetIpFromFqdn(const char *);

src/util/src/tarray.c

@@ -197,30 +197,23 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
   qsort(pArray->pData, pArray->size, pArray->elemSize, compar);
 }
 
-void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key) {
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn) {
-  assert(pArray != NULL);
+  assert(pArray != NULL && comparFn != NULL);
-  assert(compar != NULL);
   assert(key != NULL);
 
-  return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar);
+  return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
 }
 
-static int taosArrayCompareString(const void* a, const void* b) {
+void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
-  const char* x = *(const char**)a;
-  const char* y = *(const char**)b;
-  return strcmp(x, y);
-}
-
-void taosArraySortString(SArray* pArray) {
   assert(pArray != NULL);
-  qsort(pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString);
+  qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn);
 }
 
-char* taosArraySearchString(const SArray* pArray, const char* key) {
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn) {
   assert(pArray != NULL);
   assert(key != NULL);
 
-  void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString);
+  void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
   if (p == NULL) {
     return NULL;
   }

src/util/src/tcompare.c

@@ -216,30 +216,37 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
   return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
 }
 
-static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
+static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
   
-  const char* pattern = pRight;
+  char pattern[128] = {0};
-  const char* str = pLeft;
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-  
+  assert(varDataLen(pRight) < 128);
-  int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo);
   
+  int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo);
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
+int32_t taosArrayCompareString(const void* a, const void* b) {
+  const char* x = *(const char**)a;
+  const char* y = *(const char**)b;
+  
+  return compareLenPrefixedStr(x, y);
+}
+
 static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
   const SArray* arr = (const SArray*) pRight;
-  return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1;
+  return taosArraySearchString(arr, pLeft, taosArrayCompareString) == NULL ? 0 : 1;
 }
 
-static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
+static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
   
-  const wchar_t* pattern = pRight;
+  wchar_t pattern[128] = {0};
-  const wchar_t* str = pLeft;
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight)/TSDB_NCHAR_SIZE);
-  
+  assert(varDataLen(pRight) < 128);
-  int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo);
   
+  int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
@@ -248,25 +255,25 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
   
   switch (type) {
     case TSDB_DATA_TYPE_SMALLINT: {
-      comparFn = compareInt16Val; break;
+      comparFn = compareInt16Val;  break;
     }
     
     case TSDB_DATA_TYPE_INT: {
-      comparFn = compareInt32Val; break;
+      comparFn = compareInt32Val;  break;
     }
     
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP: {
-      comparFn = compareInt64Val; break;
+      comparFn = compareInt64Val;  break;
     }
 
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:{
-      comparFn = compareInt8Val; break;
+      comparFn = compareInt8Val;   break;
     }
 
     case TSDB_DATA_TYPE_FLOAT: {
-      comparFn = compareFloatVal; break;
+      comparFn = compareFloatVal;  break;
     }
     
     case TSDB_DATA_TYPE_DOUBLE: {

src/util/src/tqueue.c

@@ -65,6 +65,7 @@ taos_queue taosOpenQueue() {
 }
 
 void taosCloseQueue(taos_queue param) {
+  if (param == NULL) return;
   STaosQueue *queue = (STaosQueue *)param;
   STaosQnode *pTemp;
   STaosQnode *pNode = queue->head;  
@@ -224,6 +225,7 @@ taos_qset taosOpenQset() {
 }
 
 void taosCloseQset(taos_qset param) {
+  if (param == NULL) return;
   STaosQset *qset = (STaosQset *)param;
   pthread_mutex_destroy(&qset->mutex);
   tsem_destroy(&qset->sem);

src/util/src/tsocket.c

@@ -305,20 +305,11 @@ int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientI
     sockFd = -1;
   }
 
+  // taosKeepTcpAlive(sockFd);
+
   return sockFd;
 }
 
-void taosCloseTcpSocket(int sockFd) {
-  struct linger linger;
-  linger.l_onoff = 1;
-  linger.l_linger = 0;
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) {
-    uError("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno));
-  }
-
-  taosCloseSocket(sockFd);
-}
-
 int taosKeepTcpAlive(int sockFd) {
   int alive = 1;
   if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) {
@@ -355,6 +346,15 @@ int taosKeepTcpAlive(int sockFd) {
     return -1;
   }
 
+  struct linger linger = {0};
+  linger.l_onoff = 1;
+  //linger.l_linger = 0;
+  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) {
+    uError("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno));
+    close(sockFd);
+    return -1;
+  }
+
   return 0;
 }
 

src/util/src/ttime.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _BSD_SOURCE
 #define _XOPEN_SOURCE
 #define _DEFAULT_SOURCE
 

src/util/src/ttimer.c

@@ -405,19 +405,19 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) {
     tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param);
     return reusable;
   }
-  
+
   if (state != TIMER_STATE_EXPIRED) {
     // timer already stopped or cancelled, has nothing to do in this case
     return false;
   }
-  
+
   if (timer->executedBy == taosGetPthreadId()) {
     // taosTmrReset is called in the timer callback, should do nothing in this
     // case to avoid dead lock. note taosTmrReset must be the last statement
     // of the callback funtion, will be a bug otherwise.
     return false;
   }
-  
+
   // timer callback is executing in another thread, we SHOULD wait it to stop,
   // BUT this may result in dead lock if current thread are holding a lock which
   // the timer callback need to acquire. so, we HAVE TO return directly.
@@ -501,6 +501,7 @@ static void taosTmrModuleInit(void) {
     tmr_ctrl_t* ctrl = tmrCtrls + i;
     ctrl->next = ctrl + 1;
   }
+  (tmrCtrls + taosMaxTmrCtrl - 1)->next = NULL;
   unusedTmrCtrl = tmrCtrls;
 
   pthread_mutex_init(&tmrCtrlMutex, NULL);
@@ -574,12 +575,12 @@ void taosTmrCleanUp(void* handle) {
 
   if (numOfTmrCtrl <=0) {
     taosUninitTimer();
-    
+
     taosCleanUpScheduler(tmrQhandle);
 
     for (int i = 0; i < tListLen(wheels); i++) {
       time_wheel_t* wheel = wheels + i;
-      pthread_mutex_destroy(&wheel->mutex); 
+      pthread_mutex_destroy(&wheel->mutex);
       free(wheel->slots);
     }
 

src/vnode/src/vnodeMain.c

@@ -35,7 +35,7 @@ static void     vnodeCleanUp(SVnodeObj *pVnode);
 static int32_t  vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
 static int32_t  vnodeReadCfg(SVnodeObj *pVnode);
 static int32_t  vnodeSaveVersion(SVnodeObj *pVnode);
-static bool     vnodeReadVersion(SVnodeObj *pVnode);
+static int32_t  vnodeReadVersion(SVnodeObj *pVnode);
 static int      vnodeProcessTsdbStatus(void *arg, int status);
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion);
 static int      vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
@@ -46,9 +46,9 @@ static pthread_once_t  vnodeModuleInit = PTHREAD_ONCE_INIT;
 
 #ifndef _SYNC
 tsync_h syncStart(const SSyncInfo *info) { return NULL; }
-int     syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
+int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
 void    syncStop(tsync_h shandle) {}
-int     syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
+int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
 int     syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
 void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
 #endif
@@ -73,13 +73,18 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
     return TSDB_CODE_SUCCESS;
   }
 
+  mkdir(tsVnodeDir, 0755);
+
   char rootDir[TSDB_FILENAME_LEN] = {0};
   sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
   if (mkdir(rootDir, 0755) != 0) {
+    vPrint("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir);
     if (errno == EACCES) {
       return TSDB_CODE_NO_DISK_PERMISSIONS;
     } else if (errno == ENOSPC) {
       return TSDB_CODE_SERV_NO_DISKSPACE;
+    } else if (errno == ENOENT) {
+      return TSDB_CODE_NOT_SUCH_FILE_OR_DIR;
     } else if (errno == EEXIST) {
     } else {
       return TSDB_CODE_VG_INIT_FAILED;
@@ -143,35 +148,20 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
   pVnode->status = TAOS_VN_STATUS_UPDATING;
 
   int32_t code = vnodeSaveCfg(pVnodeCfg);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS) return code; 
-    vError("vgId:%d, failed to save vnode cfg, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code));
-    return code;
-  }
 
   code = vnodeReadCfg(pVnode);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS) return code; 
-    vError("vgId:%d, failed to read cfg file", pVnode->vgId);
-    taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
-    return code;
-  }
 
   code = syncReconfig(pVnode->sync, &pVnode->syncCfg);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS) return code; 
-    vTrace("vgId:%d, failed to alter vnode, canot reconfig sync, result:%s", pVnode->vgId,
-           tstrerror(code));
-    return code;
-  }
 
   code = tsdbConfigRepo(pVnode->tsdb, &pVnode->tsdbCfg);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS) return code; 
-    vTrace("vgId:%d, failed to alter vnode, canot reconfig tsdb, result:%s", pVnode->vgId,
-           tstrerror(code));
-    return code;
-  }
 
   pVnode->status = TAOS_VN_STATUS_READY;
-
   vTrace("vgId:%d, vnode is altered", pVnode->vgId);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -180,26 +170,40 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   pthread_once(&vnodeModuleInit, vnodeInit);
 
   SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1);
+  if (pVnode == NULL) {
+    vError("vgId:%d, failed to open vnode since no enough memory", vnode);
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  atomic_add_fetch_32(&tsOpennedVnodes, 1);
+  atomic_add_fetch_32(&pVnode->refCount, 1);
+
   pVnode->vgId     = vnode;
   pVnode->status   = TAOS_VN_STATUS_INIT;
-  pVnode->refCount = 1;
   pVnode->version  = 0;  
   pVnode->tsdbCfg.tsdbId = pVnode->vgId;
   pVnode->rootDir = strdup(rootDir);
-  taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
 
   int32_t code = vnodeReadCfg(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
-    vError("vgId:%d, failed to read cfg file", pVnode->vgId);
+    vnodeCleanUp(pVnode);
-    taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
+    return code;
+  } 
+
+  code = vnodeReadVersion(pVnode);
+  if (code != TSDB_CODE_SUCCESS) {
+    vnodeCleanUp(pVnode);
     return code;
   }
 
-  vnodeReadVersion(pVnode);
   pVnode->fversion = pVnode->version;
   
   pVnode->wqueue = dnodeAllocateWqueue(pVnode);
   pVnode->rqueue = dnodeAllocateRqueue(pVnode);
+  if (pVnode->wqueue == NULL || pVnode->rqueue == NULL) {
+    vnodeCleanUp(pVnode);
+    return terrno;
+  }
 
   SCqCfg cqCfg = {0};
   sprintf(cqCfg.user, "root");
@@ -207,22 +211,29 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   cqCfg.vgId = vnode;
   cqCfg.cqWrite = vnodeWriteToQueue;
   pVnode->cq = cqOpen(pVnode, &cqCfg);
+  if (pVnode->cq == NULL) {
+    vnodeCleanUp(pVnode);
+    return terrno;
+  }
 
   STsdbAppH appH = {0};
   appH.appH = (void *)pVnode;
   appH.notifyStatus = vnodeProcessTsdbStatus;
   appH.cqH = pVnode->cq;
-
   sprintf(temp, "%s/tsdb", rootDir);
   pVnode->tsdb = tsdbOpenRepo(temp, &appH);
   if (pVnode->tsdb == NULL) {
-    vError("vgId:%d, failed to open tsdb at %s(%s)", pVnode->vgId, temp, tstrerror(terrno));
+    vnodeCleanUp(pVnode);
-    taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
     return terrno;
   }
 
   sprintf(temp, "%s/wal", rootDir);
   pVnode->wal = walOpen(temp, &pVnode->walCfg);
+  if (pVnode->wal == NULL) { 
+    vnodeCleanUp(pVnode);
+    return terrno;
+  }
+
   walRestore(pVnode->wal, pVnode, vnodeWriteToQueue);
 
   SSyncInfo syncInfo;
@@ -239,16 +250,25 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
   pVnode->sync = syncStart(&syncInfo);
 
+#ifndef _SYNC
+  pVnode->role = TAOS_SYNC_ROLE_MASTER;
+#else
+  if (pVnode->sync == NULL) {
+    vnodeCleanUp(pVnode);
+    return terrno;
+  }
+#endif
+
   // start continuous query
   if (pVnode->role == TAOS_SYNC_ROLE_MASTER) 
     cqStart(pVnode->cq);
 
   pVnode->events = NULL;
-
   pVnode->status = TAOS_VN_STATUS_READY;
   vTrace("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
 
-  atomic_add_fetch_32(&tsOpennedVnodes, 1);
+  taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -277,13 +297,6 @@ void vnodeRelease(void *pVnodeRaw) {
   }
 
   tfree(pVnode->rootDir);
-  // remove read queue
-  dnodeFreeRqueue(pVnode->rqueue);
-  pVnode->rqueue = NULL;
-
-  // remove write queue
-  dnodeFreeWqueue(pVnode->wqueue);
-  pVnode->wqueue = NULL;
 
   if (pVnode->status == TAOS_VN_STATUS_DELETING) {
     char rootDir[TSDB_FILENAME_LEN] = {0};
@@ -378,15 +391,26 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
     pVnode->sync = NULL;
   }
 
-  cqClose(pVnode->cq);
+  if (pVnode->wal) 
-  pVnode->cq = NULL;
+    walClose(pVnode->wal);
-
-  tsdbCloseRepo(pVnode->tsdb, 1);
-  pVnode->tsdb = NULL;
-
-  walClose(pVnode->wal);
   pVnode->wal = NULL;
 
+  if (pVnode->tsdb)
+    tsdbCloseRepo(pVnode->tsdb, 1);
+  pVnode->tsdb = NULL;
+
+  if (pVnode->cq) 
+    cqClose(pVnode->cq);
+  pVnode->cq = NULL;
+
+  if (pVnode->wqueue) 
+    dnodeFreeWqueue(pVnode->wqueue);
+  pVnode->wqueue = NULL;
+
+  if (pVnode->rqueue) 
+    dnodeFreeRqueue(pVnode->rqueue);
+  pVnode->rqueue = NULL;
+ 
   vnodeRelease(pVnode);
 }
 
@@ -429,7 +453,7 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
 
 static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) {
   SVnodeObj *pVnode = ahandle;
-  vTrace("vgId:%d, data file is synced, fversion:%" PRId64 "", pVnode->vgId, fversion);
+  vTrace("vgId:%d, data file is synced, fversion:%" PRId64, pVnode->vgId, fversion);
 
   pVnode->fversion = fversion;
   pVnode->version = fversion;
@@ -453,7 +477,8 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
   if (!fp) {
     vError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile,
            strerror(errno));
-    return errno;
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return terrno;
   }
 
   int32_t len = 0;
@@ -503,27 +528,30 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
 }
 
 static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
-  char cfgFile[TSDB_FILENAME_LEN + 30] = {0};
+  cJSON  *root = NULL;
+  char   *content = NULL;
+  char    cfgFile[TSDB_FILENAME_LEN + 30] = {0};
+  int     maxLen = 1000;
+
+  terrno = TSDB_CODE_OTHERS;
   sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId);
   FILE *fp = fopen(cfgFile, "r");
   if (!fp) {
-    vError("vgId:%d, failed to open vnode cfg file for read, file:%s, error:%s", pVnode->vgId,
+    vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId,
            cfgFile, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto PARSE_OVER;
+  }
+
+  content = calloc(1, maxLen + 1);
+  if (content == NULL) goto PARSE_OVER;
+  int   len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    vError("vgId:%d, failed to read vnode cfg, content is null", pVnode->vgId);
     return errno;
   }
 
-  int   ret = TSDB_CODE_OTHERS;
+  root = cJSON_Parse(content);
-  int   maxLen = 1000;
-  char *content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    free(content);
-    fclose(fp);
-    vError("vgId:%d, failed to read vnode cfg, content is null", pVnode->vgId);
-    return false;
-  }
-
-  cJSON *root = cJSON_Parse(content);
   if (root == NULL) {
     vError("vgId:%d, failed to read vnode cfg, invalid json format", pVnode->vgId);
     goto PARSE_OVER;
@@ -682,19 +710,19 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
     pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
   }
 
-  ret = 0;
+  terrno = TSDB_CODE_SUCCESS;
 
-  vPrint("vgId:%d, read vnode cfg successed, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica);
+  vPrint("vgId:%d, read vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica);
   for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) {
     vPrint("vgId:%d, dnode:%d, %s:%d", pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId,
            pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort);
   }
 
 PARSE_OVER:
-  free(content);
+  tfree(content);
   cJSON_Delete(root);
-  fclose(fp);
+  if (fp) fclose(fp);
-  return ret;
+  return terrno;
 }
 
 static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
@@ -704,7 +732,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   if (!fp) {
     vError("vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode->vgId,
            versionFile, strerror(errno));
-    return errno;
+    return TAOS_SYSTEM_ERROR(errno); 
   }
 
   int32_t len = 0;
@@ -724,29 +752,33 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   return 0;
 }
 
-static bool vnodeReadVersion(SVnodeObj *pVnode) {
+static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
-  char versionFile[TSDB_FILENAME_LEN + 30] = {0};
+  char    versionFile[TSDB_FILENAME_LEN + 30] = {0};
+  char   *content = NULL;
+  cJSON  *root = NULL;
+  int     maxLen = 100;
+
+  terrno = TSDB_CODE_OTHERS;
   sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
   FILE *fp = fopen(versionFile, "r");
   if (!fp) {
     if (errno != ENOENT) {
       vError("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, versionFile, strerror(errno));
+      terrno = TAOS_SYSTEM_ERROR(errno);
+    } else {
+      terrno = TSDB_CODE_SUCCESS;
     }
-    return false;
+    goto PARSE_OVER;
   }
 
-  bool  ret = false;
+  content = calloc(1, maxLen + 1);
-  int   maxLen = 100;
-  char *content = calloc(1, maxLen + 1);
   int   len = fread(content, 1, maxLen, fp);
   if (len <= 0) {
-    free(content);
+    vError("vgId:%d, failed to read vnode version, content is null", pVnode->vgId);
-    fclose(fp);
+    goto PARSE_OVER;
-    vPrint("vgId:%d, failed to read vnode version, content is null", pVnode->vgId);
-    return false;
   }
 
-  cJSON *root = cJSON_Parse(content);
+  root = cJSON_Parse(content);
   if (root == NULL) {
     vError("vgId:%d, failed to read vnode version, invalid json format", pVnode->vgId);
     goto PARSE_OVER;
@@ -759,13 +791,12 @@ static bool vnodeReadVersion(SVnodeObj *pVnode) {
   }
   pVnode->version = version->valueint;
 
-  ret = true;
+  terrno = TSDB_CODE_SUCCESS;
-
+  vPrint("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version);
-  vPrint("vgId:%d, read vnode version succeed, version:%" PRId64, pVnode->vgId, pVnode->version);
 
 PARSE_OVER:
-  free(content);
+  tfree(content);
   cJSON_Delete(root);
-  fclose(fp);
+  if(fp) fclose(fp);
-  return ret;
+  return terrno;
 }

src/wal/src/walMain.c

@@ -25,6 +25,7 @@
 #include "tlog.h"
 #include "tchecksum.h"
 #include "tutil.h"
+#include "taoserror.h"
 #include "twal.h"
 #include "tqueue.h"
 
@@ -56,7 +57,10 @@ static int walRemoveWalFiles(const char *path);
 
 void *walOpen(const char *path, const SWalCfg *pCfg) {
   SWal *pWal = calloc(sizeof(SWal), 1);
-  if (pWal == NULL) return NULL;
+  if (pWal == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return NULL;
+  }
 
   pWal->fd = -1;
   pWal->max = pCfg->wals;
@@ -75,6 +79,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
     walRenew(pWal);
 
   if (pWal->fd <0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
     wError("wal:%s, failed to open", path);
     pthread_mutex_destroy(&pWal->mutex);
     free(pWal);
@@ -112,9 +117,10 @@ void walClose(void *handle) {
 }
 
 int walRenew(void *handle) {
+  if (handle == NULL) return 0;
   SWal *pWal = handle;
   int   code = 0;
-  
+
   pthread_mutex_lock(&pWal->mutex);
 
   if (pWal->fd >=0) {
@@ -156,6 +162,7 @@ int walRenew(void *handle) {
 int walWrite(void *handle, SWalHead *pHead) {
   SWal *pWal = handle;
   int   code = 0;
+  if (pWal == NULL) return -1;
 
   // no wal  
   if (pWal->level == TAOS_WAL_NOLOG) return 0;
@@ -178,6 +185,7 @@ int walWrite(void *handle, SWalHead *pHead) {
 void walFsync(void *handle) {
 
   SWal *pWal = handle;
+  if (pWal == NULL) return;
 
   if (pWal->level == TAOS_WAL_FSYNC && pWal->fd >=0) {
     if (fsync(pWal->fd) < 0) {

tests/pytest/account/account_create.py

@@ -19,9 +19,9 @@ from util.sql import tdSql
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         try:

tests/pytest/account/account_del.py

@@ -19,9 +19,9 @@ from util.sql import tdSql
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         print("==========step1")

tests/pytest/alter/alter_stable.py

@@ -9,9 +9,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
         self.types = [
             "int",
             "bigint",

tests/pytest/alter/alter_table.py

@@ -9,9 +9,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
         self.types = [
             "int",
             "bigint",

tests/pytest/alter/file_corrupt.py

@@ -19,9 +19,9 @@ from util.sql import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         tdSql.prepare()

tests/pytest/dbmgmt/database-name-boundary.py

@@ -23,9 +23,9 @@ from util.sql import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
 

tests/pytest/dbmgmt/dropDB_memory_test.py

@@ -19,9 +19,9 @@ from util.sql import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         tdSql.prepare()

tests/pytest/fulltest.sh

@@ -1,127 +1,134 @@
 #!/bin/bash
-python3 ./test.py $1 -f insert/basic.py
+python3 ./test.py -f insert/basic.py
-python3 ./test.py $1 -f insert/int.py
+python3 ./test.py -f insert/int.py
-python3 ./test.py $1 -f insert/float.py
+python3 ./test.py -f insert/float.py
-python3 ./test.py $1 -f insert/bigint.py
+python3 ./test.py -f insert/bigint.py
-python3 ./test.py $1 -f insert/bool.py
+python3 ./test.py -f insert/bool.py
-python3 ./test.py $1 -f insert/double.py
+python3 ./test.py -f insert/double.py
-python3 ./test.py $1 -f insert/smallint.py
+python3 ./test.py -f insert/smallint.py
-python3 ./test.py $1 -f insert/tinyint.py
+python3 ./test.py -f insert/tinyint.py
-python3 ./test.py $1 -f insert/date.py
+python3 ./test.py -f insert/date.py
-python3 ./test.py $1 -f insert/binary.py
+python3 ./test.py -f insert/binary.py
-python3 ./test.py $1 -f insert/nchar.py
+python3 ./test.py -f insert/nchar.py
-python3 ./test.py $1 -f insert/nchar-boundary.py
+python3 ./test.py -f insert/nchar-boundary.py
-python3 ./test.py $1 -f insert/nchar-unicode.py
+python3 ./test.py -f insert/nchar-unicode.py
-python3 ./test.py $1 -f insert/multi.py
+python3 ./test.py -f insert/multi.py
+python3 ./test.py -f insert/randomNullCommit.py
 
-python3 ./test.py $1 -f table/column_name.py
+python3 ./test.py -f table/column_name.py
-python3 ./test.py $1 -f table/column_num.py
+python3 ./test.py -f table/column_num.py
-python3 ./test.py $1 -f table/db_table.py
+python3 ./test.py -f table/db_table.py
-python3 ./test.py $1 -f table/tablename-boundary.py
+python3 ./test.py -f table/tablename-boundary.py
 
 # tag
-python3 ./test.py $1 -f tag_lite/filter.py
+python3 ./test.py -f tag_lite/filter.py
-python3 ./test.py $1 -f tag_lite/create-tags-boundary.py
+python3 ./test.py -f tag_lite/create-tags-boundary.py
-python3 ./test.py $1 -f tag_lite/3.py
+python3 ./test.py -f tag_lite/3.py
-python3 ./test.py $1 -f tag_lite/4.py
+python3 ./test.py -f tag_lite/4.py
-python3 ./test.py $1 -f tag_lite/5.py
+python3 ./test.py -f tag_lite/5.py
-python3 ./test.py $1 -f tag_lite/6.py
+python3 ./test.py -f tag_lite/6.py
-python3 ./test.py $1 -f tag_lite/add.py
+python3 ./test.py -f tag_lite/add.py
-python3 ./test.py $1 -f tag_lite/bigint.py
+python3 ./test.py -f tag_lite/bigint.py
-python3 ./test.py $1 -f tag_lite/binary_binary.py
+python3 ./test.py -f tag_lite/binary_binary.py
-python3 ./test.py $1 -f tag_lite/binary.py
+python3 ./test.py -f tag_lite/binary.py
-python3 ./test.py $1 -f tag_lite/bool_binary.py
+python3 ./test.py -f tag_lite/bool_binary.py
-python3 ./test.py $1 -f tag_lite/bool_int.py
+python3 ./test.py -f tag_lite/bool_int.py
-python3 ./test.py $1 -f tag_lite/bool.py
+python3 ./test.py -f tag_lite/bool.py
-python3 ./test.py $1 -f tag_lite/change.py
+python3 ./test.py -f tag_lite/change.py
-python3 ./test.py $1 -f tag_lite/column.py
+python3 ./test.py -f tag_lite/column.py
-python3 ./test.py $1 -f tag_lite/commit.py
+python3 ./test.py -f tag_lite/commit.py
-python3 ./test.py $1 -f tag_lite/create.py
+python3 ./test.py -f tag_lite/create.py
-python3 ./test.py $1 -f tag_lite/datatype.py
+python3 ./test.py -f tag_lite/datatype.py
-python3 ./test.py $1 -f tag_lite/datatype-without-alter.py
+python3 ./test.py -f tag_lite/datatype-without-alter.py
-python3 ./test.py $1 -f tag_lite/delete.py
+python3 ./test.py -f tag_lite/delete.py
-python3 ./test.py $1 -f tag_lite/double.py
+python3 ./test.py -f tag_lite/double.py
-python3 ./test.py $1 -f tag_lite/float.py
+python3 ./test.py -f tag_lite/float.py
-python3 ./test.py $1 -f tag_lite/int_binary.py
+python3 ./test.py -f tag_lite/int_binary.py
-python3 ./test.py $1 -f tag_lite/int_float.py
+python3 ./test.py -f tag_lite/int_float.py
-python3 ./test.py $1 -f tag_lite/int.py
+python3 ./test.py -f tag_lite/int.py
-python3 ./test.py $1 -f tag_lite/set.py
+python3 ./test.py -f tag_lite/set.py
-python3 ./test.py $1 -f tag_lite/smallint.py
+python3 ./test.py -f tag_lite/smallint.py
-python3 ./test.py $1 -f tag_lite/tinyint.py
+python3 ./test.py -f tag_lite/tinyint.py
 
-python3 ./test.py $1 -f dbmgmt/database-name-boundary.py
+python3 ./test.py -f dbmgmt/database-name-boundary.py
 
-python3 ./test.py $1 -f import_merge/importBlock1HO.py
+python3 ./test.py -f import_merge/importBlock1HO.py
-python3 ./test.py $1 -f import_merge/importBlock1HPO.py
+python3 ./test.py -f import_merge/importBlock1HPO.py
-python3 ./test.py $1 -f import_merge/importBlock1H.py
+python3 ./test.py -f import_merge/importBlock1H.py
-python3 ./test.py $1 -f import_merge/importBlock1S.py
+python3 ./test.py -f import_merge/importBlock1S.py
-python3 ./test.py $1 -f import_merge/importBlock1Sub.py
+python3 ./test.py -f import_merge/importBlock1Sub.py
-python3 ./test.py $1 -f import_merge/importBlock1TO.py
+python3 ./test.py -f import_merge/importBlock1TO.py
-python3 ./test.py $1 -f import_merge/importBlock1TPO.py
+python3 ./test.py -f import_merge/importBlock1TPO.py
-python3 ./test.py $1 -f import_merge/importBlock1T.py
+python3 ./test.py -f import_merge/importBlock1T.py
-python3 ./test.py $1 -f import_merge/importBlock2HO.py
+python3 ./test.py -f import_merge/importBlock2HO.py
-python3 ./test.py $1 -f import_merge/importBlock2HPO.py
+python3 ./test.py -f import_merge/importBlock2HPO.py
-python3 ./test.py $1 -f import_merge/importBlock2H.py
+python3 ./test.py -f import_merge/importBlock2H.py
-python3 ./test.py $1 -f import_merge/importBlock2S.py
+python3 ./test.py -f import_merge/importBlock2S.py
-python3 ./test.py $1 -f import_merge/importBlock2Sub.py
+python3 ./test.py -f import_merge/importBlock2Sub.py
-python3 ./test.py $1 -f import_merge/importBlock2TO.py
+python3 ./test.py -f import_merge/importBlock2TO.py
-python3 ./test.py $1 -f import_merge/importBlock2TPO.py
+python3 ./test.py -f import_merge/importBlock2TPO.py
-python3 ./test.py $1 -f import_merge/importBlock2T.py
+python3 ./test.py -f import_merge/importBlock2T.py
-python3 ./test.py $1 -f import_merge/importBlockbetween.py
+python3 ./test.py -f import_merge/importBlockbetween.py
-python3 ./test.py $1 -f import_merge/importCacheFileHO.py
+python3 ./test.py -f import_merge/importCacheFileHO.py
-python3 ./test.py $1 -f import_merge/importCacheFileHPO.py
+python3 ./test.py -f import_merge/importCacheFileHPO.py
-python3 ./test.py $1 -f import_merge/importCacheFileH.py
+python3 ./test.py -f import_merge/importCacheFileH.py
-python3 ./test.py $1 -f import_merge/importCacheFileS.py
+python3 ./test.py -f import_merge/importCacheFileS.py
-python3 ./test.py $1 -f import_merge/importCacheFileSub.py
+python3 ./test.py -f import_merge/importCacheFileSub.py
-python3 ./test.py $1 -f import_merge/importCacheFileTO.py
+python3 ./test.py -f import_merge/importCacheFileTO.py
-python3 ./test.py $1 -f import_merge/importCacheFileTPO.py
+python3 ./test.py -f import_merge/importCacheFileTPO.py
-python3 ./test.py $1 -f import_merge/importCacheFileT.py
+python3 ./test.py -f import_merge/importCacheFileT.py
-python3 ./test.py $1 -f import_merge/importDataH2.py
+python3 ./test.py -f import_merge/importDataH2.py
-python3 ./test.py $1 -f import_merge/importDataHO2.py
+python3 ./test.py -f import_merge/importDataHO2.py
-python3 ./test.py $1 -f import_merge/importDataHO.py
+python3 ./test.py -f import_merge/importDataHO.py
-python3 ./test.py $1 -f import_merge/importDataHPO.py
+python3 ./test.py -f import_merge/importDataHPO.py
-python3 ./test.py $1 -f import_merge/importDataLastHO.py
+python3 ./test.py -f import_merge/importDataLastHO.py
-python3 ./test.py $1 -f import_merge/importDataLastHPO.py
+python3 ./test.py -f import_merge/importDataLastHPO.py
-python3 ./test.py $1 -f import_merge/importDataLastH.py
+python3 ./test.py -f import_merge/importDataLastH.py
-python3 ./test.py $1 -f import_merge/importDataLastS.py
+python3 ./test.py -f import_merge/importDataLastS.py
-python3 ./test.py $1 -f import_merge/importDataLastSub.py
+python3 ./test.py -f import_merge/importDataLastSub.py
-python3 ./test.py $1 -f import_merge/importDataLastTO.py
+python3 ./test.py -f import_merge/importDataLastTO.py
-python3 ./test.py $1 -f import_merge/importDataLastTPO.py
+python3 ./test.py -f import_merge/importDataLastTPO.py
-python3 ./test.py $1 -f import_merge/importDataLastT.py
+python3 ./test.py -f import_merge/importDataLastT.py
-python3 ./test.py $1 -f import_merge/importDataS.py
+python3 ./test.py -f import_merge/importDataS.py
-python3 ./test.py $1 -f import_merge/importDataSub.py
+python3 ./test.py -f import_merge/importDataSub.py
-python3 ./test.py $1 -f import_merge/importDataTO.py
+python3 ./test.py -f import_merge/importDataTO.py
-python3 ./test.py $1 -f import_merge/importDataTPO.py
+python3 ./test.py -f import_merge/importDataTPO.py
-python3 ./test.py $1 -f import_merge/importDataT.py
+python3 ./test.py -f import_merge/importDataT.py
-python3 ./test.py $1 -f import_merge/importHeadOverlap.py
+python3 ./test.py -f import_merge/importHeadOverlap.py
-python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py
+python3 ./test.py -f import_merge/importHeadPartOverlap.py
-python3 ./test.py $1 -f import_merge/importHead.py
+python3 ./test.py -f import_merge/importHead.py
-python3 ./test.py $1 -f import_merge/importHORestart.py
+python3 ./test.py -f import_merge/importHORestart.py
-python3 ./test.py $1 -f import_merge/importHPORestart.py
+python3 ./test.py -f import_merge/importHPORestart.py
-python3 ./test.py $1 -f import_merge/importHRestart.py
+python3 ./test.py -f import_merge/importHRestart.py
-python3 ./test.py $1 -f import_merge/importLastHO.py
+python3 ./test.py -f import_merge/importLastHO.py
-python3 ./test.py $1 -f import_merge/importLastHPO.py
+python3 ./test.py -f import_merge/importLastHPO.py
-python3 ./test.py $1 -f import_merge/importLastH.py
+python3 ./test.py -f import_merge/importLastH.py
-python3 ./test.py $1 -f import_merge/importLastS.py
+python3 ./test.py -f import_merge/importLastS.py
-python3 ./test.py $1 -f import_merge/importLastSub.py
+python3 ./test.py -f import_merge/importLastSub.py
-python3 ./test.py $1 -f import_merge/importLastTO.py
+python3 ./test.py -f import_merge/importLastTO.py
-python3 ./test.py $1 -f import_merge/importLastTPO.py
+python3 ./test.py -f import_merge/importLastTPO.py
-python3 ./test.py $1 -f import_merge/importLastT.py
+python3 ./test.py -f import_merge/importLastT.py
-python3 ./test.py $1 -f import_merge/importSpan.py
+python3 ./test.py -f import_merge/importSpan.py
-python3 ./test.py $1 -f import_merge/importSRestart.py
+python3 ./test.py -f import_merge/importSRestart.py
-python3 ./test.py $1 -f import_merge/importSubRestart.py
+python3 ./test.py -f import_merge/importSubRestart.py
-python3 ./test.py $1 -f import_merge/importTailOverlap.py
+python3 ./test.py -f import_merge/importTailOverlap.py
-python3 ./test.py $1 -f import_merge/importTailPartOverlap.py
+python3 ./test.py -f import_merge/importTailPartOverlap.py
-python3 ./test.py $1 -f import_merge/importTail.py
+python3 ./test.py -f import_merge/importTail.py
-python3 ./test.py $1 -f import_merge/importToCommit.py
+python3 ./test.py -f import_merge/importToCommit.py
-python3 ./test.py $1 -f import_merge/importTORestart.py
+python3 ./test.py -f import_merge/importTORestart.py
-python3 ./test.py $1 -f import_merge/importTPORestart.py
+python3 ./test.py -f import_merge/importTPORestart.py
-python3 ./test.py $1 -f import_merge/importTRestart.py
+python3 ./test.py -f import_merge/importTRestart.py
+python3 ./test.py -f import_merge/importInsertThenImport.py
 
 # user
-python3 ./test.py $1 -f user/user_create.py
+python3 ./test.py -f user/user_create.py
-python3 ./test.py $1 -f user/pass_len.py
+python3 ./test.py -f user/pass_len.py
 
 # table
-#python3 ./test.py $1 -f table/del_stable.py
+#python3 ./test.py -f table/del_stable.py
 
+#query
+python3 ./test.py -f query/filter.py
+python3 ./test.py $1 -f query/filterCombo.py
+python3 ./test.py $1 -f query/queryNormal.py
+python3 ./test.py $1 -f query/queryError.py

tests/pytest/import_merge/importBlock1H.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1

tests/pytest/import_merge/importBlock1HO.py

@@ -20,9 +20,9 @@ from util.dnodes import *
 
 
 class TDTestCase:
-    def init(self, conn):
+    def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)
 
     def run(self):
         self.ntables = 1