Merge branch 'develop' into hotfix/test

.gitignore

@@ -65,3 +65,4 @@ CMakeError.log
 /out/isenseconfig/WSL-GCC-Debug
 /test/cfg
 /src/.vs
+*.o

src/client/inc/tsclient.h

@@ -207,11 +207,11 @@ typedef struct SQueryInfo {
   SLimitVal        slimit;
   STagCond         tagCond;
   SOrderVal        order;
-  int16_t          fillType;  // interpolate type
+  int16_t          fillType;    // final result fill type
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
   struct STSBuf *  tsBuf;
-  int64_t *        fillVal;      // default value for interpolation
+  int64_t *        fillVal;      // default value for fill
   char *           msg;          // pointer to the pCmd->payload to keep error message temporarily
   int64_t          clauseLimit;  // limit for current sub clause
 
@@ -222,7 +222,6 @@ typedef struct SQueryInfo {
 typedef struct {
   int     command;
   uint8_t msgType;
-
   bool    autoCreated;        // if the table is missing, on-the-fly create it. during getmeterMeta
   int8_t  dataSourceType;     // load data from file or not
 
@@ -231,6 +230,7 @@ typedef struct {
     int32_t numOfTablesInSubmit;
   };
 
+  int32_t      insertType;
   int32_t      clauseIndex;  // index of multiple subclause query
   int8_t       parseFinished;
   short        numOfCols;
@@ -239,14 +239,12 @@ typedef struct {
   int32_t      payloadLen;
   SQueryInfo **pQueryInfo;
   int32_t      numOfClause;
-
-  SDataBlockList *pDataBlocks;  // submit data blocks after parsing sql
   char *       curSql;       // current sql, resume position of sql after parsing paused
   void *       pTableList;   // referred table involved in sql
-
+  int32_t      batchSize;    // for parameter ('?') binding and batch processing
-  // for parameter ('?') binding and batch processing
-  int32_t batchSize;
   int32_t      numOfParams;
+
+  SDataBlockList *pDataBlocks;  // submit data blocks after parsing sql
 } SSqlCmd;
 
 typedef struct SResRec {
@@ -287,8 +285,8 @@ typedef struct STscObj {
   void *             pTimer;
   char               user[TSDB_USER_LEN];
   char               pass[TSDB_KEY_LEN];
-  char               acctId[TSDB_DB_NAME_LEN];
+  char               acctId[TSDB_ACCT_LEN];
-  char               db[TSDB_TABLE_ID_LEN];
+  char               db[TSDB_DB_NAME_LEN];
   char               sversion[TSDB_VERSION_LEN];
   char               writeAuth : 1;
   char               superAuth : 1;
@@ -316,7 +314,6 @@ typedef struct SSqlObj {
   SRpcIpSet        ipList;
   char             freed : 4;
   char             listed : 4;
-  uint32_t         insertType;
   tsem_t           rspSem;
   SSqlCmd          cmd;
   SSqlRes          res;
@@ -361,7 +358,7 @@ int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet);
 int  tscProcessSql(SSqlObj *pSql);
 
-int  tscRenewMeterMeta(SSqlObj *pSql, char *tableId);
+int  tscRenewTableMeta(SSqlObj *pSql, char *tableId);
 void tscQueueAsyncRes(SSqlObj *pSql);
 
 void tscQueueAsyncError(void(*fp), void *param, int32_t code);

src/client/src/tscAsync.c

@@ -442,10 +442,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   }
 
   if (pSql->pStream == NULL) {
-    // check if it is a sub-query of super table query first, if true, enter another routine
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
-    if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) {
+    // check if it is a sub-query of super table query first, if true, enter another routine
+    if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY)) {
+      tscTrace("%p update table meta in local cache, continue to process sql and send corresponding subquery", pSql);
+
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
       if (pTableMetaInfo->pTableMeta == NULL){
         code = tscGetTableMeta(pSql, pTableMetaInfo);
@@ -460,32 +462,37 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
       assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
           tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0);
 
-      tscTrace("%p get metricMeta during super table query successfully", pSql);      
+      // NOTE: the vgroupInfo for the queried super table must be existed here.
-
+      assert(pTableMetaInfo->vgroupList != NULL);
-      code = tscGetSTableVgroupInfo(pSql, 0);
+      if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) {
-      pRes->code = code;
+        return;
-
+      }
-      if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
+    } else {  // continue to process normal async query
-    } else {  // normal async query continues
       if (pCmd->parseFinished) {
-        tscTrace("%p re-send data to vnode in table Meta callback since sql parsed completed", pSql);
+        tscTrace("%p update table meta in local cache, continue to process sql and send corresponding query", pSql);
 
         STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
         code = tscGetTableMeta(pSql, pTableMetaInfo);
         assert(code == TSDB_CODE_SUCCESS);
 
-        if (pTableMetaInfo->pTableMeta) {
+        // if failed to process sql, go to error handler
-          // todo update the submit message according to the new table meta
+        if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) {
-          // 1. table uid, 2. ip address
+          return;
-          code = tscSendMsgToServer(pSql);
-          if (code == TSDB_CODE_SUCCESS) return;
         }
+//          // todo update the submit message according to the new table meta
+//          // 1. table uid, 2. ip address
+//          code = tscSendMsgToServer(pSql);
+//          if (code == TSDB_CODE_SUCCESS) return;
+//        }
       } else {
+        tscTrace("%p continue parse sql after get table meta", pSql);
+
         code = tsParseSql(pSql, false);
-        if ((pQueryInfo->type & TSDB_QUERY_TYPE_STMT_INSERT) == TSDB_QUERY_TYPE_STMT_INSERT) {
+        if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STMT_INSERT)) {
           STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
           code = tscGetTableMeta(pSql, pTableMetaInfo);
           assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->pTableMeta != NULL);
+
           (*pSql->fp)(pSql->param, pSql, code);
           return;
         }

src/client/src/tscFunctionImpl.c

@@ -1293,7 +1293,7 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   minMax_function_f(pCtx, index, 0);
   
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-  if (pResInfo->hasResult == DATA_SET_FLAG) {
+  if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
     char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
     *flag = DATA_SET_FLAG;
   }
@@ -1309,7 +1309,7 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   minMax_function_f(pCtx, index, 1);
   
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-  if (pResInfo->hasResult == DATA_SET_FLAG) {
+  if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
     char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
     *flag = DATA_SET_FLAG;
   }

src/client/src/tscLocal.c

@@ -132,7 +132,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
   for (int32_t i = 0; i < numOfRows; ++i) {
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
     char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN);
+    STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
 
     char *type = tDataTypeDesc[pSchema[i].type].aName;
 
@@ -171,7 +171,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     // field name
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
     char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN);
+    STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
 
     // type name
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
@@ -211,18 +211,18 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   pQueryInfo->order.order = TSDB_ORDER_ASC;
 
-  TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE};
+  TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
-  strncpy(f.name, "Field", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Field", sizeof(f.name));
   
   SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
-      TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE, TSDB_COL_NAME_LEN, false);
+      (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false);
   
-  rowLen += (TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE);
+  rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
 
   f.bytes = typeColLength;
   f.type = TSDB_DATA_TYPE_BINARY;
-  strncpy(f.name, "Type", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Type", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength,
@@ -232,7 +232,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
 
   f.bytes = sizeof(int32_t);
   f.type = TSDB_DATA_TYPE_INT;
-  strncpy(f.name, "Length", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Length", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t),
@@ -242,7 +242,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
 
   f.bytes = noteColLength;
   f.type = TSDB_DATA_TYPE_BINARY;
-  strncpy(f.name, "Note", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Note", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength,
@@ -286,7 +286,7 @@ static void tscProcessCurrentUser(SSqlObj *pSql) {
 }
 
 static void tscProcessCurrentDB(SSqlObj *pSql) {
-  char db[TSDB_DB_NAME_LEN + 1] = {0};
+  char db[TSDB_DB_NAME_LEN] = {0};
   extractDBName(pSql->pTscObj->db, db);
   
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);

src/client/src/tscParseInsert.c

@@ -795,7 +795,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX);
     tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql);
 
-    strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN);
+    tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
     code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -989,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
 
 int validateTableName(char *tblName, int len) {
   char buf[TSDB_TABLE_ID_LEN] = {0};
-  strncpy(buf, tblName, len);
+  tstrncpy(buf, tblName, sizeof(buf));
 
   SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
   tSQLGetToken(buf, &token.type);
@@ -1314,7 +1314,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
   tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
 
   TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT);
-  TSDB_QUERY_SET_TYPE(pQueryInfo->type, pSql->insertType);
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, pCmd->insertType);
 
   sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL);
   if (sToken.type != TK_INTO) {
@@ -1342,7 +1342,7 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
      * Set the fp before parse the sql string, in case of getTableMeta failed, in which
      * the error handle callback function can rightfully restore the user-defined callback function (fp).
      */
-    if (initialParse && (pSql->insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
+    if (initialParse && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
       pSql->fetchFp = pSql->fp;
       pSql->fp = (void(*)())tscHandleMultivnodeInsert;
     }
@@ -1354,9 +1354,7 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
       return ret;
     }
     
-    SSqlInfo SQLInfo = {0};
+    SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr);
-    tSQLParse(&SQLInfo, pSql->sqlstr);
-
     ret = tscToSQLCmd(pSql, &SQLInfo);
     SQLInfoDestroy(&SQLInfo);
   }
@@ -1514,7 +1512,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
     }
     pCmd->count = 1;
 
-    strncpy(path, pDataBlock->filename, PATH_MAX);
+    tstrncpy(path, pDataBlock->filename, sizeof(path));
 
     FILE *fp = fopen(path, "r");
     if (fp == NULL) {
@@ -1522,7 +1520,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
       continue;
     }
 
-    strncpy(pTableMetaInfo->name, pDataBlock->tableId, TSDB_TABLE_ID_LEN);
+    tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
     memset(pDataBlock->pData, 0, pDataBlock->nAllocSize);
 
     int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo);

src/client/src/tscPrepare.c

@@ -451,7 +451,7 @@ static int insertStmtExecute(STscStmt* stmt) {
   
   pRes->qhandle = 0;
 
-  pSql->insertType = 0;
+  pSql->cmd.insertType = 0;
   pSql->fetchFp    = waitForQueryRsp;
   pSql->fp         = (void(*)())tscHandleMultivnodeInsert;
 
@@ -515,7 +515,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
   SSqlRes *pRes    = &pSql->res;
   pSql->param      = (void*) pSql;
   pSql->fp         = waitForQueryRsp;
-  pSql->insertType = TSDB_QUERY_TYPE_STMT_INSERT;
+  pSql->cmd.insertType = TSDB_QUERY_TYPE_STMT_INSERT;
   
   if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     tscError("%p failed to malloc payload buffer", pSql);

src/client/src/tscProfile.c

@@ -239,8 +239,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
       continue;
     }
 
-    strncpy(pQdesc->sql, pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
+    tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql));
-    pQdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
     pQdesc->stime = htobe64(pSql->stime);
     pQdesc->queryId = htonl(pSql->queryId);
     pQdesc->useconds = htobe64(pSql->res.useconds);
@@ -256,8 +255,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
 
   SSqlStream *pStream = pObj->streamList;
   while (pStream) {
-    strncpy(pSdesc->sql, pStream->pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
+    tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql));
-    pSdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
     pSdesc->streamId = htonl(pStream->streamId);
     pSdesc->num = htobe64(pStream->num);
 

src/client/src/tscSQLParser.c

@@ -348,7 +348,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
       }
 
-      if (pToken->n > TSDB_TABLE_NAME_LEN) {
+      if (pToken->n >= TSDB_TABLE_NAME_LEN) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
@@ -543,6 +543,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression");
   }
 
+  pSql->cmd.parseFinished = true;
   return tscBuildMsg[pCmd->command](pSql, pInfo);
 }
 
@@ -932,7 +933,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
   SSchema* pSchema = tscGetTableSchema(pTableMeta);
 
   for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
-    if (strncasecmp(pTagField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) {
+    if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       return false;
     }
@@ -992,7 +993,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
 
   // field name must be unique
   for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
-    if (strncasecmp(pColField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) {
+    if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       return false;
     }
@@ -1004,7 +1005,8 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
 /* is contained in pFieldList or not */
 static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) {
   for (int32_t j = startIdx; j < pFieldList->nField; ++j) {
-    if (strncasecmp(name, pFieldList->p[j].name, TSDB_COL_NAME_LEN) == 0) return true;
+    TAOS_FIELD* field = pFieldList->p + j;
+    if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true;
   }
 
   return false;
@@ -1040,7 +1042,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
 
   /* db name is not specified, the tableName dose not include db name */
   if (pDB != NULL) {
-    if (pDB->n > TSDB_DB_NAME_LEN) {
+    if (pDB->n >= TSDB_DB_NAME_LEN) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 
@@ -1059,7 +1061,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
       }
     } else {  // pDB == NULL, the db prefix name is specified in tableName
       /* the length limitation includes tablename + dbname + sep */
-      if (tableName->n > TSDB_TABLE_NAME_LEN + TSDB_DB_NAME_LEN + tListLen(TS_PATH_DELIMITER)) {
+      if (tableName->n > (TSDB_TABLE_NAME_LEN - 1) + (TSDB_DB_NAME_LEN - 1) + sizeof(TS_PATH_DELIMITER)) {
         return TSDB_CODE_TSC_INVALID_SQL;
       }
     }
@@ -1165,7 +1167,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
         
         /* todo alias name should use the original sql string */
         char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr;
-        strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN);
+        tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName));
         
         tExprNode* pNode = NULL;
         SArray* colList = taosArrayInit(10, sizeof(SColIndex));
@@ -1307,7 +1309,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
   SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
 
   char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName;
-  strncpy(pExpr->aliasName, colName, tListLen(pExpr->aliasName));
+  tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName));
   
   SColumnList ids = {0};
   ids.num = 1;
@@ -1357,7 +1359,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
 
   for (int32_t j = 0; j < numOfTotalColumns; ++j) {
     SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos + j, j, pIndex->tableIndex);
-    strncpy(pExpr->aliasName, pSchema[j].name, tListLen(pExpr->aliasName));
+    tstrncpy(pExpr->aliasName, pSchema[j].name, sizeof(pExpr->aliasName));
 
     pIndex->columnIndex = j;
     SColumnList ids = {0};
@@ -1400,7 +1402,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI
     }
 
     if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
-      SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE};
+      SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
       strcpy(colSchema.name, TSQL_TBNAME_L);
 
       tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, true);
@@ -1447,11 +1449,11 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
   if (aliasName != NULL) {
     strcpy(columnName, aliasName);
   } else {
-    getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name);
+    getRevisedName(columnName, functionID, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
   }
   
   SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
-  strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName));
+  tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
   
   // for all queries, the timestamp column needs to be loaded
   SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
@@ -1535,7 +1537,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
       }
       
       memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
-      getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN);
+      getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
       
       SColumnList ids = getColumnList(1, index.tableIndex, index.columnIndex);
       if (finalResult) {
@@ -1650,7 +1652,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
       ids.ids[0] = index;
   
       memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
-      getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN);
+      getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
   
       if (finalResult) {
         int32_t numOfOutput = tscNumOfFields(pQueryInfo);
@@ -1850,7 +1852,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
       }
   
       memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
-      getColumnName(pItem, pExpr->aliasName, TSDB_COL_NAME_LEN);
+      getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1);
   
       SColumnList ids = getColumnList(1, 0, index.columnIndex);
       if (finalResult) {
@@ -2201,7 +2203,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     if (pDbPrefixToken->type != 0) {
       assert(pDbPrefixToken->n >= 0);
 
-      if (pDbPrefixToken->n > TSDB_DB_NAME_LEN) {  // db name is too long
+      if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) {  // db name is too long
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       }
 
@@ -2228,7 +2230,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
       }
 
-      if (pCmd->payloadLen > TSDB_TABLE_NAME_LEN) {
+      if (pCmd->payloadLen >= TSDB_TABLE_NAME_LEN) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
     }
@@ -3996,7 +3998,7 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) {
 
   for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
     char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name;
-    for (int32_t j = 0; j < TSDB_COL_NAME_LEN && fieldName[j] != 0; ++j) {
+    for (int32_t j = 0; j < (TSDB_COL_NAME_LEN - 1) && fieldName[j] != 0; ++j) {
       for (int32_t k = 0; k < tListLen(rep); ++k) {
         if (fieldName[j] == rep[k]) {
           fieldName[j] = '_';
@@ -4012,7 +4014,7 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) {
   for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
     char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name;
     for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) {
-      if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, TSDB_COL_NAME_LEN) == 0) {
+      if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, (TSDB_COL_NAME_LEN - 1)) == 0) {
         const char* msg = "duplicated column name in new table";
         return invalidSqlErrMsg(pQueryInfo->msg, msg);
       }
@@ -4367,7 +4369,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     }
 
     tVariantListItem* pItem = &pAlterSQL->varList->a[0];
-    if (pItem->pVar.nLen > TSDB_COL_NAME_LEN) {
+    if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg9);
     }
 
@@ -4419,7 +4421,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 
-    char name[TSDB_COL_NAME_LEN + 1] = {0};
+    char name[TSDB_COL_NAME_LEN] = {0};
     strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen);
     TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
     tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
@@ -4521,8 +4523,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg18);
     }
 
-    char name1[TSDB_COL_NAME_LEN + 1] = {0};
+    char name1[TSDB_COL_NAME_LEN] = {0};
-    strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen);
+    tstrncpy(name1, pItem->pVar.pz, sizeof(name1));
     TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
     tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   }
@@ -5231,7 +5233,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
     int16_t colIndex = pColIndex->colIndex;
     if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
       type = TSDB_DATA_TYPE_BINARY;
-      bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; // todo extract method
+      bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE; // todo extract method
       name = TSQL_TBNAME_L;
     } else {
       if (TSDB_COL_IS_TAG(pColIndex->flag)) {
@@ -5253,8 +5255,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
       SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex};
       SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true);
       
-      memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
+      memset(pExpr->aliasName, 0, sizeof(pExpr->aliasName));
-      strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN);
+      tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName));
       
       pExpr->colInfo.flag = TSDB_COL_TAG;
 
@@ -5418,7 +5420,7 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
                                       tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
   
   const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
-  strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
+  tstrncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
   
   return TSDB_CODE_SUCCESS;
 }
@@ -6018,7 +6020,7 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
   
       if (pCols != NULL) {  // record the involved columns
         SColIndex colIndex = {0};
-        strncpy(colIndex.name, pSchema->name, TSDB_COL_NAME_LEN);
+        tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name));
         colIndex.colId = pSchema->colId;
         colIndex.colIndex = index.columnIndex;
         

src/client/src/tscSchemaUtil.c

@@ -115,7 +115,7 @@ bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) {
 
     // 3. valid column names
     for (int32_t j = i + 1; j < numOfCols; ++j) {
-      if (strncasecmp(pSchema[i].name, pSchema[j].name, TSDB_COL_NAME_LEN) == 0) {
+      if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
         return false;
       }
     }

src/client/src/tscSecondaryMerge.c

@@ -324,7 +324,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
     tfree(pReducer->discardData);
     tfree(pReducer->pResultBuf);
     tfree(pReducer->pFinalRes);
-//    tfree(pReducer->pBufForInterpo);
     tfree(pReducer->prevRowOfInput);
 
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -363,7 +362,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   if (pQueryInfo->fillType != TSDB_FILL_NONE) {
     SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
     pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
-                                           4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->fillType, pFillCol);
+                                           4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit,
+                                           tinfo.precision, pQueryInfo->fillType, pFillCol);
   }
 
   int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
@@ -494,7 +494,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
       tscTrace("%p waiting for delete procedure, status: %d", pSql, status);
     }
 
-    taosDestoryFillInfo(pLocalReducer->pFillInfo);
+    pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo);
 
     if (pLocalReducer->pCtx != NULL) {
       for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
@@ -980,8 +980,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
         }
 
         /* all output for current group are completed */
-        int32_t totalRemainRows =
+        int32_t totalRemainRows = getFilledNumOfRes(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
-            taosGetNumOfResultWithFill(pFillInfo, rpoints, pFillInfo->slidingTime, actualETime);
         if (totalRemainRows <= 0) {
           break;
         }
@@ -1185,7 +1184,9 @@ bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage
   int32_t ret = 0;  // merge all result by default
   
   int16_t functionId = pLocalReducer->pCtx[0].functionId;
-  if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_ARITHM) {  // column projection query
+
+  // todo opt performance
+  if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0))) {  // column projection query
     ret = 1;                                                            // disable merge procedure
   } else {
     tOrderDescriptor *pDesc = pLocalReducer->pDesc;
@@ -1265,13 +1266,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
   
   SFillInfo* pFillInfo = pLocalReducer->pFillInfo;
   if (pFillInfo != NULL) {
-    STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
+    taosFillSetStartInfo(pFillInfo, pResBuf->num, pQueryInfo->window.ekey);
-    STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
-  
-    TSKEY ekey = taosGetRevisedEndKey(pQueryInfo->window.ekey, pFillInfo->order, pFillInfo->slidingTime,
-                                      pQueryInfo->slidingTimeUnit, tinfo.precision);
-  
-    taosFillSetStartInfo(pFillInfo, pResBuf->num, ekey);
     taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf);
   }
   
@@ -1325,23 +1320,15 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) {
   SLocalReducer *pLocalReducer = pRes->pLocalReducer;
   SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
 
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
-  
-  int8_t p = tinfo.precision;
-
   if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) {
     assert(pQueryInfo->fillType != TSDB_FILL_NONE);
 
     tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf;
     int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
 
-    int32_t remain = taosNumOfRemainRows(pFillInfo);
-    TSKEY ekey = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, p);
-    
     // the first column must be the timestamp column
-    int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pLocalReducer->resColModel->capacity);
+    int32_t rows = getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
-    if (rows > 0) {  // do interpo
+    if (rows > 0) {  // do fill gap
       doFillResult(pSql, pLocalReducer, false);
     }
 
@@ -1360,10 +1347,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
 
   bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow;
 
-  SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  
-  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
 
   if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL ||
       prevGroupCompleted) {
@@ -1371,9 +1355,8 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
     if (pQueryInfo->fillType != TSDB_FILL_NONE) {
       int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey;
 
-      etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime,
+      assert(pFillInfo->numOfRows == 0);
-                                   pQueryInfo->slidingTimeUnit, tinfo.precision);
+      int32_t rows = getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
-      int32_t rows = taosGetNumOfResultWithFill(pFillInfo, 0, etime, pLocalReducer->resColModel->capacity);
       if (rows > 0) {  // do interpo
         doFillResult(pSql, pLocalReducer, true);
       }

src/client/src/tscServer.c

@@ -239,16 +239,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
     STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
     if (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || 
         rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-      /*
-       * not_active_table: 1. the virtual node may fail to create table, since the procedure of create table is asynchronized,
-       *                   the virtual node may have not create table till now, so try again by using the new metermeta.
-       *                   2. this requested table may have been removed by other client, so we need to renew the
-       *                   metermeta here.
-       *
-       * not_active_vnode: current vnode is move to other node due to node balance procedure or virtual node have been
-       *                   removed. So, renew metermeta and try again.
-       * not_active_session: db has been move to other node, the vnode does not exist on this dnode anymore.
-       */
       if (pCmd->command == TSDB_SQL_CONNECT) {
         rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
         rpcFreeCont(rpcMsg->pCont);
@@ -258,8 +248,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
         rpcFreeCont(rpcMsg->pCont);
         return;
       } else if (pCmd->command == TSDB_SQL_META) {
-//        rpcFreeCont(rpcMsg->pCont);
+        // get table meta query will not retry, do nothing
-//        return;
       } else {
         tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry);
         
@@ -267,17 +256,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
         if (pSql->retry > pSql->maxRetry) {
           tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
         } else {
-          rpcMsg->code = tscRenewMeterMeta(pSql, pTableMetaInfo->name);
+          rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name);
-          if (pTableMetaInfo->pTableMeta) {
-            tscSendMsgToServer(pSql);
-          }
 
+          // if there is an error occurring, proceed to the following error handling procedure.
+          // todo add test cases
+          if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
             rpcFreeCont(rpcMsg->pCont);
             return;
           }
         }
       }
     }
+  }
   
   pRes->rspLen = 0;
   
@@ -330,9 +320,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
     }
   }
   
-  if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command])
+  if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) {
     rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql);
-  
+  }
 
   if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? pRes->numOfRows: pRes->code;
@@ -431,7 +421,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
 
     /*
      * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause
-     * sub-queries not correctly released and master sql object of metric query reaches an abnormal state.
+     * sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
      */
     pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
     //taosStopRpcConn(pSql->pSubs[i]->thandle);
@@ -565,7 +555,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
 
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
     pMsg += sizeof(STableIdInfo);
-  } else {
+  } else { // it is a subquery of the super table query, this IP info is acquired from vgroupInfo
     int32_t index = pTableMetaInfo->vgroupIndex;
     int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
     assert(index >= 0 && index < numOfVgroups);
@@ -876,7 +866,7 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   assert(pCmd->numOfClause == 1);
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  strncpy(pCreateDbMsg->db, pTableMetaInfo->name, tListLen(pCreateDbMsg->db));
+  tstrncpy(pCreateDbMsg->db, pTableMetaInfo->name, sizeof(pCreateDbMsg->db));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1028,7 +1018,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  strcpy(pDrop->ep, pTableMetaInfo->name);
+  tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep));
   pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
 
   return TSDB_CODE_SUCCESS;
@@ -1046,7 +1036,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  strcpy(pDropMsg->user, pTableMetaInfo->name);
+  tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1101,9 +1091,9 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   size_t nameLen = strlen(pTableMetaInfo->name);
   if (nameLen > 0) {
-    strcpy(pShowMsg->db, pTableMetaInfo->name);  // prefix is set here
+    tstrncpy(pShowMsg->db, pTableMetaInfo->name, sizeof(pShowMsg->db));  // prefix is set here
   } else {
-    strcpy(pShowMsg->db, pObj->db);
+    tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db));
   }
 
   SShowInfo *pShowInfo = &pInfo->pDCLInfo->showOpt;
@@ -1310,7 +1300,7 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  strcpy(pAlterDbMsg->db, pTableMetaInfo->name);
+  tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1440,9 +1430,9 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   char *db;  // ugly code to move the space
   db = strstr(pObj->db, TS_PATH_DELIMITER);
   db = (db == NULL) ? pObj->db : db + 1;
-  strcpy(pConnect->db, db);
+  tstrncpy(pConnect->db, db, sizeof(pConnect->db));
-  strcpy(pConnect->clientVersion, version);
+  tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion));
-  strcpy(pConnect->msgVersion, "");
+  tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1822,7 +1812,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  tscTrace("%p recv table meta: %"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
+  tscTrace("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
   free(pTableMeta);
   
   return TSDB_CODE_SUCCESS;
@@ -2160,11 +2150,11 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
 
   SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp;
-  strcpy(pObj->acctId, pConnect->acctId);  // copy acctId from response
+  tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId));  // copy acctId from response
   int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db);
 
-  assert(len <= tListLen(pObj->db));
+  assert(len <= sizeof(pObj->db));
-  strncpy(pObj->db, temp, tListLen(pObj->db));
+  tstrncpy(pObj->db, temp, sizeof(pObj->db));
   
   if (pConnect->ipList.numOfIps > 0) 
     tscSetMgmtIpList(&pConnect->ipList);
@@ -2182,7 +2172,7 @@ int tscProcessUseDbRsp(SSqlObj *pSql) {
   STscObj *       pObj = pSql->pTscObj;
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
 
-  strcpy(pObj->db, pTableMetaInfo->name);
+  tstrncpy(pObj->db, pTableMetaInfo->name, sizeof(pObj->db));
   return 0;
 }
 
@@ -2348,7 +2338,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo);
   assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
 
-  strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name));
+  tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
   memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen);  // tag information if table does not exists.
   pNew->cmd.payloadLen = pSql->cmd.payloadLen;
   tscTrace("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
@@ -2358,7 +2348,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
 
   int32_t code = tscProcessSql(pNew);
   if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;
+    code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;  // notify upper application that current process need to be terminated
   }
 
   return code;
@@ -2389,56 +2379,26 @@ int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create
   return tscGetTableMeta(pSql, pTableMetaInfo);
 }
 
-/*
- * in handling the renew metermeta problem during insertion,
- *
- * If the meter is created on demand during insertion, the routine usually waits for a short
- * period to re-issue the getMeterMeta msg, in which makes a greater change that vnode has
- * successfully created the corresponding table.
- */
-static void tscWaitingForCreateTable(SSqlCmd *pCmd) {
-  if (pCmd->command == TSDB_SQL_INSERT) {
-    taosMsleep(50);  // todo: global config
-  }
-}
-
 /**
- * in renew metermeta, do not retrieve metadata in cache.
+ * retrieve table meta from mnode, and update the local table meta cache.
  * @param pSql          sql object
- * @param tableId       meter id
+ * @param tableId       table full name
  * @return              status code
  */
-int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) {
+int tscRenewTableMeta(SSqlObj *pSql, char *tableId) {
-  int code = 0;
-
-  // handle table meta renew process
   SSqlCmd *pCmd = &pSql->cmd;
 
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  /*
-   * 1. only update the metermeta in force model metricmeta is not updated
-   * 2. if get metermeta failed, still get the metermeta
-   */
-  if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnSTable(pCmd)) {
   STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
   if (pTableMetaInfo->pTableMeta) {
-      tscTrace("%p update table meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql,
+    tscTrace("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql,
              tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->uid, pTableMeta);
   }
 
-    tscWaitingForCreateTable(pCmd);
   taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
-
+  return getTableMetaFromMgmt(pSql, pTableMetaInfo);
-    code = getTableMetaFromMgmt(pSql, pTableMetaInfo);  // todo ??
-  } else {
-    tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql,
-             tscGetNumOfTags(pTableMetaInfo->pTableMeta), pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid,
-             pTableMetaInfo->pTableMeta);
-  }
-
-  return code;
 }
 
 static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) {

src/client/src/tscSql.c

@@ -86,21 +86,21 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   pObj->signature = pObj;
 
-  strncpy(pObj->user, user, TSDB_USER_LEN);
+  tstrncpy(pObj->user, user, sizeof(pObj->user));
   taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass);
 
   if (db) {
     int32_t len = strlen(db);
     /* db name is too long */
-    if (len > TSDB_DB_NAME_LEN) {
+    if (len >= TSDB_DB_NAME_LEN) {
       terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
       rpcClose(pDnodeConn);
       free(pObj);
       return NULL;
     }
 
-    char tmp[TSDB_DB_NAME_LEN + 1] = {0};
+    char tmp[TSDB_DB_NAME_LEN] = {0};
-    strcpy(tmp, db);
+    tstrncpy(tmp, db, sizeof(tmp));
 
     strdequote(tmp);
     strtolower(pObj->db, tmp);

src/client/src/tscSub.c

@@ -301,7 +301,9 @@ void tscSaveSubscriptionProgress(void* sub) {
   char path[256];
   sprintf(path, "%s/subscribe", tsDataDir);
   if (access(path, 0) != 0) {
-    mkdir(path, 0777);
+    if (mkdir(path, 0777) != 0 && errno != EEXIST) {
+      tscError("failed to create subscribe dir: %s", path);
+    }
   }
 
   sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);

src/client/src/tscSubquery.c

@@ -1921,7 +1921,7 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId)
   
   for (int32_t i = 0; i < pSupport->numOfCols; ++i) {
     pExpr = taosArrayGetP(pSupport->exprList, i);
-    if (strncmp(name, pExpr->aliasName, TSDB_COL_NAME_LEN) == 0) {
+    if (strncmp(name, pExpr->aliasName, sizeof(pExpr->aliasName) - 1) == 0) {
       index = i;
       break;
     }

src/client/src/tscSystem.c

@@ -81,7 +81,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) {
 
 void taos_init_imp() {
   char temp[128];
-  struct stat dirstat;
   
   errno = TSDB_CODE_SUCCESS;
   srand(taosGetTimestampSec());
@@ -94,7 +93,9 @@ void taos_init_imp() {
     taosReadGlobalLogCfg();
 
     // For log directory
-    if (stat(tsLogDir, &dirstat) < 0) mkdir(tsLogDir, 0755);
+    if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) {
+      printf("failed to create log dir:%s\n", tsLogDir);
+    }
 
     sprintf(temp, "%s/taoslog", tsLogDir);
     if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) {
@@ -143,11 +144,11 @@ void taos_init_imp() {
   }
   
   int64_t refreshTime = tsTableMetaKeepTimer;
-  refreshTime = refreshTime > 2 ? 2 : refreshTime;
+  refreshTime = refreshTime > 10 ? 10 : refreshTime;
-  refreshTime = refreshTime < 1 ? 1 : refreshTime;
+  refreshTime = refreshTime < 10 ? 10 : refreshTime;
 
   if (tscCacheHandle == NULL) {
-    tscCacheHandle = taosCacheInit(tscTmr, refreshTime);
+    tscCacheHandle = taosCacheInit(refreshTime);
   }
 
   tscTrace("client is initialized successfully");
@@ -179,7 +180,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       assert(cfg != NULL);
     
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        strncpy(configDir, pStr, TSDB_FILENAME_LEN);
+        tstrncpy(configDir, pStr, TSDB_FILENAME_LEN);
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscPrint("set config file directory:%s", pStr);
       } else {
@@ -200,7 +201,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
         tscPrint("set shellActivityTimer:%d", tsShellActivityTimer);
       } else {
         tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr,
-                tsCfgStatusStr[cfg->cfgStatus], (int32_t *)cfg->ptr);
+                tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
       }
       break;
 
@@ -233,7 +234,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
           tscPrint("failed to set locale:%s, current locale:%s", pStr, tsLocale);
         }
 
-        strncpy(tsLocale, locale, tListLen(tsLocale));
+        tstrncpy(tsLocale, locale, sizeof(tsLocale));
 
         char *charset = strrchr(tsLocale, sep);
         if (charset != NULL) {
@@ -248,7 +249,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
               tscPrint("charset changed from %s to %s", tsCharset, charset);
             }
 
-            strncpy(tsCharset, charset, tListLen(tsCharset));
+            tstrncpy(tsCharset, charset, sizeof(tsCharset));
             cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
 
           } else {
@@ -285,7 +286,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
             tscPrint("charset changed from %s to %s", tsCharset, pStr);
           }
 
-          strncpy(tsCharset, pStr, tListLen(tsCharset));
+          tstrncpy(tsCharset, pStr, sizeof(tsCharset));
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         } else {
           tscPrint("charset:%s not valid", pStr);
@@ -323,7 +324,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
 //          return -1;
 //        }
 
-        strncpy(tsSocketType, pStr, tListLen(tsSocketType));
+        tstrncpy(tsSocketType, pStr, sizeof(tsSocketType));
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscPrint("socket type is set:%s", tsSocketType);
       }

src/client/src/tscUtil.c

@@ -593,7 +593,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
   dataBuf->size = startOffset;
   dataBuf->tsSource = -1;
 
-  strncpy(dataBuf->tableId, name, TSDB_TABLE_ID_LEN);
+  tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId));
 
   /*
    * The table meta may be released since the table meta cache are completed clean by other thread
@@ -801,7 +801,7 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) {
 
 TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) {
   TAOS_FIELD f = { .type = type, .bytes = bytes, };
-  strncpy(f.name, name, TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, name, sizeof(f.name));
   return f;
 }
 
@@ -966,12 +966,12 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
     if (isTagCol) {
       SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
       pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId;
-      strncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, TSDB_COL_NAME_LEN);
+      tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name));
     } else if (pTableMetaInfo->pTableMeta != NULL) {
       // in handling select database/version/server_status(), the pTableMeta is NULL
       SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex);
       pExpr->colInfo.colId = pSchema->colId;
-      strncpy(pExpr->colInfo.name, pSchema->name, TSDB_COL_NAME_LEN);
+      tstrncpy(pExpr->colInfo.name, pSchema->name, sizeof(pExpr->colInfo.name));
     }
   }
   
@@ -1666,7 +1666,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
   assert(pTableMetaInfo != NULL);
 
   if (name != NULL) {
-    strncpy(pTableMetaInfo->name, name, TSDB_TABLE_ID_LEN);
+    tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name));
   }
 
   pTableMetaInfo->pTableMeta = pTableMeta;

src/common/inc/tdataformat.h

@@ -31,7 +31,7 @@ extern "C" {
   do {                                    \
     VarDataLenT __len = strlen(str);      \
     *(VarDataLenT *)(x) = __len;          \
-    strncpy(varDataVal(x), (str), __len); \
+    memcpy(varDataVal(x), (str), __len); \
   } while (0);
 
 #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)      \
@@ -43,7 +43,7 @@ extern "C" {
 #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \
   do {                                         \
     *(VarDataLenT *)(x) = (_size);             \
-    strncpy(varDataVal(x), (str), (_size));    \
+    memcpy(varDataVal(x), (str), (_size));    \
   } while (0);
 
 // ----------------- TSDB COLUMN DEFINITION
@@ -69,7 +69,8 @@ typedef struct {
   int      version;    // version
   int      numOfCols;  // Number of columns appended
   int      tlen;       // maximum length of a SDataRow without the header part
-  int      flen;       // First part length in a SDataRow after the header part
+  int16_t  flen;       // First part length in a SDataRow after the header part
+  int16_t  vlen;       // pure value part length, excluded the overhead
   STColumn columns[];
 } STSchema;
 
@@ -77,6 +78,7 @@ typedef struct {
 #define schemaVersion(s) ((s)->version)
 #define schemaTLen(s) ((s)->tlen)
 #define schemaFLen(s) ((s)->flen)
+#define schemaVLen(s) ((s)->vlen)
 #define schemaColAt(s, i) ((s)->columns + i)
 #define tdFreeSchema(s) tfree((s))
 
@@ -105,7 +107,8 @@ typedef struct {
   int       tCols;
   int       nCols;
   int       tlen;
-  int       flen;
+  int16_t   flen;
+  int16_t   vlen;
   int       version;
   STColumn *columns;
 } STSchemaBuilder;

src/common/src/tdataformat.c

@@ -100,6 +100,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
   pBuilder->nCols = 0;
   pBuilder->tlen = 0;
   pBuilder->flen = 0;
+  pBuilder->vlen = 0;
   pBuilder->version = version;
 }
 
@@ -124,10 +125,12 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int3
 
   if (IS_VAR_DATA_TYPE(type)) {
     colSetBytes(pCol, bytes);
-    pBuilder->tlen += (TYPE_BYTES[type] + sizeof(VarDataLenT) + bytes);
+    pBuilder->tlen += (TYPE_BYTES[type] + bytes);
+    pBuilder->vlen += bytes - sizeof(VarDataLenT);
   } else {
     colSetBytes(pCol, TYPE_BYTES[type]);
     pBuilder->tlen += TYPE_BYTES[type];
+    pBuilder->vlen += TYPE_BYTES[type];
   }
 
   pBuilder->nCols++;
@@ -150,6 +153,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
   schemaNCols(pSchema) = pBuilder->nCols;
   schemaTLen(pSchema) = pBuilder->tlen;
   schemaFLen(pSchema) = pBuilder->flen;
+  schemaVLen(pSchema) = pBuilder->vlen;
 
   memcpy(schemaColAt(pSchema, 0), pBuilder->columns, sizeof(STColumn) * pBuilder->nCols);
 
@@ -269,8 +273,7 @@ void dataColSetNullAt(SDataCol *pCol, int index) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
     char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
-    varDataLen(ptr) = (pCol->type == TSDB_DATA_TYPE_BINARY) ? sizeof(char) : TSDB_NCHAR_SIZE;
+    setVardataNull(ptr, pCol->type);
-    setNull(varDataVal(ptr), pCol->type, pCol->bytes);
     pCol->len += varDataTLen(ptr);
   } else {
     setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);

src/common/src/tglobal.c

@@ -708,7 +708,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_DB_NAME_LEN;
+  cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -923,7 +923,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_DB_NAME_LEN;
+  cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 

src/common/src/tname.c

@@ -28,8 +28,7 @@ void extractTableName(const char* tableId, char* name) {
   size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]);
   size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]);
   
-  strncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
+  tstrncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
-  name[TSDB_TABLE_NAME_LEN] = 0;
 }
 
 char* extractDBName(const char* tableId, char* name) {

src/connector/python/linux/python2/taos/cursor.py

@@ -137,7 +137,7 @@ class TDengineCursor(object):
         else:
             raise ProgrammingError(
                 CTaosInterface.errStr(
-                    self._result ))
+                    self._result ), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.

src/connector/python/linux/python3/taos/cinterface.py

@@ -130,9 +130,9 @@ _CONVERT_FUNC = {
 
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
-    _fields_ = [('name', ctypes.c_char * 64),
+    _fields_ = [('name', ctypes.c_char * 65),
-                ('bytes', ctypes.c_short),
+                ('type', ctypes.c_char),
-                ('type', ctypes.c_char)]
+                ('bytes', ctypes.c_short)]
 
 # C interface class
 class CTaosInterface(object):

src/connector/python/linux/python3/taos/cursor.py

@@ -139,7 +139,7 @@ class TDengineCursor(object):
         else:
             raise ProgrammingError(
                 CTaosInterface.errStr(
-                    self._result ))
+                    self._result), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.

src/connector/python/windows/python2/taos/cursor.py

@@ -117,7 +117,7 @@ class TDengineCursor(object):
                 self._fields = CTaosInterface.useResult(self._result)
                 return self._handle_result()
         else:
-            raise ProgrammingError(CTaosInterface.errStr(self._result))
+            raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.

src/connector/python/windows/python3/taos/cursor.py

@@ -117,7 +117,7 @@ class TDengineCursor(object):
                 self._fields = CTaosInterface.useResult(self._result )
                 return self._handle_result()
         else:
-            raise ProgrammingError(CTaosInterface.errStr(self._result ))
+            raise ProgrammingError(CTaosInterface.errStr(self._result ), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.

src/cq/src/cqMain.c

@@ -73,8 +73,8 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
     return NULL;
   }
 
-  strcpy(pContext->user, pCfg->user);
+  tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user));
-  strcpy(pContext->pass, pCfg->pass);
+  tstrncpy(pContext->pass, pCfg->pass, sizeof(pContext->pass));
   const char* db = pCfg->db;
   for (const char* p = db; *p != 0; p++) {
     if (*p == '.') {
@@ -82,7 +82,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
       break;
     }
   }
-  strcpy(pContext->db, db);
+  tstrncpy(pContext->db, db, sizeof(pContext->db));
   pContext->vgId = pCfg->vgId;
   pContext->cqWrite = pCfg->cqWrite;
   pContext->ahandle = ahandle;
@@ -215,7 +215,7 @@ void cqDrop(void *handle) {
   cTrace("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); 
   free(pObj);
 
-  pthread_mutex_lock(&pContext->mutex);
+  pthread_mutex_unlock(&pContext->mutex);
 }
 
 static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {

src/dnode/src/dnodeMain.c

@@ -61,12 +61,10 @@ static const SDnodeComponent tsDnodeComponents[] = {
 };
 
 static int dnodeCreateDir(const char *dir) {
-  struct stat dirstat;
-  if (stat(dir, &dirstat) < 0) {
   if (mkdir(dir, 0755) != 0 && errno != EEXIST) {
     return -1;
   }
-  }
+  
   return 0;
 }
 

src/dnode/src/dnodeMgmt.c

@@ -177,10 +177,12 @@ void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
     memcpy(item, pMsg, sizeof(SRpcMsg));
     taosWriteQitem(tsMgmtQueue, 1, item);
   } else {
-    SRpcMsg  rsp;
+    SRpcMsg rsp = {
-    rsp.handle = pMsg->handle;
+      .handle = pMsg->handle,
-    rsp.pCont  = NULL;
+      .pCont  = NULL,
-    rsp.code = TSDB_CODE_DND_OUT_OF_MEMORY;
+      .code   = TSDB_CODE_DND_OUT_OF_MEMORY
+    };
+    
     rpcSendResponse(&rsp);
     rpcFreeCont(pMsg->pCont);
   }
@@ -188,9 +190,9 @@ void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
 
 static void *dnodeProcessMgmtQueue(void *param) {
   SRpcMsg *pMsg;
-  SRpcMsg  rsp;
+  SRpcMsg  rsp = {0};
   int      type;
-  void    *handle;
+  void *   handle;
 
   while (1) {
     if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
@@ -251,6 +253,7 @@ static int32_t dnodeOpenVnodes() {
 
   if (status != TSDB_CODE_SUCCESS) {
     dPrint("Get dnode list failed");
+    free(vnodeList);
     return status;
   }
 
@@ -290,6 +293,7 @@ static void dnodeCloseVnodes() {
 
   if (status != TSDB_CODE_SUCCESS) {
     dPrint("Get dnode list failed");
+    free(vnodeList);
     return;
   }
 
@@ -456,6 +460,7 @@ static bool dnodeReadMnodeInfos() {
     return false;
   }
 
+  content[len] = 0;
   cJSON* root = cJSON_Parse(content);
   if (root == NULL) {
     dError("failed to read mnodeIpList.json, invalid json format");
@@ -628,6 +633,7 @@ static bool dnodeReadDnodeCfg() {
     return false;
   }
 
+  content[len] = 0;
   cJSON* root = cJSON_Parse(content);
   if (root == NULL) {
     dError("failed to read dnodeCfg.json, invalid json format");

src/dnode/src/dnodePeer.c

@@ -83,10 +83,11 @@ void dnodeCleanupServer() {
 }
 
 static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
-  SRpcMsg rspMsg;
+  SRpcMsg rspMsg = {
-  rspMsg.handle  = pMsg->handle;
+    .handle  = pMsg->handle,
-  rspMsg.pCont   = NULL;
+    .pCont   = NULL,
-  rspMsg.contLen = 0;
+    .contLen = 0
+  };
   
   if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
     rspMsg.code = TSDB_CODE_RPC_NOT_READY;

src/dnode/src/dnodeShell.c

@@ -19,6 +19,7 @@
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tglobal.h"
+#include "tutil.h"
 #include "http.h"
 #include "mnode.h"
 #include "dnode.h"
@@ -108,10 +109,11 @@ void dnodeCleanupShell() {
 }
 
 void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
-  SRpcMsg rpcMsg;
+  SRpcMsg rpcMsg = {
-  rpcMsg.handle = pMsg->handle;
+    .handle  = pMsg->handle,
-  rpcMsg.pCont = NULL;
+    .pCont   = NULL,
-  rpcMsg.contLen = 0;
+    .contLen = 0
+  };
 
   if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
     dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
@@ -143,7 +145,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
   if (code != TSDB_CODE_RPC_NOT_READY) return code;
 
   SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
-  strcpy(pMsg->user, user);
+  tstrncpy(pMsg->user, user, TSDB_USER_LEN);
 
   SRpcMsg rpcMsg = {0};
   rpcMsg.pCont = pMsg;
@@ -201,7 +203,7 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid) {
     int16_t   numOfTags = htons(pTable->numOfTags);
     int32_t   sid = htonl(pTable->sid);
     uint64_t  uid = htobe64(pTable->uid);
-    dPrint("table:%s, numOfColumns:%d numOfTags:%d sid:%d uid:%d", pTable->tableId, numOfColumns, numOfTags, sid, uid);
+    dPrint("table:%s, numOfColumns:%d numOfTags:%d sid:%d uid:%" PRIu64, pTable->tableId, numOfColumns, numOfTags, sid, uid);
 
     return rpcRsp.pCont;
   }

src/dnode/src/dnodeSystem.c

@@ -33,7 +33,7 @@ int32_t main(int32_t argc, char *argv[]) {
           printf("config file path overflow");
           exit(EXIT_FAILURE);
         }
-        strcpy(configDir, argv[i]);
+        tstrncpy(configDir, argv[i], TSDB_FILENAME_LEN);
       } else {
         printf("'-c' requires a parameter, default:%s\n", configDir);
         exit(EXIT_FAILURE);
@@ -92,6 +92,7 @@ int32_t main(int32_t argc, char *argv[]) {
   // Initialize the system
   if (dnodeInitSystem() < 0) {
     syslog(LOG_ERR, "Error initialize TDengine system");
+    dPrint("Failed to start TDengine, please check the log at:%s", tsLogDir);
     closelog();
     exit(EXIT_FAILURE);
   }

src/inc/taos.h

@@ -53,9 +53,9 @@ typedef enum {
 } TSDB_OPTION;
 
 typedef struct taosField {
-  char     name[64];
+  char     name[65];
-  short    bytes;
   uint8_t  type;
+  short    bytes;
 } TAOS_FIELD;
 
 #ifdef _TD_GO_DLL_

src/inc/taosdef.h

@@ -189,7 +189,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_USERID_LEN           9
 #define TS_PATH_DELIMITER_LEN     1
 
-#define TSDB_METER_ID_LEN_MARGIN  10
+#define TSDB_METER_ID_LEN_MARGIN  8
 #define TSDB_TABLE_ID_LEN         (TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*TS_PATH_DELIMITER_LEN+TSDB_USERID_LEN+TSDB_METER_ID_LEN_MARGIN) //TSDB_DB_NAME_LEN+TSDB_TABLE_NAME_LEN+2*strlen(TS_PATH_DELIMITER)+strlen(USERID)
 #define TSDB_UNI_LEN              24
 #define TSDB_USER_LEN             TSDB_UNI_LEN
@@ -200,9 +200,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MIN_COLUMNS          2       //PRIMARY COLUMN(timestamp) + other columns
 
 #define TSDB_NODE_NAME_LEN        64
-#define TSDB_TABLE_NAME_LEN       192
+#define TSDB_TABLE_NAME_LEN       193
-#define TSDB_DB_NAME_LEN          32
+#define TSDB_DB_NAME_LEN          33
-#define TSDB_COL_NAME_LEN         64
+#define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
 #define TSDB_MAX_SQL_SHOW_LEN     256

src/inc/taoserror.h

@@ -136,16 +136,17 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS,           0, 0x0355, "mnode too
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST,      0, 0x0360, "mnode table already exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID,         0, 0x0361, "mnode invalid table id")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE,       0, 0x0362, "mnode invalid table type")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME,       0, 0x0362, "mnode invalid table name")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS,            0, 0x0363, "mnode too many tags")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE,       0, 0x0363, "mnode invalid table type")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TABLES,          0, 0x0364, "mnode too many tables")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS,            0, 0x0364, "mnode too many tags")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES,      0, 0x0365, "mnode not enough time series")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TABLES,          0, 0x0365, "mnode too many tables")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE,          0, 0x0366, "mnode no super table")           // operation only available for super table
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES,      0, 0x0366, "mnode not enough time series")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG,        0, 0x0367, "mnode column name too long")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE,          0, 0x0367, "mnode no super table")           // operation only available for super table
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST,         0, 0x0368, "mnode tag already exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG,        0, 0x0368, "mnode column name too long")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST,            0, 0x0369, "mnode tag not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST,         0, 0x0369, "mnode tag already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST,       0, 0x036A, "mnode field already exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST,            0, 0x036A, "mnode tag not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST,          0, 0x036B, "mnode field not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST,       0, 0x036B, "mnode field already exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST,          0, 0x036C, "mnode field not exist")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED,          0, 0x0380, "mnode db not selected")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST,         0, 0x0381, "mnode database aleady exist")

src/inc/taosmsg.h

@@ -224,7 +224,7 @@ typedef struct {
 
 typedef struct SSchema {
   uint8_t type;
-  char    name[TSDB_COL_NAME_LEN + 1];
+  char    name[TSDB_COL_NAME_LEN];
   int16_t colId;
   int16_t bytes;
 } SSchema;
@@ -250,7 +250,7 @@ typedef struct {
 
 typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN + 1];
-  char    db[TSDB_DB_NAME_LEN + 1];
+  char    db[TSDB_DB_NAME_LEN];
   int8_t  igExists;
   int8_t  getMeta;
   int16_t numOfTags;
@@ -268,7 +268,7 @@ typedef struct {
 
 typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN + 1];
-  char    db[TSDB_DB_NAME_LEN + 1];
+  char    db[TSDB_DB_NAME_LEN];
   int16_t type; /* operation type   */
   int16_t numOfCols; /* number of schema */
   int32_t tagValLen;
@@ -296,7 +296,7 @@ typedef struct {
 } SCMConnectMsg;
 
 typedef struct {
-  char      acctId[TSDB_ACCT_LEN + 1];
+  char      acctId[TSDB_ACCT_LEN];
   char      serverVersion[TSDB_VERSION_LEN];
   int8_t    writeAuth;
   int8_t    superAuth;
@@ -502,7 +502,7 @@ typedef struct {
 
 typedef struct {
   char     acct[TSDB_USER_LEN + 1];
-  char     db[TSDB_DB_NAME_LEN + 1];
+  char     db[TSDB_DB_NAME_LEN];
   int32_t  cacheBlockSize; //MB
   int32_t  totalBlocks;
   int32_t  maxTables;
@@ -606,7 +606,7 @@ typedef struct {
 } SMDVnodeDesc;
 
 typedef struct {
-  char db[TSDB_DB_NAME_LEN + 1];
+  char db[TSDB_DB_NAME_LEN];
   SMDVnodeCfg  cfg;
   SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
 } SMDCreateVnodeMsg;
@@ -671,7 +671,7 @@ typedef struct {
  */
 typedef struct {
   int8_t   type;
-  char     db[TSDB_DB_NAME_LEN + 1];
+  char     db[TSDB_DB_NAME_LEN];
   uint16_t payloadLen;
   char     payload[];
 } SCMShowMsg;

src/inc/tcq.h

@@ -27,7 +27,7 @@ typedef struct {
   int      vgId;
   char     user[TSDB_USER_LEN];
   char     pass[TSDB_PASSWORD_LEN];
-  char     db[TSDB_DB_NAME_LEN + 1];
+  char     db[TSDB_DB_NAME_LEN];
   FCqWrite cqWrite;
 } SCqCfg;
 

src/inc/tsdb.h

@@ -65,6 +65,13 @@ typedef struct {
   int8_t  compression;
 } STsdbCfg;
 
+// --------- TSDB REPOSITORY USAGE STATISTICS
+typedef struct {
+  int64_t totalStorage;  // total bytes occupie
+  int64_t compStorage;
+  int64_t pointsWritten;  // total data points written
+} STsdbStat;
+
 typedef void TsdbRepoT;  // use void to hide implementation details from outside
 
 void      tsdbSetDefaultCfg(STsdbCfg *pCfg);
@@ -306,6 +313,15 @@ int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGr
  */
 void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
 
+/**
+ * get the statistics of repo usage
+ * @param repo. point to the tsdbrepo
+ * @param totalPoints. total data point written
+ * @param totalStorage. total bytes took by the tsdb
+ * @param compStorage. total bytes took by the tsdb after compressed
+ */
+void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
+
 #ifdef __cplusplus
 }
 #endif

src/kit/shell/src/shellEngine.c

@@ -727,10 +727,6 @@ void read_history() {
   char f_history[TSDB_FILENAME_LEN];
   get_history_path(f_history);
 
-  if (access(f_history, R_OK) == -1) {
-    return;
-  }
-
   FILE *f = fopen(f_history, "r");
   if (f == NULL) {
     fprintf(stderr, "Opening file %s\n", f_history);
@@ -809,14 +805,6 @@ void source_file(TAOS *con, char *fptr) {
     return;
   }
   
-  if (access(fname, R_OK) != 0) {
-    fprintf(stderr, "ERROR: file %s is not readable\n", fptr);
-  
-    wordfree(&full_path);
-    free(cmd);
-    return;
-  }
-
   FILE *f = fopen(fname, "r");
   if (f == NULL) {
     fprintf(stderr, "ERROR: failed to open file %s\n", fname);

src/kit/shell/src/shellImport.c

@@ -148,6 +148,10 @@ static void shellSourceFile(TAOS *con, char *fptr) {
   }
 
   char *fname = full_path.we_wordv[0];
+  if (fname == NULL) {
+    fprintf(stderr, "ERROR: invalid filename\n");
+    return;
+  }
 
   if (access(fname, F_OK) != 0) {
     fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
@@ -169,6 +173,7 @@ static void shellSourceFile(TAOS *con, char *fptr) {
   if (f == NULL) {
     fprintf(stderr, "ERROR: failed to open file %s\n", fname);
     wordfree(&full_path);
+    free(cmd);
     return;
   }
 

src/kit/taosdemo/taosdemo.c

@@ -43,6 +43,7 @@ extern char configDir[];
 #define MAX_DATA_SIZE    1024
 #define MAX_NUM_DATATYPE 8
 #define OPT_ABORT        1 /* –abort */
+#define STRING_LEN       512
 
 /* The options we understand. */
 static struct argp_option options[] = {
@@ -380,10 +381,11 @@ int main(int argc, char *argv[]) {
   bool insert_only = arguments.insert_only;
   char **data_type = arguments.datatype;
   int count_data_type = 0;
-  char dataString[512];
+  char dataString[STRING_LEN];
   bool do_aggreFunc = true;
 
-  memset(dataString, 0, 512);
+  memset(dataString, 0, STRING_LEN);
+  int len = 0;
 
   if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0) {
     do_aggreFunc = false;
@@ -392,8 +394,8 @@ int main(int argc, char *argv[]) {
     if (strcasecmp(data_type[count_data_type], "") == 0) {
       break;
     }
-    strcat(dataString, data_type[count_data_type]);
+
-    strcat(dataString, " ");
+    len += snprintf(dataString + len, STRING_LEN - len, "%s ", data_type[count_data_type]);
   }
 
   FILE *fp = fopen(arguments.output_file, "a");
@@ -473,32 +475,29 @@ int main(int argc, char *argv[]) {
   sprintf(command, "create database %s;", db_name);
   taos_query(taos, command);
 
-  char cols[512] = "\0";
+  char cols[STRING_LEN] = "\0";
   int colIndex = 0;
+  len = 0;
 
   for (; colIndex < ncols_per_record - 1; colIndex++) {
     if (strcasecmp(data_type[colIndex % count_data_type], "BINARY") != 0) {
-      sprintf(command, ",f%d %s", colIndex + 1, data_type[colIndex % count_data_type]);
+      len += snprintf(cols + len, STRING_LEN - len, ",f%d %s", colIndex + 1, data_type[colIndex % count_data_type]);
-      strcat(cols, command);
     } else {
-      sprintf(command, ",f%d %s(%d)", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary);
+      len += snprintf(cols + len, STRING_LEN - len, ",f%d %s(%d)", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary);
-      strcat(cols, command);
     }
   }
 
   if (strcasecmp(data_type[colIndex % count_data_type], "BINARY") != 0) {
-    sprintf(command, ",f%d %s)", colIndex + 1, data_type[colIndex % count_data_type]);
+    len += snprintf(cols + len, STRING_LEN - len, ",f%d %s)", colIndex + 1, data_type[colIndex % count_data_type]);
   } else {
-    sprintf(command, ",f%d %s(%d))", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary);
+    len += snprintf(cols + len, STRING_LEN - len, ",f%d %s(%d))", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary);
   }
 
-  strcat(cols, command);
-
   if (!use_metric) {
     /* Create all the tables; */
     printf("Creating %d table(s)......\n", ntables);
     for (int i = 0; i < ntables; i++) {
-      sprintf(command, "create table %s.%s%d (ts timestamp%s;", db_name, tb_prefix, i, cols);
+      snprintf(command, BUFFER_SIZE, "create table %s.%s%d (ts timestamp%s;", db_name, tb_prefix, i, cols);
       queryDB(taos, command);
     }
 
@@ -508,7 +507,7 @@ int main(int argc, char *argv[]) {
   } else {
     /* Create metric table */
     printf("Creating meters super table...\n");
-    sprintf(command, "create table %s.meters (ts timestamp%s tags (areaid int, loc binary(10))", db_name, cols);
+    snprintf(command, BUFFER_SIZE, "create table %s.meters (ts timestamp%s tags (areaid int, loc binary(10))", db_name, cols);
     queryDB(taos, command);
     printf("meters created!\n");
 
@@ -522,9 +521,9 @@ int main(int argc, char *argv[]) {
         j = i % 10;
       }
     if (j % 2 == 0) {
-       sprintf(command, "create table %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j,"shanghai");
+      snprintf(command, BUFFER_SIZE, "create table %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j, "shanghai");
     } else {
-       sprintf(command, "create table %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j,"beijing");
+      snprintf(command, BUFFER_SIZE, "create table %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j, "beijing");
     }
       queryDB(taos, command);
     }
@@ -711,7 +710,7 @@ void *readTable(void *sarg) {
       int32_t code = taos_errno(pSql);
 
       if (code != 0) {
-        fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos));
+        fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql));
         taos_free_result(pSql);
         taos_close(taos);
         exit(EXIT_FAILURE);
@@ -757,7 +756,7 @@ void *readMetric(void *sarg) {
 
   for (int j = 0; j < n; j++) {
     char condition[BUFFER_SIZE - 30] = "\0";
-    char tempS[BUFFER_SIZE] = "\0";
+    char tempS[64] = "\0";
 
     int m = 10 < num_of_tables ? 10 : num_of_tables;
 
@@ -780,7 +779,7 @@ void *readMetric(void *sarg) {
       int32_t code = taos_errno(pSql);
 
       if (code != 0) {
-        fprintf(stderr, "Failed to query:%s\n", taos_errstr(taos));
+        fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql));
         taos_free_result(pSql);
         taos_close(taos);
         exit(1);
@@ -819,7 +818,9 @@ void queryDB(TAOS *taos, char *command) {
   }
 
   if (i == 0) {
-    fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(taos));
+    fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
+    taos_free_result(pSql);
+
     taos_close(taos);
     exit(EXIT_FAILURE);
   }
@@ -915,7 +916,7 @@ void callBack(void *param, TAOS_RES *res, int code) {
   int64_t tmp_time = tb_info->timestamp;
 
   if (code < 0) {
-    fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(tb_info->taos));
+    fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(res));
     exit(EXIT_FAILURE);
   }
 

src/kit/taosdump/taosdump.c

@@ -88,21 +88,21 @@ enum _describe_table_index {
 };
 
 typedef struct {
-  char field[TSDB_COL_NAME_LEN + 1];
+  char field[TSDB_COL_NAME_LEN];
   char type[16];
   int length;
   char note[128];
 } SColDes;
 
 typedef struct {
-  char name[TSDB_COL_NAME_LEN + 1];
+  char name[TSDB_COL_NAME_LEN];
   SColDes cols[];
 } STableDef;
 
 extern char version[];
 
 typedef struct {
-  char name[TSDB_DB_NAME_LEN + 1];
+  char name[TSDB_DB_NAME_LEN];
   int32_t replica;
   int32_t days;
   int32_t keep;
@@ -117,8 +117,8 @@ typedef struct {
 } SDbInfo;
 
 typedef struct {
-  char name[TSDB_TABLE_NAME_LEN + 1];
+  char name[TSDB_TABLE_NAME_LEN];
-  char metric[TSDB_TABLE_NAME_LEN + 1];
+  char metric[TSDB_TABLE_NAME_LEN];
 } STableRecord;
 
 typedef struct {
@@ -643,13 +643,14 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
   lseek(fd, 0, SEEK_SET);
 
   while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
+    tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
+    tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
     taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
   }
 
-  tclose(fd);
+  close(fd);
-  remove(".table.tmp");
 
-  return 0;
+  return remove(".table.tmp");
 }
 
 void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) {
@@ -871,7 +872,7 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
   int fd = -1;
   STableRecord tableRecord;
 
-  strcpy(tableRecord.metric, metric);
+  tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
 
   sprintf(command, "select tbname from %s", metric);
   result = taos_query(taos, command);
@@ -903,6 +904,8 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
   lseek(fd, 0, SEEK_SET);
 
   while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) {
+    tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
+    tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
     taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
   }
 

src/mnode/inc/mnodeDef.h

@@ -32,8 +32,8 @@ struct SMnodeObj;
 typedef struct SDnodeObj {
   int32_t    dnodeId;
   uint16_t   dnodePort;
-  char       dnodeFqdn[TSDB_FQDN_LEN + 1];
+  char       dnodeFqdn[TSDB_FQDN_LEN];
-  char       dnodeEp[TSDB_EP_LEN + 1];
+  char       dnodeEp[TSDB_EP_LEN];
   int64_t    createdTime;
   uint32_t   lastAccess;
   int32_t    openVnodes;
@@ -115,7 +115,7 @@ typedef struct {
 
 typedef struct SVgObj {
   uint32_t       vgId;
-  char           dbName[TSDB_DB_NAME_LEN + 1];
+  char           dbName[TSDB_DB_NAME_LEN];
   int64_t        createdTime;
   SVnodeGid      vnodeGid[TSDB_MAX_REPLICA];
   int32_t        numOfVnodes;
@@ -154,7 +154,7 @@ typedef struct {
 } SDbCfg;
 
 typedef struct SDbObj {
-  char    name[TSDB_DB_NAME_LEN + 1];
+  char    name[TSDB_DB_NAME_LEN];
   char    acct[TSDB_USER_LEN + 1];
   int64_t createdTime;
   int32_t cfgVersion;
@@ -219,7 +219,7 @@ typedef struct SAcctObj {
 typedef struct {
   int8_t   type;
   int32_t  index;
-  char     db[TSDB_DB_NAME_LEN + 1];
+  char     db[TSDB_DB_NAME_LEN];
   void *   pIter;
   int16_t  numOfColumns;
   int32_t  rowSize;

src/mnode/src/mnodeAcct.c

@@ -27,7 +27,7 @@
 
 void *  tsAcctSdb = NULL;
 static int32_t tsAcctUpdateSize;
-static void    mnodeCreateRootAcct();
+static int32_t mnodeCreateRootAcct();
 
 static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
@@ -79,7 +79,11 @@ static int32_t mnodeAcctActionDecode(SSdbOper *pOper) {
 
 static int32_t mnodeAcctActionRestored() {
   if (dnodeIsFirstDeploy()) {
-    mnodeCreateRootAcct();
+    int32_t code = mnodeCreateRootAcct();
+    if (code != TSDB_CODE_SUCCESS) {
+      mError("failed to create root account, reason:%s", tstrerror(code));
+      return code;
+    }
   }
 
   acctInit();
@@ -161,9 +165,9 @@ void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
   mnodeDecAcctRef(pAcct);
 }
 
-static void mnodeCreateRootAcct() {
+static int32_t mnodeCreateRootAcct() {
   int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb);
-  if (numOfAccts != 0) return;
+  if (numOfAccts != 0) return TSDB_CODE_SUCCESS;
 
   SAcctObj *pAcct = malloc(sizeof(SAcctObj));
   memset(pAcct, 0, sizeof(SAcctObj));
@@ -190,7 +194,8 @@ static void mnodeCreateRootAcct() {
     .table = tsAcctSdb,
     .pObj = pAcct,
   };
-  sdbInsertRow(&oper);
+
+  return sdbInsertRow(&oper);
 }
 
 #ifndef _ACCT

src/mnode/src/mnodeDb.c

@@ -328,8 +328,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
   if (code != 0) return code;
 
   pDb = calloc(1, sizeof(SDbObj));
-  strncpy(pDb->name, pCreate->db, TSDB_DB_NAME_LEN);
+  tstrncpy(pDb->name, pCreate->db, sizeof(pDb->name));
-  strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN); 
+  tstrncpy(pDb->acct, pAcct->user, sizeof(pDb->acct)); 
   pDb->createdTime = taosGetTimestampMs(); 
   pDb->cfg = (SDbCfg) {
     .cacheBlockSize      = pCreate->cacheBlockSize,
@@ -373,7 +373,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
 }
 
 bool mnodeCheckIsMonitorDB(char *db, char *monitordb) {
-  char dbName[TSDB_DB_NAME_LEN + 1] = {0};
+  char dbName[TSDB_DB_NAME_LEN] = {0};
   extractDBName(db, dbName);
 
   size_t len = strlen(dbName);
@@ -453,7 +453,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
-  pShow->bytes[cols] = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE;
+  pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
@@ -610,7 +610,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     
     char* name = mnodeGetDbStr(pDb->name);
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN - 1);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -882,7 +882,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     }
 
     if (pDb->cfg.replications - replications >= 2) {
-      mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications);
+      mError("db:%s, replica number can't change from %d to %d", pDb->name, pDb->cfg.replications, replications);
       terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
     }
   }

src/mnode/src/mnodeDnode.c

@@ -90,7 +90,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
 static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
   SDnodeObj *pSaved = mnodeGetDnode(pDnode->dnodeId);
-  if (pDnode != pSaved) {
+  if (pDnode != pSaved && pDnode != NULL && pSaved != NULL) {
     memcpy(pSaved, pDnode, pOper->rowSize);
     free(pDnode);
   }
@@ -237,7 +237,9 @@ void mnodeUpdateDnode(SDnodeObj *pDnode) {
     .pObj = pDnode
   };
 
-  sdbUpdateRow(&oper);
+  if (sdbUpdateRow(&oper) != 0) {
+    mError("dnodeId:%d, failed update", pDnode->dnodeId);
+  }
 }
 
 static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
@@ -383,7 +385,7 @@ static int32_t mnodeCreateDnode(char *ep) {
   pDnode->createdTime = taosGetTimestampMs();
   pDnode->status = TAOS_DN_STATUS_OFFLINE; 
   pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM; 
-  strcpy(pDnode->dnodeEp, ep);
+  tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN);
   taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
 
   SSdbOper oper = {

src/mnode/src/mnodeMain.c

@@ -88,9 +88,9 @@ int32_t mnodeStartSystem() {
   }
 
   mPrint("starting to initialize mnode ...");
-  struct stat dirstat;
+  if (mkdir(tsMnodeDir, 0755) != 0 && errno != EEXIST) {
-  if (stat(tsMnodeDir, &dirstat) < 0) {
+    mError("failed to init mnode dir:%s, reason:%s", tsMnodeDir, strerror(errno));
-    mkdir(tsMnodeDir, 0755);
+    return -1;
   }
 
   dnodeAllocateMnodeWqueue();
@@ -134,8 +134,12 @@ void mnodeStopSystem() {
   }
   
   mnodeCleanupSystem();
+
+  if (remove(tsMnodeDir) != 0) {
+    mPrint("failed to remove mnode file, reason:%s", strerror(errno));
+  } else {
     mPrint("mnode file is removed");
-  remove(tsMnodeDir);
+  }
 }
 
 static void mnodeInitTimer() {

src/mnode/src/mnodeProfile.c

@@ -67,7 +67,7 @@ int32_t mnodeInitProfile() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg);
 
-  tsMnodeConnCache = taosCacheInitWithCb(tsMnodeTmr, CONN_CHECK_TIME, mnodeFreeConn);
+  tsMnodeConnCache = taosCacheInitWithCb(CONN_CHECK_TIME, mnodeFreeConn);
   return 0;
 }
 
@@ -97,7 +97,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) {
     .connId = connId,
     .stime  = taosGetTimestampMs()
   };
-  strcpy(connObj.user, user);
+  tstrncpy(connObj.user, user, TSDB_USER_LEN);
   
   char key[10];
   sprintf(key, "%u", connId);  

src/mnode/src/mnodeSdb.c

@@ -30,6 +30,8 @@
 #include "mnodeDnode.h"
 #include "mnodeSdb.h"
 
+#define SDB_TABLE_LEN 12
+
 typedef enum {
   SDB_ACTION_INSERT,
   SDB_ACTION_DELETE,
@@ -43,7 +45,7 @@ typedef enum {
 } ESdbStatus;
 
 typedef struct _SSdbTable {
-  char      tableName[TSDB_DB_NAME_LEN + 1];
+  char      tableName[SDB_TABLE_LEN];
   ESdbTable tableId;
   ESdbKey   keyType;
   int32_t   hashSessions;
@@ -174,10 +176,10 @@ static void sdbRestoreTables() {
 
     totalRows += pTable->numOfRows;
     numOfTables++;
-    sdbTrace("table:%s, is restored, numOfRows:%d", pTable->tableName, pTable->numOfRows);
+    sdbTrace("table:%s, is restored, numOfRows:%" PRId64, pTable->tableName, pTable->numOfRows);
   }
 
-  sdbTrace("sdb is restored, version:%d totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
+  sdbTrace("sdb is restored, version:%" PRId64 " totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
 }
 
 void sdbUpdateMnodeRoles() {
@@ -449,7 +451,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
 
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%d version:%" PRIu64, pTable->tableName,
+  sdbTrace("table:%s, insert record:%s to hash, rowSize:%d vnumOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion());
 
   (*pTable->insertFp)(pOper);
@@ -473,7 +475,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
   pTable->numOfRows--;
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
+  sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d" PRId64 "version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
 
   int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
@@ -484,7 +486,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
 }
 
 static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
-  sdbTrace("table:%s, update record:%s in hash, numOfRows:%d version:%" PRIu64, pTable->tableName,
+  sdbTrace("table:%s, update record:%s in hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
 
   (*pTable->updateFp)(pOper);
@@ -717,7 +719,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
   
   if (pTable == NULL) return NULL;
 
-  strcpy(pTable->tableName, pDesc->tableName);
+  tstrncpy(pTable->tableName, pDesc->tableName, SDB_TABLE_LEN);
   pTable->keyType      = pDesc->keyType;
   pTable->tableId      = pDesc->tableId;
   pTable->hashSessions = pDesc->hashSessions;

src/mnode/src/mnodeShow.c

@@ -65,7 +65,7 @@ int32_t mnodeInitShow() {
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg);
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg);
   
-  tsMnodeShowCache = taosCacheInitWithCb(tsMnodeTmr, 10, mnodeFreeShowObj);
+  tsMnodeShowCache = taosCacheInitWithCb(10, mnodeFreeShowObj);
   return 0;
 }
 
@@ -316,7 +316,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
   }
 
   sprintf(pConnectRsp->acctId, "%x", pAcct->acctId);
-  strcpy(pConnectRsp->serverVersion, version);
+  memcpy(pConnectRsp->serverVersion, version, TSDB_VERSION_LEN);
   pConnectRsp->writeAuth = pUser->writeAuth;
   pConnectRsp->superAuth = pUser->superAuth;
   
@@ -324,6 +324,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
 
 connect_over:
   if (code != TSDB_CODE_SUCCESS) {
+    rpcFreeCont(pConnectRsp);
     mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));
   } else {
     mLPrint("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));

src/mnode/src/mnodeTable.c

@@ -714,7 +714,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
       return TSDB_CODE_SUCCESS;
     } else {
       mError("table:%s, failed to drop table, table not exist", pDrop->tableId);
-      return TSDB_CODE_MND_INVALID_TABLE_ID;
+      return TSDB_CODE_MND_INVALID_TABLE_NAME;
     }
   }
 
@@ -742,7 +742,7 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
   if (pMsg->pTable == NULL) {
     if (!pInfo->createFlag) {
       mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
-      return TSDB_CODE_MND_INVALID_TABLE_ID;
+      return TSDB_CODE_MND_INVALID_TABLE_NAME;
     } else {
       mTrace("table:%s, failed to get table meta, start auto create table ", pInfo->tableId);
       return mnodeAutoCreateChildTable(pMsg);
@@ -779,7 +779,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
   if (pStable->schema == NULL) {
     free(pStable);
     mError("table:%s, failed to create, no schema input", pCreate->tableId);
-    return TSDB_CODE_MND_INVALID_TABLE_ID;
+    return TSDB_CODE_MND_INVALID_TABLE_NAME;
   }
   memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema));
 
@@ -951,7 +951,7 @@ static int32_t mnodeModifySuperTableTagName(SSuperTableObj *pStable, char *oldTa
 
   // update
   SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col);
-  strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN);
+  tstrncpy(schema->name, newTagName, sizeof(schema->name));
 
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
@@ -1078,7 +1078,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
 
-  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
@@ -1138,7 +1138,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
   prefixLen = strlen(prefix);
 
   SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
-  char stableName[TSDB_TABLE_NAME_LEN + 1] = {0};
+  char stableName[TSDB_TABLE_NAME_LEN] = {0};
 
   while (numOfRows < rows) {    
     pShow->pIter = mnodeGetNextSuperTable(pShow->pIter, &pTable);
@@ -1151,7 +1151,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
     memset(stableName, 0, tListLen(stableName));
     mnodeExtractTableName(pTable->info.tableId, stableName);
 
-    if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
+    if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, sizeof(stableName) - 1, &info) != TSDB_PATTERN_MATCH) {
       mnodeDecTableRef(pTable);
       continue;
     }
@@ -1160,7 +1160,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
   
-    int16_t len = strnlen(stableName, TSDB_DB_NAME_LEN);
+    int16_t len = strnlen(stableName, TSDB_DB_NAME_LEN - 1);
     *(int16_t*) pWrite = len;
     pWrite += sizeof(int16_t); // todo refactor
   
@@ -1228,7 +1228,7 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT
   assert(numOfCols <= TSDB_MAX_COLUMNS);
   
   for (int32_t i = 0; i < numOfCols; ++i) {
-    strncpy(pSchema->name, pTable->schema[i].name, TSDB_COL_NAME_LEN);
+    tstrncpy(pSchema->name, pTable->schema[i].name, sizeof(pSchema->name));
     pSchema->type  = pTable->schema[i].type;
     pSchema->bytes = htons(pTable->schema[i].bytes);
     pSchema->colId = htons(pTable->schema[i].colId);
@@ -1340,7 +1340,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
 
   if (pRsp->numOfTables != numOfTable) {
     rpcFreeCont(pRsp);
-    return TSDB_CODE_MND_INVALID_TABLE_ID;
+    return TSDB_CODE_MND_INVALID_TABLE_NAME;
   } else {
     pRsp->numOfTables = htonl(pRsp->numOfTables);
     pMsg->rpcRsp.rsp = pRsp;
@@ -1452,7 +1452,7 @@ static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgOb
     if (pSuperTable == NULL) {
       mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name);
       mnodeDestroyChildTable(pTable);
-      terrno = TSDB_CODE_MND_INVALID_TABLE_ID;
+      terrno = TSDB_CODE_MND_INVALID_TABLE_NAME;
       return NULL;
     }
     mnodeDecTableRef(pSuperTable);
@@ -1769,8 +1769,8 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
-  strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId));
+  tstrncpy(pCreateMsg->tableId, pInfo->tableId, sizeof(pInfo->tableId));
-  strcpy(pCreateMsg->db, pMsg->pDb->name);
+  tstrncpy(pCreateMsg->db, pMsg->pDb->name, sizeof(pCreateMsg->db));
   pCreateMsg->igExists = 1;
   pCreateMsg->getMeta = 1;
   pCreateMsg->contLen = htonl(contLen);
@@ -2079,7 +2079,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
 
-  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "table_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
@@ -2097,7 +2097,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
-  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "stable_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
@@ -2141,12 +2141,12 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
       continue;
     }
 
-    char tableName[TSDB_TABLE_NAME_LEN + 1] = {0};
+    char tableName[TSDB_TABLE_NAME_LEN] = {0};
     
     // pattern compare for table name
     mnodeExtractTableName(pTable->info.tableId, tableName);
 
-    if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
+    if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
       mnodeDecTableRef(pTable);
       continue;
     }
@@ -2155,7 +2155,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
 
     char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
 
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -2173,10 +2173,10 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     
-    memset(tableName, 0, tListLen(tableName));
+    memset(tableName, 0, sizeof(tableName));
     if (pTable->info.type == TSDB_CHILD_TABLE) {
       mnodeExtractTableName(pTable->superTable->info.tableId, tableName);
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
     }
     
     cols++;
@@ -2212,7 +2212,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
   if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableId);
   if (pMsg->pTable == NULL) {
     mError("table:%s, failed to alter table, table not exist", pMsg->pTable->tableId);
-    return TSDB_CODE_MND_INVALID_TABLE_ID;
+    return TSDB_CODE_MND_INVALID_TABLE_NAME;
   }
 
   pAlter->type = htons(pAlter->type);
@@ -2268,7 +2268,7 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
 
-  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pShow->bytes[cols] = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "table_name");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
@@ -2331,12 +2331,12 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
       continue;
     }
 
-    char tableName[TSDB_TABLE_NAME_LEN + 1] = {0};
+    char tableName[TSDB_TABLE_NAME_LEN] = {0};
     
     // pattern compare for table name
     mnodeExtractTableName(pTable->info.tableId, tableName);
 
-    if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) {
+    if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
       mnodeDecTableRef(pTable);
       continue;
     }
@@ -2345,7 +2345,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
 
     char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
 
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, sizeof(tableName) - 1);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

src/mnode/src/mnodeVgroup.c

@@ -383,11 +383,11 @@ int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
     pTable = mnodeGetTable(pShow->payload);
     if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) {
       mnodeDecTableRef(pTable);
-      return TSDB_CODE_MND_INVALID_TABLE_ID;
+      return TSDB_CODE_MND_INVALID_TABLE_NAME;
     }
     mnodeDecTableRef(pTable);
     pVgroup = mnodeGetVgroup(((SChildTableObj*)pTable)->vgId);
-    if (NULL == pVgroup) return TSDB_CODE_MND_INVALID_TABLE_ID;
+    if (NULL == pVgroup) return TSDB_CODE_MND_INVALID_TABLE_NAME;
     mnodeDecVgroupRef(pVgroup);
     maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica;
   } else {

src/os/darwin/src/darwinPlatform.c

@@ -189,7 +189,7 @@ void taosGetSystemLocale() {
   if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
     char *locale = setlocale(LC_CTYPE, "chs");
     if (locale != NULL) {
-      strncpy(tsLocale, locale, sizeof(tsLocale) / sizeof(tsLocale[0]));
+      tstrncpy(tsLocale, locale, sizeof(tsLocale));
       cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
       uPrint("locale not configured, set to default:%s", tsLocale);
     }

src/os/linux/src/linuxSysPara.c

@@ -160,7 +160,7 @@ static void taosGetSystemTimezone() {
 
   /* load time zone string from /etc/timezone */
   FILE *f = fopen("/etc/timezone", "r");
-  char  buf[64] = {0};
+  char  buf[65] = {0};
   if (f != NULL) {
     fread(buf, 64, 1, f);
     fclose(f);
@@ -229,7 +229,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       uError("can't get locale from system, set it to en_US.UTF-8");
       strcpy(tsLocale, "en_US.UTF-8");
     } else {
-      strncpy(tsLocale, locale, tListLen(tsLocale));
+      tstrncpy(tsLocale, locale, sizeof(tsLocale));
       uError("locale not configured, set to system default:%s", tsLocale);
     }
   }
@@ -242,7 +242,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       str++;
 
       char *revisedCharset = taosCharsetReplace(str);
-      strncpy(tsCharset, revisedCharset, tListLen(tsCharset));
+      tstrncpy(tsCharset, revisedCharset, sizeof(tsCharset));
 
       free(revisedCharset);
       uWarn("charset not configured, set to system default:%s", tsCharset);
@@ -331,66 +331,7 @@ bool taosGetDisk() {
   return true;
 }
 
-static bool taosGetCardName(char *ip, char *name) {
-  struct ifaddrs *ifaddr, *ifa;
-  int             family, s;
-  char            host[NI_MAXHOST];
-  bool            ret = false;
-
-  if (getifaddrs(&ifaddr) == -1) {
-    return false;
-  }
-
-  /* Walk through linked list, maintaining head pointer so we can free list
-   * later */
-  for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
-    if (ifa->ifa_addr == NULL) continue;
-
-    family = ifa->ifa_addr->sa_family;
-    if (family != AF_INET) {
-      continue;
-    }
-
-    s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6), host,
-                    NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-    if (s != 0) {
-      break;
-    }
-
-    if (strcmp(host, "127.0.0.1") == 0) {
-      continue;
-    }
-
-    // TODO: the ip not config
-    // if (strcmp(host, ip) == 0) {
-    strcpy(name, ifa->ifa_name);
-    ret = true;
-    // }
-  }
-
-  freeifaddrs(ifaddr);
-  return ret;
-}
-
 static bool taosGetCardInfo(int64_t *bytes) {
-  static char tsPublicCard[1000] = {0};
-  static char tsPrivateIp[40];
-
-  if (tsPublicCard[0] == 0) {
-    if (!taosGetCardName(tsPrivateIp, tsPublicCard)) {
-      uError("can't get card name from ip:%s", tsPrivateIp);
-      return false;
-    }
-    int cardNameLen = (int)strlen(tsPublicCard);
-    for (int i = 0; i < cardNameLen; ++i) {
-      if (tsPublicCard[i] == ':') {
-        tsPublicCard[i] = 0;
-        break;
-      }
-    }
-    // uTrace("card name of public ip:%s is %s", tsPublicIp, tsPublicCard);
-  }
-
   FILE *fp = fopen(tsSysNetFile, "r");
   if (fp == NULL) {
     uError("open file:%s failed", tsSysNetFile);
@@ -403,6 +344,7 @@ static bool taosGetCardInfo(int64_t *bytes) {
 
   size_t len;
   char * line = NULL;
+  *bytes = 0;
 
   while (!feof(fp)) {
     tfree(line);
@@ -411,23 +353,20 @@ static bool taosGetCardInfo(int64_t *bytes) {
     if (line == NULL) {
       break;
     }
-    if (strstr(line, tsPublicCard) != NULL) {
+    if (strstr(line, "lo:") != NULL) {
-      break;
+      continue;
     }
+
+    sscanf(line,
+           "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64,
+           nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &tbytes, &tpackets);
+    *bytes += (rbytes + tbytes);
   }
-  if (line != NULL) {
+
-    sscanf(line, "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64, nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3,
-           &nouse4, &nouse5, &nouse6, &tbytes, &tpackets);
-    *bytes = rbytes + tbytes;
   tfree(line);
   fclose(fp);
+
   return true;
-  } else {
-    uWarn("can't get card:%s info from device:%s", tsPublicCard, tsSysNetFile);
-    *bytes = 0;
-    fclose(fp);
-    return false;
-  }
 }
 
 bool taosGetBandSpeed(float *bandSpeedKb) {
@@ -443,13 +382,15 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
   if (lastTime == 0 || lastBytes == 0) {
     lastTime = curTime;
     lastBytes = curBytes;
-    return false;
+    *bandSpeedKb = 0;
+    return true;
   }
 
   if (lastTime >= curTime || lastBytes > curBytes) {
     lastTime = curTime;
     lastBytes = curBytes;
-    return false;
+    *bandSpeedKb = 0;
+    return true;
   }
 
   double totalBytes = (double)(curBytes - lastBytes) / 1024 * 8;  // Kb

src/os/windows/src/twindows.c

@@ -226,7 +226,7 @@ void taosGetSystemLocale() {
   if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
     char *locale = setlocale(LC_CTYPE, "chs");
     if (locale != NULL) {
-      strncpy(tsLocale, locale, sizeof(tsLocale) / sizeof(tsLocale[0]));
+      tstrncpy(tsLocale, locale, sizeof(tsLocale));
       cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
       uPrint("locale not configured, set to default:%s", tsLocale);
     }

src/plugins/http/src/gcHandle.c

@@ -22,9 +22,27 @@
 #include "taosdef.h"
 
 static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest};
-static HttpEncodeMethod gcHeartBeatMethod = {NULL, gcSendHeartBeatResp, NULL, NULL, NULL, NULL, NULL, NULL};
+static HttpEncodeMethod gcHeartBeatMethod = {
+  .startJsonFp          = NULL,         
+  .stopJsonFp           = gcSendHeartBeatResp, 
+  .buildQueryJsonFp     = NULL,
+  .buildAffectRowJsonFp = NULL, 
+  .initJsonFp           = NULL, 
+  .cleanJsonFp          = NULL,
+  .checkFinishedFp      = NULL,
+  .setNextCmdFp         = NULL
+};
+
 static HttpEncodeMethod gcQueryMethod = {
-    NULL, gcStopQueryJson, gcBuildQueryJson, NULL, gcInitQueryJson, gcCleanQueryJson, NULL, NULL};
+  .startJsonFp          = NULL,         
+  .stopJsonFp           = gcStopQueryJson, 
+  .buildQueryJsonFp     = gcBuildQueryJson,
+  .buildAffectRowJsonFp = NULL, 
+  .initJsonFp           = gcInitQueryJson, 
+  .cleanJsonFp          = gcCleanQueryJson,
+  .checkFinishedFp      = NULL,
+  .setNextCmdFp         = NULL
+};
 
 void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod); }
 

src/plugins/http/src/httpAuth.c

@@ -88,8 +88,8 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) {
     free(base64);
     return false;
   } else {
-    strncpy(pContext->user, descrypt, TSDB_USER_LEN);
+    tstrncpy(pContext->user, descrypt, sizeof(pContext->user));
-    strncpy(pContext->pass, descrypt + TSDB_USER_LEN, TSDB_PASSWORD_LEN);
+    tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, TSDB_PASSWORD_LEN);
 
     httpTrace("context:%p, fd:%d, ip:%s, taosd token:%s parsed success, user:%s", pContext, pContext->fd,
               pContext->ipstr, token, pContext->user);

src/plugins/http/src/httpJson.c

@@ -445,7 +445,7 @@ void httpJsonPairStatus(JsonBuf* buf, int code) {
       httpJsonItemToken(buf);
       if (code == TSDB_CODE_MND_DB_NOT_SELECTED) {
         httpJsonPair(buf, "desc", 4, "failed to create database", 23);
-      } else if (code == TSDB_CODE_MND_INVALID_TABLE_ID) {
+      } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
         httpJsonPair(buf, "desc", 4, "failed to create table", 22);
       } else
         httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int)strlen(tstrerror(code)));

src/plugins/http/src/httpSql.c

@@ -87,7 +87,7 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
   }
 
   if (code < 0) {
-    if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code >= 0 ? 0 : code)) {
+    if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, -code)) {
       singleCmd->code = code;
       httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s",
                 pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql);

src/plugins/http/src/httpUtil.c

@@ -32,12 +32,12 @@ bool httpCheckUsedbSql(char *sql) {
 
 void httpTimeToString(time_t t, char *buf, int buflen) {
   memset(buf, 0, (size_t)buflen);
-  char ts[30] = {0};
+  char ts[32] = {0};
 
   struct tm *ptm;
   time_t     tt = t / 1000;
   ptm = localtime(&tt);
-  strftime(ts, 64, "%Y-%m-%d %H:%M:%S", ptm);
+  strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm);
   sprintf(buf, "%s.%03ld", ts, t % 1000);
 }
 
@@ -307,7 +307,7 @@ void httpTrimTableName(char *name) {
   for (int i = 0; name[i] != 0; i++) {
     if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'')
       name[i] = '_';
-    if (i == TSDB_TABLE_NAME_LEN + 1) {
+    if (i == TSDB_TABLE_NAME_LEN) {
       name[i] = 0;
       break;
     }
@@ -323,7 +323,7 @@ int httpShrinkTableName(HttpContext *pContext, int pos, char *name) {
     len++;
   }
 
-  if (len < TSDB_TABLE_NAME_LEN) {
+  if (len < TSDB_TABLE_NAME_LEN - 1) {
     return pos;
   }
 

src/plugins/http/src/restHandle.c

@@ -22,11 +22,37 @@
 static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest};
 static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest};
 static HttpEncodeMethod restEncodeSqlTimestampMethod = {
-    restStartSqlJson, restStopSqlJson, restBuildSqlTimestampJson, restBuildSqlAffectRowsJson, NULL, NULL, NULL, NULL};
+  .startJsonFp          = restStartSqlJson,         
+  .stopJsonFp           = restStopSqlJson, 
+  .buildQueryJsonFp     = restBuildSqlTimestampJson,
+  .buildAffectRowJsonFp = restBuildSqlAffectRowsJson, 
+  .initJsonFp           = NULL, 
+  .cleanJsonFp          = NULL,
+  .checkFinishedFp      = NULL,
+  .setNextCmdFp         = NULL
+};
+
 static HttpEncodeMethod restEncodeSqlLocalTimeStringMethod = {
-    restStartSqlJson, restStopSqlJson, restBuildSqlLocalTimeStringJson, restBuildSqlAffectRowsJson, NULL, NULL, NULL, NULL};
+  .startJsonFp          = restStartSqlJson,         
+  .stopJsonFp           = restStopSqlJson, 
+  .buildQueryJsonFp     = restBuildSqlLocalTimeStringJson,
+  .buildAffectRowJsonFp = restBuildSqlAffectRowsJson, 
+  .initJsonFp           = NULL, 
+  .cleanJsonFp          = NULL,
+  .checkFinishedFp      = NULL,
+  .setNextCmdFp         = NULL
+};
+
 static HttpEncodeMethod restEncodeSqlUtcTimeStringMethod = {
-    restStartSqlJson, restStopSqlJson, restBuildSqlUtcTimeStringJson, restBuildSqlAffectRowsJson, NULL, NULL, NULL, NULL};
+  .startJsonFp          = restStartSqlJson,         
+  .stopJsonFp           = restStopSqlJson, 
+  .buildQueryJsonFp     = restBuildSqlUtcTimeStringJson,
+  .buildAffectRowJsonFp = restBuildSqlAffectRowsJson, 
+  .initJsonFp           = NULL, 
+  .cleanJsonFp          = NULL,
+  .checkFinishedFp      = NULL,
+  .setNextCmdFp         = NULL
+};
 
 void restInitHandle(HttpServer* pServer) {
   httpAddMethod(pServer, &restDecodeMethod);

src/plugins/http/src/tgHandle.c

@@ -62,9 +62,16 @@
 #define TG_MAX_SORT_TAG_SIZE 20
 
 static HttpDecodeMethod tgDecodeMethod = {"telegraf", tgProcessRquest};
-static HttpEncodeMethod tgQueryMethod = {tgStartQueryJson,         tgStopQueryJson, NULL,
+static HttpEncodeMethod tgQueryMethod = {
-                                         tgBuildSqlAffectRowsJson, tgInitQueryJson, tgCleanQueryJson,
+  .startJsonFp          = tgStartQueryJson,         
-                                         tgCheckFinished,          tgSetNextCmd};
+  .stopJsonFp           = tgStopQueryJson, 
+  .buildQueryJsonFp     = NULL,
+  .buildAffectRowJsonFp = tgBuildSqlAffectRowsJson, 
+  .initJsonFp           = tgInitQueryJson, 
+  .cleanJsonFp          = tgCleanQueryJson,
+  .checkFinishedFp      = tgCheckFinished,
+  .setNextCmdFp         = tgSetNextCmd
+};
 
 static const char DEFAULT_TELEGRAF_CFG[] =
         "{\"metrics\":["
@@ -202,7 +209,7 @@ void tgParseSchemaMetric(cJSON *metric) {
         goto ParseEnd;
       }
       int nameLen = (int)strlen(field->valuestring);
-      if (nameLen == 0 || nameLen > TSDB_TABLE_NAME_LEN) {
+      if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) {
         parsedOk = false;
         goto ParseEnd;
       }
@@ -303,7 +310,7 @@ bool tgGetUserFromUrl(HttpContext *pContext) {
     return false;
   }
 
-  strcpy(pContext->user, pParser->path[TG_USER_URL_POS].pos);
+  tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].pos, TSDB_USER_LEN);
   return true;
 }
 
@@ -313,7 +320,7 @@ bool tgGetPassFromUrl(HttpContext *pContext) {
     return false;
   }
 
-  strcpy(pContext->pass, pParser->path[TG_PASS_URL_POS].pos);
+  tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
   return true;
 }
 
@@ -402,7 +409,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
     httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL);
     return false;
   }
-  if (nameLen >= TSDB_TABLE_NAME_LEN - 7) {
+  if (nameLen >= TSDB_TABLE_NAME_LEN - 8) {
     httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_LONG);
     return false;
   }
@@ -491,7 +498,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
     return false;
   }
 
-  if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN) {
+  if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN - 1) {
     httpSendErrorResp(pContext, HTTP_TG_TABLE_SIZE);
     return false;
   }

src/plugins/http/src/tgJson.c

@@ -111,7 +111,7 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) {
                     pContext->ipstr);
           return false;
         }
-      } else if (code == TSDB_CODE_MND_INVALID_TABLE_ID) {
+      } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
         cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
         if (multiCmds->cmds[multiCmds->pos - 1].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) {
           multiCmds->pos = (int16_t)(multiCmds->pos - 2);

src/plugins/monitor/src/monitorMain.c

@@ -156,7 +156,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
              ", io_read float, io_write float"
              ", req_http int, req_select int, req_insert int"
              ") tags (dnodeid int, fqdn binary(%d))",
-             tsMonitorDbName, TSDB_FQDN_LEN + 1);
+             tsMonitorDbName, TSDB_FQDN_LEN);
   } else if (cmd == MONITOR_CMD_CREATE_TB_DN) {
     snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName,
              dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp);

src/query/inc/qfill.h

@@ -50,7 +50,8 @@ typedef struct SFillInfo {
   char *  nextValues;           // next row of data
   char**  pData;                // original result data block involved in filling data
   int32_t capacityInRows;       // data buffer size in rows
-  
+  int8_t  slidingUnit;          // sliding time unit
+  int8_t  precision;            // time resoluation
   SFillColInfo* pFillCol;       // column info for fill operations
 } SFillInfo;
 
@@ -61,12 +62,13 @@ typedef struct SPoint {
 
 int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision);
 
-SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity,
+SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
-                            int32_t numOfCols, int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol);
+                            int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
+                            SFillColInfo* pFillCol);
 
 void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
 
-void taosDestoryFillInfo(SFillInfo *pFillInfo);
+void* taosDestoryFillInfo(SFillInfo *pFillInfo);
 
 void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
 
@@ -74,9 +76,7 @@ void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput)
 
 void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
 
-TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision);
+int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
-
-int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows);
 
 int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
 

src/query/inc/qsqlparser.h

@@ -322,7 +322,7 @@ enum {
 #define NORMAL_ARITHMETIC  1
 #define AGG_ARIGHTMEIC     2
 
-int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pSql);
+SSqlInfo qSQLParse(const char *str);
 
 #ifdef __cplusplus
 }

src/query/src/qExecutor.c

@@ -373,7 +373,6 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
         SPosInfo pos = {-1, -1};
         createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos);
       }
-
       pWindowResInfo->capacity = newCap;
     }
 
@@ -1369,7 +1368,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     int32_t index = pSqlFuncMsg->colInfo.colIndex;
     if (TSDB_COL_IS_TAG(pIndex->flag)) {
       if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) {  // todo refactor
-        pCtx->inputBytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+        pCtx->inputBytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
         pCtx->inputType = TSDB_DATA_TYPE_BINARY;
       } else {
         pCtx->inputBytes = pQuery->tagColList[index].bytes;
@@ -1467,7 +1466,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
     tfree(pRuntimeEnv->pCtx);
   }
 
-  taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
+  pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
 
   destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo);
   tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
@@ -1566,11 +1565,6 @@ static bool isFirstLastRowQuery(SQuery *pQuery) {
   return false;
 }
 
-static UNUSED_FUNC bool notHasQueryTimeRange(SQuery *pQuery) {
-  return (pQuery->window.skey == 0 && pQuery->window.ekey == INT64_MAX && QUERY_IS_ASC_QUERY(pQuery)) ||
-         (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery)));
-}
-
 static bool needReverseScan(SQuery *pQuery) {
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
@@ -1768,61 +1762,6 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool stableQuery) {
   }
 }
 
-static UNUSED_FUNC void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) {
-  assert(pCtx->param[index].pz == NULL);
-
-  int32_t len = 0;
-  size_t  t = 0;
-
-  if (type == TSDB_DATA_TYPE_BINARY) {
-    t = strlen(data);
-
-    len = t + 1 + TSDB_KEYSIZE;
-    pCtx->param[index].pz = calloc(1, len);
-  } else if (type == TSDB_DATA_TYPE_NCHAR) {
-    t = wcslen((const wchar_t *)data);
-
-    len = (t + 1) * TSDB_NCHAR_SIZE + TSDB_KEYSIZE;
-    pCtx->param[index].pz = calloc(1, len);
-  } else {
-    len = TSDB_KEYSIZE * 2;
-    pCtx->param[index].pz = malloc(len);
-  }
-
-  pCtx->param[index].nType = TSDB_DATA_TYPE_BINARY;
-
-  char *z = pCtx->param[index].pz;
-  *(TSKEY *)z = ts;
-  z += TSDB_KEYSIZE;
-
-  switch (type) {
-    case TSDB_DATA_TYPE_FLOAT:
-      *(double *)z = GET_FLOAT_VAL(data);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      *(double *)z = GET_DOUBLE_VAL(data);
-      break;
-    case TSDB_DATA_TYPE_INT:
-    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_TINYINT:
-    case TSDB_DATA_TYPE_SMALLINT:
-    case TSDB_DATA_TYPE_TIMESTAMP:
-      *(int64_t *)z = GET_INT64_VAL(data);
-      break;
-    case TSDB_DATA_TYPE_BINARY:
-      strncpy(z, data, t);
-      break;
-    case TSDB_DATA_TYPE_NCHAR: {
-      wcsncpy((wchar_t *)z, (const wchar_t *)data, t);
-    } break;
-    default:
-      assert(0);
-  }
-
-  pCtx->param[index].nLen = len;
-}
-
 static int32_t getInitialPageNum(SQInfo *pQInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
   int32_t INITIAL_RESULT_ROWS_VALUE = 16;
@@ -3618,9 +3557,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
    * first result row in the actual result set will fill nothing.
    */
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-    TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
+    int32_t numOfTotal = getFilledNumOfRes(pFillInfo, pQuery->window.ekey, pQuery->rec.capacity);
-                                         pQuery->slidingTimeUnit, pQuery->precision);
-    int32_t numOfTotal = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pQuery->rec.capacity);
     return numOfTotal > 0;
   }
 
@@ -3662,7 +3599,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
   }
 }
 
-int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t numOfRows, int32_t *numOfInterpo) {
+int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t *numOfInterpo) {
   SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
   SQuery *pQuery = pRuntimeEnv->pQuery;
   SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo;
@@ -4071,45 +4008,20 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
     initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type);
   }
 
-  setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
-
-  /*
-   * in case of last_row query without query range, we set the query timestamp to be
-   * STable->lastKey. Otherwise, keep the initial query time range unchanged.
-   */
-//  if (isFirstLastRowQuery(pQuery)) {
-//    if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) {
-//      sem_post(&pQInfo->dataReady);
-//      pointInterpSupporterDestroy(&interpInfo);
-//      return TSDB_CODE_SUCCESS;
-//    }
-//  }
-
   if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) {
     SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery);
     pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, 0, 0, pQuery->rec.capacity, pQuery->numOfOutput,
-                                              pQuery->slidingTime, pQuery->fillType, pColInfo);
+                                              pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision,
+                                              pQuery->fillType, pColInfo);
   }
 
+  // todo refactor
   pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery);
+  setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
+
   return TSDB_CODE_SUCCESS;
 }
 
-static UNUSED_FUNC bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, STableGroupInfo *pSidset) {
-  if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) {
-    return false;
-  }
-
-  for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) {
-    SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i);
-    if (pColIndex->flag == TSDB_COL_TAG) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
 static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
@@ -4753,13 +4665,11 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
       limitResults(pRuntimeEnv);
       break;
     } else {
-      TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
+      taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, pQuery->window.ekey);
-                                        pQuery->slidingTimeUnit, pQuery->precision);
-      taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, ekey);
       taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
       numOfInterpo = 0;
       
-      pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, pQuery->rec.rows, &numOfInterpo);
+      pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo);
       if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
         limitResults(pRuntimeEnv);
         break;
@@ -4791,8 +4701,7 @@ static void tableQueryImpl(SQInfo *pQInfo) {
      * So, we do keep in this procedure instead of launching retrieve procedure for next results.
      */
     int32_t numOfInterpo = 0;
-    int32_t remain = taosNumOfRemainRows(pRuntimeEnv->pFillInfo);
+    pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo);
-    pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, remain, &numOfInterpo);
   
     if (pQuery->rec.rows > 0) {
       limitResults(pRuntimeEnv);
@@ -5235,8 +5144,8 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
       bytes = tDataTypeDesc[type].nSize;
     } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) {  // parse the normal column
       type  = TSDB_DATA_TYPE_BINARY;
-      bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+      bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
-    } else {
+    } else{
       int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols);
       assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags || j == TSDB_TBNAME_COLUMN_INDEX);
 
@@ -5329,7 +5238,7 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
     if (pQuery->colList[i].numOfFilters > 0) {
       SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j];
 
-      memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData));
+      memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfo));
       pFilterInfo->info = pQuery->colList[i];
 
       pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters;
@@ -5907,10 +5816,11 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
 _over:
   tfree(tagCond);
   tfree(tbnameCond);
+  tfree(pGroupColIndex);
   taosArrayDestroy(pTableIdList);
 
-  if (code != TSDB_CODE_SUCCESS) {
   //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
+  if (code != TSDB_CODE_SUCCESS) {
     *pQInfo = NULL;
   }
 
@@ -6124,7 +6034,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
       for(int32_t j = 0; j < pQuery->numOfOutput; ++j) {
         if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
           char* data = tsdbGetTableName(pQInfo->tsdb, &item->id);
-          char* dst = pQuery->sdata[j]->data + count * (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE);
+          char* dst = pQuery->sdata[j]->data + count * ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
           memcpy(dst, data, varDataTLen(data));
         } else {// todo refactor
           int16_t type = pExprInfo[j].type;

src/query/src/qast.c

@@ -138,7 +138,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken
       memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema));
     } else {
       pNode->pSchema->type = TSDB_DATA_TYPE_BINARY;
-      pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN;
+      pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
       strcpy(pNode->pSchema->name, TSQL_TBNAME_L);
       pNode->pSchema->colId = -1;
     }
@@ -1127,7 +1127,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
   left->pSchema = pSchema;
 
   pSchema->type = TSDB_DATA_TYPE_BINARY;
-  pSchema->bytes = TSDB_TABLE_NAME_LEN;
+  pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
   strcpy(pSchema->name, TSQL_TBNAME_L);
   pSchema->colId = -1;
 

src/query/src/qfill.c

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qfill.h"
 #include "os.h"
+#include "qfill.h"
 #include "qextbuffer.h"
 #include "taosdef.h"
 #include "taosmsg.h"
@@ -58,7 +58,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, ch
 }
 
 SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
-    int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol) {
+    int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol) {
   if (fillType == TSDB_FILL_NONE) {
     return NULL;
   }
@@ -72,7 +72,9 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
   pFillInfo->pFillCol  = pFillCol;
   pFillInfo->numOfTags = numOfTags;
   pFillInfo->numOfCols = numOfCols;
+  pFillInfo->precision = precision;
   pFillInfo->slidingTime = slidingTime;
+  pFillInfo->slidingUnit = slidingUnit;
 
   pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
   
@@ -102,9 +104,9 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
   pFillInfo->numOfTotal   = 0;
 }
 
-void taosDestoryFillInfo(SFillInfo* pFillInfo) {
+void* taosDestoryFillInfo(SFillInfo* pFillInfo) {
   if (pFillInfo == NULL) {
-    return;
+    return NULL;
   }
 
   tfree(pFillInfo->prevValues);
@@ -119,6 +121,15 @@ void taosDestoryFillInfo(SFillInfo* pFillInfo) {
   tfree(pFillInfo->pFillCol);
   
   tfree(pFillInfo);
+  return NULL;
+}
+
+static TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
+  if (order == TSDB_ORDER_ASC) {
+    return ekey;
+  } else {
+    return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
+  }
 }
 
 void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
@@ -126,8 +137,10 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
     return;
   }
 
+  pFillInfo->endKey = taosGetRevisedEndKey(endKey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
+      pFillInfo->precision);
+
   pFillInfo->rowIdx    = 0;
-  pFillInfo->endKey    = endKey;
   pFillInfo->numOfRows = numOfRows;
   
   // ensure the space
@@ -165,36 +178,29 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu
   }
 }
 
-TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
+int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
-  if (order == TSDB_ORDER_ASC) {
+  int64_t* tsList = (int64_t*) pFillInfo->pData[0];
-    return ekey;
-  } else {
-    return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
-  }
-}
 
-static int32_t taosGetTotalNumOfFilledRes(SFillInfo* pFillInfo, const TSKEY* tsArray, int32_t remain,
+  int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
-    int64_t nInterval, int64_t ekey) {
 
-  if (remain > 0) {  // still fill gap within current data block, not generating data after the result set.
+  TSKEY ekey1 = taosGetRevisedEndKey(ekey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
-    TSKEY lastKey = tsArray[pFillInfo->numOfRows - 1];
+      pFillInfo->precision);
-    int32_t total = (int32_t)(labs(lastKey - pFillInfo->start) / nInterval) + 1;
 
-    assert(total >= remain);
+  int64_t numOfRes = -1;
-    return total;
+  if (numOfRows > 0) {  // still fill gap within current data block, not generating data after the result set.
+    TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
+
+    numOfRes = (int64_t)(labs(lastKey - pFillInfo->start) / pFillInfo->slidingTime) + 1;
+    assert(numOfRes >= numOfRows);
   } else { // reach the end of data
-    if ((ekey < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
+    if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
-        (ekey > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
+        (ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
       return 0;
-    } else {
+    } else {  // the numOfRes rows are all filled with specified policy
-      return (int32_t)(labs(ekey - pFillInfo->start) / nInterval) + 1;
+      numOfRes = (labs(ekey1 - pFillInfo->start) / pFillInfo->slidingTime) + 1;
     }
   }
-}
 
-int64_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows) {
-  int32_t numOfRes = taosGetTotalNumOfFilledRes(pFillInfo, (int64_t*) pFillInfo->pData[0], numOfRows,
-                                                pFillInfo->slidingTime, ekey);
   return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
 }
 
@@ -496,8 +502,8 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu
 
 int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
   int32_t remain = taosNumOfRemainRows(pFillInfo);  // todo use iterator?
-  int32_t rows   = taosGetNumOfResultWithFill(pFillInfo, remain, pFillInfo->endKey, capacity);
 
+  int32_t rows = getFilledNumOfRes(pFillInfo, pFillInfo->endKey, capacity);
   int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData);
   assert(numOfRes == rows);
   

src/query/src/qparserImpl.c

@@ -26,16 +26,18 @@
 #include "tstrbuild.h"
 #include "queryLog.h"
 
-int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pStr) {
+SSqlInfo qSQLParse(const char *pStr) {
   void *pParser = ParseAlloc(malloc);
-  pSQLInfo->valid = true;
+
+  SSqlInfo sqlInfo = {0};
+  sqlInfo.valid = true;
 
   int32_t i = 0;
   while (1) {
     SSQLToken t0 = {0};
 
     if (pStr[i] == 0) {
-      Parse(pParser, 0, t0, pSQLInfo);
+      Parse(pParser, 0, t0, &sqlInfo);
       goto abort_parse;
     }
 
@@ -49,19 +51,19 @@ int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pStr) {
         break;
       }
       case TK_SEMI: {
-        Parse(pParser, 0, t0, pSQLInfo);
+        Parse(pParser, 0, t0, &sqlInfo);
         goto abort_parse;
       }
       
       case TK_QUESTION:
       case TK_ILLEGAL: {
-        snprintf(pSQLInfo->pzErrMsg, tListLen(pSQLInfo->pzErrMsg), "unrecognized token: \"%s\"", t0.z);
+        snprintf(sqlInfo.pzErrMsg, tListLen(sqlInfo.pzErrMsg), "unrecognized token: \"%s\"", t0.z);
-        pSQLInfo->valid = false;
+        sqlInfo.valid = false;
         goto abort_parse;
       }
       default:
-        Parse(pParser, t0.type, t0, pSQLInfo);
+        Parse(pParser, t0.type, t0, &sqlInfo);
-        if (pSQLInfo->valid == false) {
+        if (sqlInfo.valid == false) {
           goto abort_parse;
         }
     }
@@ -69,7 +71,7 @@ int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pStr) {
 
 abort_parse:
   ParseFree(pParser, free);
-  return 0;
+  return sqlInfo;
 }
 
 tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken) {

src/query/src/qtsbuf.c

@@ -51,7 +51,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
     return NULL;
   }
   
-  strncpy(pTSBuf->path, path, PATH_MAX);
+  tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
   
   pTSBuf->f = fopen(pTSBuf->path, "r+");
   if (pTSBuf->f == NULL) {

src/rpc/src/rpcMain.c

@@ -458,7 +458,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
   pInfo->clientPort = pConn->peerPort;
   // pInfo->serverIp = pConn->destIp;
 
-  strncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
+  tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
   return 0;
 }
 

src/rpc/src/rpcTcp.c

@@ -67,7 +67,7 @@ static void   *taosProcessTcpData(void *param);
 static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd);
 static void    taosFreeFdObj(SFdObj *pFdObj);
 static void    taosReportBrokenLink(SFdObj *pFdObj);
-static void*   taosAcceptTcpConnection(void *arg);
+static void   *taosAcceptTcpConnection(void *arg);
 
 void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
   SServerObj *pServerObj;
@@ -80,6 +80,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     return NULL;
   }
 
+  pServerObj->fd = -1;
   pServerObj->thread = 0;
   pServerObj->ip = ip;
   pServerObj->port = port;
@@ -99,6 +100,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
 
+  // initialize parameters in case it may encounter error later 
   pThreadObj = pServerObj->pThreadObj;
   for (int i = 0; i < numOfThreads; ++i) {
     pThreadObj->pollFd = -1;
@@ -106,18 +108,21 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     pThreadObj->processData = fp;
     tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
     pThreadObj->shandle = shandle;
+    pThreadObj++;
+  }
 
+  // initialize mutex, thread, fd which may fail
+  pThreadObj = pServerObj->pThreadObj;
+  for (int i = 0; i < numOfThreads; ++i) {
     code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
     if (code < 0) {
       tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno); 
       break;;
     }
 
     pThreadObj->pollFd = epoll_create(10);  // size does not matter
     if (pThreadObj->pollFd < 0) {
       tError("%s failed to create TCP epoll", label);
-      terrno = TAOS_SYSTEM_ERROR(errno); 
       code = -1;
       break;
     }
@@ -125,7 +130,6 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj));
     if (code != 0) {
       tError("%s failed to create TCP process data thread(%s)", label, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno); 
       break;
     }
 
@@ -133,15 +137,18 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     pThreadObj++;
   }
 
+  pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
+  if (pServerObj->fd < 0) code = -1; 
+
   if (code == 0) { 
-    code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj));
+    code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj);
     if (code != 0) {
-      terrno = TAOS_SYSTEM_ERROR(errno); 
       tError("%s failed to create TCP accept thread(%s)", label, strerror(errno));
     }
   }
 
   if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno); 
     taosCleanUpTcpServer(pServerObj);
     pServerObj = NULL;
   } else {
@@ -204,7 +211,7 @@ void taosCleanUpTcpServer(void *handle) {
   tfree(pServerObj);
 }
 
-static void* taosAcceptTcpConnection(void *arg) {
+static void *taosAcceptTcpConnection(void *arg) {
   int                connFd = -1;
   struct sockaddr_in caddr;
   int                threadId = 0;
@@ -212,10 +219,6 @@ static void* taosAcceptTcpConnection(void *arg) {
   SServerObj        *pServerObj;
 
   pServerObj = (SServerObj *)arg;
-
-  pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
-  if (pServerObj->fd < 0) return NULL; 
-
   tTrace("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
 
   while (1) {

src/rpc/src/rpcUdp.c

@@ -19,6 +19,7 @@
 #include "ttimer.h"
 #include "tutil.h"
 #include "taosdef.h"
+#include "taoserror.h"
 #include "rpcLog.h"
 #include "rpcUdp.h"
 #include "rpcHead.h"
@@ -65,6 +66,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
   pSet = (SUdpConnSet *)malloc((size_t)size);
   if (pSet == NULL) {
     tError("%s failed to allocate UdpConn", label);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return NULL;
   }
 
@@ -73,30 +75,34 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
   pSet->port = port;
   pSet->shandle = shandle;
   pSet->fp = fp;
+  pSet->threads = threads;
   tstrncpy(pSet->label, label, sizeof(pSet->label));
 
+  pthread_attr_t thAttr;
+  pthread_attr_init(&thAttr);
+  pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+  int i;
   uint16_t ownPort;
-  for (int i = 0; i < threads; ++i) {
+  for (i = 0; i < threads; ++i) {
     pConn = pSet->udpConn + i;
     ownPort = (port ? port + i : 0);
     pConn->fd = taosOpenUdpSocket(ip, ownPort);
     if (pConn->fd < 0) {
       tError("%s failed to open UDP socket %x:%hu", label, ip, port);
-      taosCleanUpUdpConnection(pSet);
+      break;
-      return NULL;
     }
 
     pConn->buffer = malloc(RPC_MAX_UDP_SIZE);
     if (NULL == pConn->buffer) {
       tError("%s failed to malloc recv buffer", label);
-      taosCleanUpUdpConnection(pSet);
+      break;
-      return NULL;
     }
 
     struct sockaddr_in sin;
     unsigned int addrlen = sizeof(sin);
-    if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET &&
+    if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && 
-        addrlen == sizeof(sin)) {
+        sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
       pConn->localPort = (uint16_t)ntohs(sin.sin_port);
     }
 
@@ -107,23 +113,22 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
     pConn->pSet = pSet;
     pConn->signature = pConn;
 
-    pthread_attr_t thAttr;
-    pthread_attr_init(&thAttr);
-    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
     int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
-    pthread_attr_destroy(&thAttr);
     if (code != 0) {
-      tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno));
+      tError("%s failed to create thread to process UDP data(%s)", label, strerror(errno));
-      taosCloseSocket(pConn->fd);
+      break;
+    }
+  }
+
+  pthread_attr_destroy(&thAttr);
+
+  if (i != threads) { 
+    terrno = TAOS_SYSTEM_ERROR(errno);
     taosCleanUpUdpConnection(pSet);
     return NULL;
   }
 
-    ++pSet->threads;
+  tTrace("%s UDP connection is initialized, ip:%x:%hu threads:%d", label, ip, port, threads);
-  }
-
-  tTrace("%s UDP connection is initialized, ip:%x port:%hu threads:%d", label, ip, port, threads);
-
   return pSet;
 }
 
@@ -136,16 +141,17 @@ void taosCleanUpUdpConnection(void *handle) {
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
     pConn->signature = NULL;
+
     // shutdown to signal the thread to exit
-    shutdown(pConn->fd, SHUT_RD);
+    if ( pConn->fd >=0) shutdown(pConn->fd, SHUT_RD);
   }
 
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
-    pthread_join(pConn->thread, NULL);
+    if (pConn->thread) pthread_join(pConn->thread, NULL);
-    free(pConn->buffer);
+    if (pConn->fd >=0) taosCloseSocket(pConn->fd);
-    taosCloseSocket(pConn->fd);
+    tfree(pConn->buffer);
-    tTrace("chandle:%p is closed", pConn);
+    tTrace("UDP chandle:%p is closed", pConn);
   }
 
   tfree(pSet);
@@ -159,7 +165,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
   SUdpConn *pConn = pSet->udpConn + pSet->index;
   pConn->port = port;
 
-  tTrace("%s UDP connection is setup, ip:%x:%hu, local:%x:%d", pConn->label, ip, port, pSet->ip, pConn->localPort);
+  tTrace("%s UDP connection is setup, ip:%x:%hu", pConn->label, ip, port);
 
   return pConn;
 }

src/tsdb/inc/tsdbMain.h

@@ -357,6 +357,8 @@ typedef struct STsdbRepo {
 
   STsdbAppH appH;
 
+  STsdbStat stat;
+
   // The meter meta handle of this TSDB repository
   STsdbMeta *tsdbMeta;
 

src/tsdb/src/tsdbMain.c

@@ -953,6 +953,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
 static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
+  int64_t    points = 0;
 
   STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
   STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId);
@@ -964,7 +965,9 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
 
   // Check schema version
   int32_t tversion = pBlock->sversion;
-  int16_t nversion = schemaVersion(tsdbGetTableSchema(pMeta, pTable));
+  STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable);
+  ASSERT(pSchema != NULL);
+  int16_t nversion = schemaVersion(pSchema);
   if (tversion > nversion) {
     tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.",
               pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion);
@@ -1014,7 +1017,10 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
       return -1;
     }
      (*affectedrows)++;
+     points++;
   }
+  atomic_fetch_add_64(&(pRepo->stat.pointsWritten), points * (pSchema->numOfCols));
+  atomic_fetch_add_64(&(pRepo->stat.totalStorage), points * pSchema->vlen);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1381,3 +1387,11 @@ uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t
 
   return magic;
 }
+
+void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage){
+    ASSERT(repo != NULL);
+    STsdbRepo * pRepo = repo;
+    *totalPoints = pRepo->stat.pointsWritten;
+    *totalStorage = pRepo->stat.totalStorage;
+    *compStorage = pRepo->stat.compStorage;
+}

src/tsdb/src/tsdbMeta.c

@@ -74,14 +74,16 @@ void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) {
 STable *tsdbDecodeTable(void *cont, int contLen) {
   STable *pTable = (STable *)calloc(1, sizeof(STable));
   if (pTable == NULL) return NULL;
+
+  void *ptr = cont;
+  T_READ_MEMBER(ptr, int8_t, pTable->type);
+  if (pTable->type != TSDB_CHILD_TABLE) {
     pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS);
     if (pTable->schema == NULL) {
       free(pTable);
       return NULL;
     }
-
+  }
-  void *ptr = cont;
-  T_READ_MEMBER(ptr, int8_t, pTable->type);
   int len = *(int *)ptr;
   ptr = (char *)ptr + sizeof(int);
   pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1);
@@ -332,7 +334,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
     pTable->schema[0] = tdDupSchema(pCfg->schema);
     pTable->tagSchema = tdDupSchema(pCfg->tagSchema);
 
-    tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN);
+    tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN - 1);
     pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
     if (pTable->name == NULL) {
       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -353,7 +355,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
     pTable->tableId.tid = pCfg->tableId.tid;
     pTable->lastKey = TSKEY_INITIAL_VAL;
 
-    tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN);
+    tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN - 1);
     pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1);
     if (pTable->name == NULL) {
       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -620,7 +622,10 @@ static int tsdbFreeTable(STable *pTable) {
   if (pTable->type == TSDB_CHILD_TABLE) {
     kvRowFree(pTable->tagVal);
   } else {
+    if (pTable->schema) {
       for (int i = 0; i < pTable->numOfSchemas; i++) tdFreeSchema(pTable->schema[i]);
+      free(pTable->schema);
+    }
   }
 
   if (pTable->type == TSDB_STREAM_TABLE) {

src/tsdb/src/tsdbRWHelper.c

@@ -764,8 +764,8 @@ static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
 
 static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock,
                                 bool isLast, bool isSuperBlock) {
-  ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows &&
+  ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
-         rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
+  ASSERT(isLast ? rowsToWrite < pHelper->config.minRowsPerFileBlock : true);
 
   SCompData *pCompData = (SCompData *)(pHelper->pBuffer);
   int64_t offset = 0;
@@ -905,14 +905,15 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
 
     rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfRows), pDataCols->numOfRows);
     if ((blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) &&
-        (blockAtIdx(pHelper, blkIdx)->numOfRows + rowsWritten < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd) > 0) {
+        (blockAtIdx(pHelper, blkIdx)->numOfRows + rowsWritten < pHelper->config.minRowsPerFileBlock) &&
+        (pHelper->files.nLastF.fd) < 0) {
       if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, rowsWritten, &compBlock, true, false) < 0)
         goto _err;
       if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err;
     } else {
       // Load
       if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err;
-      ASSERT(pHelper->pDataCols[0]->numOfRows == blockAtIdx(pHelper, blkIdx)->numOfRows);
+      ASSERT(pHelper->pDataCols[0]->numOfRows <= blockAtIdx(pHelper, blkIdx)->numOfRows);
       // Merge
       if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err;
       // Write
@@ -936,21 +937,21 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
     // Key must overlap with the block
     ASSERT(keyFirst <= blockAtIdx(pHelper, blkIdx)->keyLast);
 
-    TSKEY keyLimit =
+    TSKEY keyLimit = (blkIdx == pIdx->numOfBlocks - 1) ? INT64_MAX : blockAtIdx(pHelper, blkIdx + 1)->keyFirst - 1;
-        (blkIdx == pIdx->numOfBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1;
 
     // rows1: number of rows must merge in this block
     int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast);
-    // rows2: max nuber of rows the block can have more
+    // rows2: max number of rows the block can have more
     int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfRows;
     // rows3: number of rows between this block and the next block
     int rows3 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, keyLimit);
 
     ASSERT(rows3 >= rows1);
 
-    if ((rows2 >= rows1) &&
+    if ((rows2 >= rows1) && (blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) &&
-        (( blockAtIdx(pHelper, blkIdx)->last) ||
+        ((!blockAtIdx(pHelper, blkIdx)->last) ||
-         ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd < 0)))) {
+         ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock) &&
+          (pHelper->files.nLastF.fd < 0)))) {
       rowsWritten = rows1;
       bool   isLast = false;
       SFile *pFile = NULL;
@@ -1106,16 +1107,16 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
     for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) {
       SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i;
       if (pTCompBlock->numOfSubBlocks > 1) {
-        ptr = (void *)((char *)(pHelper->pCompInfo) + pTCompBlock->offset + pTCompBlock->len);
+        ptr = POINTER_SHIFT(pHelper->pCompInfo, pTCompBlock->offset);
         break;
       }
     }
 
-    if (ptr == NULL) ptr = (void *)((char *)(pHelper->pCompInfo) + pIdx->len - sizeof(TSCKSUM));
+    if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len-sizeof(TSCKSUM));
 
     size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo));
     if (tsize > 0) {
-      memmove((void *)((char *)ptr + sizeof(SCompBlock) * 2), ptr, tsize);
+      memmove(POINTER_SHIFT(ptr, sizeof(SCompBlock) * 2), ptr, tsize);
       for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) {
         SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i;
         if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2);

src/tsdb/src/tsdbRead.c

@@ -233,8 +233,6 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond*
   STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
   
   pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
-//  pQueryHandle->outputCapacity = 2;  // only allowed two rows to be loaded
-  
   changeQueryHandleForInterpQuery(pQueryHandle);
   return pQueryHandle;
 }
@@ -618,54 +616,19 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
 static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
+
   /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
+  SDataRow row = getSDataRowInTableMem(pCheckInfo);
 
-  TSKEY k1 = TSKEY_INITIAL_VAL, k2 = TSKEY_INITIAL_VAL;
+  TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
-  if (pCheckInfo->iter != NULL && tSkipListIterGet(pCheckInfo->iter) != NULL) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
-    
-    SDataRow row = SL_GET_NODE_DATA(node);
-    k1 = dataRowKey(row);
-    
-    if (k1 == binfo.window.skey) {
-      if (tSkipListIterNext(pCheckInfo->iter)) {
-        node = tSkipListIterGet(pCheckInfo->iter);
-        row = SL_GET_NODE_DATA(node);
-        k1 = dataRowKey(row);
-      } else {
-        k1 = TSKEY_INITIAL_VAL;
-      }
-    }
-  }
-  
-  if (pCheckInfo->iiter != NULL && tSkipListIterGet(pCheckInfo->iiter) != NULL) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
-    
-    SDataRow row = SL_GET_NODE_DATA(node);
-    k2 = dataRowKey(row);
-    
-    if (k2 == binfo.window.skey) {
-      if (tSkipListIterNext(pCheckInfo->iiter)) {
-        node = tSkipListIterGet(pCheckInfo->iiter);
-        row = SL_GET_NODE_DATA(node);
-        k2 = dataRowKey(row);
-      } else {
-        k2 = TSKEY_INITIAL_VAL;
-      }
-    }
-  }
-  
   cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
   
-  if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
+  if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
-      ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey))) ||
+      (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
-      (!ASCENDING_TRAVERSE(pQueryHandle->order) &&
+    
-      ((k1 != TSKEY_INITIAL_VAL && k1 >= binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 >= binfo.window.skey)))) {
+    if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
+        (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
 
-    if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
-    ((k1 != TSKEY_INITIAL_VAL && k1 < binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 < binfo.window.skey))) ||
-        (!ASCENDING_TRAVERSE(pQueryHandle->order) &&
-         (((k1 != TSKEY_INITIAL_VAL && k1 > binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 > binfo.window.skey))))) {
       // do not load file block into buffer
       int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
 
@@ -756,7 +719,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
   return pQueryHandle->realNumOfRows > 0;
 }
 
-static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
+static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
   int    firstPos, lastPos, midPos = -1;
   int    numOfRows;
   TSKEY* keyList;
@@ -868,14 +831,21 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
   return numOfRows + num;
 }
 
-static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t capacity,
+static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row,
-    int32_t numOfRows, SDataRow row, STSchema* pSchema) {
+                              STsdbMeta *pMeta, int32_t numOfCols, STable* pTable) {
-  int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
-  int32_t numOfTableCols = schemaNCols(pSchema);
-  
   char* pData = NULL;
-  for (int32_t i = 0; i < numOfCols; ++i) {
+
+  // the schema version info is embeded in SDataRow
+  STSchema* pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row));
+  int32_t numOfRowCols = schemaNCols(pSchema);
+
+  int32_t i = 0, j = 0;
+  while(i < numOfCols && j < numOfRowCols) {
     SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+    if (pSchema->columns[j].colId < pColInfo->info.colId) {
+      j++;
+      continue;
+    }
 
     if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
       pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
@@ -883,22 +853,41 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, STableCheckInfo* p
       pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
     }
 
-    int32_t offset = 0;
+    if (pSchema->columns[j].colId == pColInfo->info.colId) {
-    for (int32_t j = 0; j < numOfTableCols; ++j) {
+      void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
-      if (pColInfo->info.colId == pSchema->columns[j].colId) {
-        offset = pSchema->columns[j].offset;
-        break;
-      }
-    }
-    
-    assert(offset != -1);  // todo handle error
-    void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
-    
       if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
         memcpy(pData, value, varDataTLen(value));
       } else {
         memcpy(pData, value, pColInfo->info.bytes);
       }
+
+      j++;
+      i++;
+    } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
+      if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
+        setVardataNull(pData, pColInfo->info.type);
+      } else {
+        setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
+      }
+      i++;
+    }
+  }
+
+  while (i < numOfCols) { // the remain columns are all null data
+    SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+    if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+      pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
+    } else {
+      pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
+    }
+
+    if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(pData, pColInfo->info.type);
+    } else {
+      setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
+    }
+
+    i++;
   }
 }
 
@@ -912,6 +901,15 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   initTableMemIterator(pQueryHandle, pCheckInfo);
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
 
+  // for search the endPos, so the order needs to reverse
+  int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
+
+  int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
+  int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
+
+  STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
+  STable* pTable = pCheckInfo->pTableObj;
+
   int32_t endPos = cur->pos;
   if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
     endPos = blockInfo.rows - 1;
@@ -920,8 +918,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     endPos = 0;
     cur->mixBlock = (cur->pos != blockInfo.rows - 1);
   } else {
-    int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
+    assert(pCols->numOfRows > 0);
-    endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
+    endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
     cur->mixBlock = true;
   }
   
@@ -933,7 +931,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   
   int32_t numOfRows = 0;
   pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
-  int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
 
   // no data in buffer, load data from file directly
   if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
@@ -953,9 +950,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     // if the buffer is not full in case of descending order query, move the data in the front of the buffer
     if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) {
       int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
-      int32_t reqNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
 
-      for(int32_t i = 0; i < reqNumOfCols; ++i) {
+      for(int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
         memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
       }
@@ -969,19 +965,14 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     pQueryHandle->realNumOfRows = numOfRows;
     cur->rows = numOfRows;
     return;
-  } else if (pCheckInfo->iter != NULL && pCheckInfo->iiter == NULL) {
+  } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
-    //  } else if (pCheckInfo->iter == NULL && pCheckInfo->iiter != NULL) {
-    //  } else { // iter and iiter are all not NULL, three-way merge data block
-    STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj);
     SSkipListNode* node = NULL;
-    
     do {
-      node = tSkipListIterGet(pCheckInfo->iter);
+      SDataRow row = getSDataRowInTableMem(pCheckInfo);
-      if (node == NULL) {
+      if (row == NULL) {
         break;
       }
 
-      SDataRow row = SL_GET_NODE_DATA(node);
       TSKEY key = dataRowKey(row);
       if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
           (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
@@ -995,7 +986,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
 
       if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
           (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
-        copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema);
+        copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, pMeta, numOfCols, pTable);
         numOfRows += 1;
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = key;
@@ -1005,17 +996,16 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
         cur->lastKey  = key + step;
         cur->mixBlock = true;
 
-        tSkipListIterNext(pCheckInfo->iter);
+        moveToNextRow(pCheckInfo);
       } else if (key == tsArray[pos]) {  // data in buffer has the same timestamp of data in file block, ignore it
-        tSkipListIterNext(pCheckInfo->iter);
+        moveToNextRow(pCheckInfo);
       } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
                   (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = tsArray[pos];
         }
 
-        int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
+        int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
-        int32_t end = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
         if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
           tSkipListIterNext(pCheckInfo->iter);
         }
@@ -1093,9 +1083,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     // if the buffer is not full in case of descending order query, move the data in the front of the buffer
     if (numOfRows < pQueryHandle->outputCapacity) {
       int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
-  
+      for(int32_t i = 0; i < numOfCols; ++i) {
-      int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
-      for(int32_t i = 0; i < requiredNumOfCols; ++i) {
         SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
         memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
       }
@@ -1567,9 +1555,6 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
   
   for(int32_t i = 0; i < numOfTables; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
-    if (pCheckInfo->pTableObj->tableId.uid == 12094628167747) {
-      printf("abc\n");
-    }
     if (pCheckInfo->pTableObj->lastKey > key) {
       key = pCheckInfo->pTableObj->lastKey;
       index = i;
@@ -1652,8 +1637,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
   *skey = TSKEY_INITIAL_VAL;
 
   int64_t st = taosGetTimestampUs();
-  STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj);
+  STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
-  int32_t numOfTableCols = schemaNCols(pSchema);
+  STable* pTable = pCheckInfo->pTableObj;
 
   do {
     SDataRow row = getSDataRowInTableMem(pCheckInfo);
@@ -1662,10 +1647,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     }
 
     TSKEY key = dataRowKey(row);
-    
+    if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
-    if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-        (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
-      
       tsdbTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey,
           pQueryHandle->window.ekey);
       
@@ -1677,58 +1659,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     }
 
     *ekey = key;
-    char* pData = NULL;
+    copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable);
-    
-    int32_t i = 0, j = 0;
-    while(i < numOfCols && j < numOfTableCols) {
-      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-      if (pSchema->columns[j].colId < pColInfo->info.colId) {
-        j++;
-        continue;
-      }
-  
-      if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-        pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
-      } else {
-        pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes;
-      }
-      
-      if (pSchema->columns[j].colId == pColInfo->info.colId) {
-        void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
-        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
-          memcpy(pData, value, varDataTLen(value));
-        } else {
-          memcpy(pData, value, pColInfo->info.bytes);
-        }
-        
-        j++;
-        i++;
-      } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
-        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
-          setVardataNull(pData, pColInfo->info.type);
-        } else {
-          setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
-        }
-        i++;
-      }
-    }
-    
-    while (i < numOfCols) { // the remain columns are all null data
-      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-      if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-        pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
-      } else {
-        pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes;
-      }
-  
-      if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
-        setVardataNull(pData, pColInfo->info.type);
-      } else {
-        setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
-      }
-      
-      i++;
-    }
 
     if (++numOfRows >= maxRowsToRead) {
       moveToNextRow(pCheckInfo);
@@ -2011,7 +1942,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
       f1 = (char*) pTable1->name;
       f2 = (char*) pTable2->name;
       type = TSDB_DATA_TYPE_BINARY;
-      bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+      bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
     } else {
       STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex);
       bytes = pCol->bytes;

src/util/inc/tcache.h

@@ -37,8 +37,8 @@ typedef struct SCacheDataNode {
   uint64_t expiredTime;  // expiredTime expiredTime when this element should be remove from cache
   uint64_t signature;
   uint32_t size;         // allocated size for current SCacheDataNode
-  uint16_t keySize : 15;
+  uint16_t keySize: 15;
-  bool     inTrash : 1;  // denote if it is in trash or not
+  bool     inTrashCan: 1;// denote if it is in trash or not
   T_REF_DECLARE()
   char *key;
   char  data[];
@@ -50,46 +50,49 @@ typedef struct STrashElem {
   SCacheDataNode    *pData;
 } STrashElem;
 
-typedef struct {
+/*
-  int64_t totalSize;  // total allocated buffer in this hash table, SCacheObj is not included.
+ * to accommodate the old data which has the same key value of new one in hashList
-  int64_t refreshTime;
-  
-  /*
-   * to accommodate the old datanode which has the same key value of new one in hashList
  * when an new node is put into cache, if an existed one with the same key:
  * 1. if the old one does not be referenced, update it.
  * 2. otherwise, move the old one to pTrash, addedTime the new one.
  *
  * when the node in pTrash does not be referenced, it will be release at the expired expiredTime
  */
+typedef struct {
+  int64_t         totalSize;          // total allocated buffer in this hash table, SCacheObj is not included.
+  int64_t         refreshTime;
   STrashElem *    pTrash;
   void *          tmrCtrl;
   void *          pTimer;
   SCacheStatis    statistics;
   SHashObj *      pHashTable;
   _hash_free_fn_t freeFp;
-  int          numOfElemsInTrash;  // number of element in trash
+  uint32_t        numOfElemsInTrash;  // number of element in trash
-  int16_t      deleting;           // set the deleting flag to stop refreshing ASAP.
+  uint8_t         deleting;           // set the deleting flag to stop refreshing ASAP.
-  T_REF_DECLARE()
+  pthread_t       refreshWorker;
 
 #if defined(LINUX)
   pthread_rwlock_t lock;
 #else
   pthread_mutex_t lock;
 #endif
-
 } SCacheObj;
 
 /**
- *
+ * initialize the cache object
- * @param maxSessions       maximum slots available for hash elements
- * @param tmrCtrl           timer ctrl
  * @param refreshTime       refresh operation interval time, the maximum survival time when one element is expired and
  *                          not referenced by other objects
  * @return
  */
-SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTimeInSeconds);
+SCacheObj *taosCacheInit(int64_t refreshTimeInSeconds);
-SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
+
+/**
+ * initialize the cache object and set the free object callback function
+ * @param refreshTimeInSeconds
+ * @param freeCb
+ * @return
+ */
+SCacheObj *taosCacheInitWithCb(int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
 
 /**
  * add data into cache

src/util/inc/tkvstore.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TD_KVSTORE_H_
+#define _TD_KVSTORE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+typedef int (*iterFunc)(void *, void *cont, int contLen);
+typedef void (*afterFunc)(void *);
+
+typedef struct {
+  int64_t size;
+  int64_t tombSize;
+  int64_t nRecords;
+  int64_t nDels;
+} SStoreInfo;
+
+typedef struct {
+  char *     fname;
+  int        fd;
+  char *     fsnap;
+  int        sfd;
+  char *     fnew;
+  int        nfd;
+  SHashObj * map;
+  iterFunc   iFunc;
+  afterFunc  aFunc;
+  void *     appH;
+  SStoreInfo info;
+} SKVStore;
+
+int       tdCreateKVStore(char *fname);
+int       tdDestroyKVStore();
+SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
+void      tdCloseKVStore(SKVStore *pStore);
+int       tdKVStoreStartCommit(SKVStore *pStore);
+int       tdUpdateRecordInKVStore(SKVStore *pStore, uint64_t uid, void *cont, int contLen);
+int       tdKVStoreEndCommit(SKVStore *pStore);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/util/src/tcache.c

@@ -77,31 +77,7 @@ static FORCE_INLINE void taosFreeNode(void *data) {
  * @param lifespan total survial expiredTime from now
  * @return         SCacheDataNode
  */
-static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
+static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration);
-                                          uint64_t duration) {
-  size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1;
-  
-  SCacheDataNode *pNewNode = calloc(1, totalSize);
-  if (pNewNode == NULL) {
-    uError("failed to allocate memory, reason:%s", strerror(errno));
-    return NULL;
-  }
-  
-  memcpy(pNewNode->data, pData, size);
-  
-  pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
-  pNewNode->keySize = keyLen;
-  
-  memcpy(pNewNode->key, key, keyLen);
-  
-  pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
-  pNewNode->expiredTime = pNewNode->addedTime + duration;
-  
-  pNewNode->signature = (uint64_t)pNewNode;
-  pNewNode->size = (uint32_t)totalSize;
-  
-  return pNewNode;
-}
 
 /**
  * addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash
@@ -109,50 +85,15 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const
  * @param pCacheObj    Cache object
  * @param pNode   Cache slot object
  */
-static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
+static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode);
-  if (pNode->inTrash) { /* node is already in trash */
-    return;
-  }
 
-  STrashElem *pElem = calloc(1, sizeof(STrashElem));
+/**
-  pElem->pData = pNode;
+ * remove node in trash can
+ * @param pCacheObj 
+ * @param pElem 
+ */
+static void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem);
 
-  pElem->next = pCacheObj->pTrash;
-  if (pCacheObj->pTrash) {
-    pCacheObj->pTrash->prev = pElem;
-  }
-  
-  pElem->prev = NULL;
-  pCacheObj->pTrash = pElem;
-  
-  pNode->inTrash = true;
-  pCacheObj->numOfElemsInTrash++;
-  
-  uTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash);
-}
-
-static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) {
-  if (pElem->pData->signature != (uint64_t)pElem->pData) {
-    uError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData);
-    return;
-  }
-  
-  pCacheObj->numOfElemsInTrash--;
-  if (pElem->prev) {
-    pElem->prev->next = pElem->next;
-  } else { /* pnode is the header, update header */
-    pCacheObj->pTrash = pElem->next;
-  }
-  
-  if (pElem->next) {
-    pElem->next->prev = pElem->prev;
-  }
-  
-  pElem->pData->signature = 0;
-  if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data);
-  free(pElem->pData);
-  free(pElem);
-}
 /**
  * remove nodes in trash with refCount == 0 in cache
  * @param pNode
@@ -160,42 +101,7 @@ static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) {
  * @param force   force model, if true, remove data in trash without check refcount.
  *                may cause corruption. So, forece model only applys before cache is closed
  */
-static void taosTrashEmpty(SCacheObj *pCacheObj, bool force) {
+static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force);
-  __cache_wr_lock(pCacheObj);
-  
-  if (pCacheObj->numOfElemsInTrash == 0) {
-    if (pCacheObj->pTrash != NULL) {
-      uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash);
-    }
-    pCacheObj->pTrash = NULL;
-    
-    __cache_unlock(pCacheObj);
-    return;
-  }
-  
-  STrashElem *pElem = pCacheObj->pTrash;
-  
-  while (pElem) {
-    T_REF_VAL_CHECK(pElem->pData);
-    if (pElem->next == pElem) {
-      pElem->next = NULL;
-    }
-    
-    if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
-      uTrace("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData,
-             pCacheObj->numOfElemsInTrash - 1);
-      STrashElem *p = pElem;
-      
-      pElem = pElem->next;
-      taosRemoveFromTrash(pCacheObj, p);
-    } else {
-      pElem = pElem->next;
-    }
-  }
-  
-  assert(pCacheObj->numOfElemsInTrash >= 0);
-  __cache_unlock(pCacheObj);
-}
 
 /**
  * release node
@@ -304,87 +210,20 @@ static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pCacheObj, con
   return pNode;
 }
 
-static void doCleanupDataCache(SCacheObj *pCacheObj) {
+/**
-  __cache_wr_lock(pCacheObj);
+ * do cleanup the taos cache
-  
+ * @param pCacheObj
-  //if (taosHashGetSize(pCacheObj->pHashTable) > 0) {
+ */
-  taosHashCleanup(pCacheObj->pHashTable);
+static void doCleanupDataCache(SCacheObj *pCacheObj);
-  //}
-  
-  __cache_unlock(pCacheObj);
-  
-  taosTrashEmpty(pCacheObj, true);
-  __cache_lock_destroy(pCacheObj);
-  
-  memset(pCacheObj, 0, sizeof(SCacheObj));
-  free(pCacheObj);
-}
 
 /**
  * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime
  * @param handle   Cache object handle
  */
-static void taosCacheRefresh(void *handle, void *tmrId) {
+static void* taosCacheRefresh(void *handle);
-  SCacheObj *pCacheObj = (SCacheObj *)handle;
 
-  if (pCacheObj == NULL || T_REF_VAL_GET(pCacheObj) == 0) {
+SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) {
-    uTrace("object is destroyed. no refresh retry");
+  if (refreshTime <= 0) {
-    return;
-  }
-  
-  int16_t ref = T_REF_INC(pCacheObj);
-  if (ref == 1) {
-    T_REF_DEC(pCacheObj);
-    return;
-  }
-  
-  // todo add the ref before start the timer
-  int32_t num = taosHashGetSize(pCacheObj->pHashTable);
-  if (num == 0) {
-    ref = T_REF_DEC(pCacheObj);
-    if (ref == 0) {
-      doCleanupDataCache(pCacheObj);
-    } else {
-      taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
-    }
-    return;
-  }
-  
-  uint64_t expiredTime = taosGetTimestampMs();
-  pCacheObj->statistics.refreshCount++;
-  
-  SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
-  
-  __cache_wr_lock(pCacheObj);
-  while (taosHashIterNext(pIter)) {
-    if (pCacheObj->deleting == 1) {
-      taosHashDestroyIter(pIter);
-      break;
-    }
-    
-    SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
-    if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
-      taosCacheReleaseNode(pCacheObj, pNode);
-    }
-  }
-  
-  __cache_unlock(pCacheObj);
-  
-  taosHashDestroyIter(pIter);
-
-    taosTrashEmpty(pCacheObj, false);
-    
-    ref = T_REF_DEC(pCacheObj);
-    if (ref == 0)  {
-      doCleanupDataCache(pCacheObj);
-      return;
-    } else {
-      taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
-    }
-}
-
-SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb)(void *data)) {
-  if (tmrCtrl == NULL || refreshTime <= 0) {
     return NULL;
   }
   
@@ -394,7 +233,7 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
     return NULL;
   }
   
-  pCacheObj->pHashTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
+  pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
   if (pCacheObj->pHashTable == NULL) {
     free(pCacheObj);
     uError("failed to allocate memory, reason:%s", strerror(errno));
@@ -406,12 +245,8 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
   
   pCacheObj->freeFp = freeCb;
   pCacheObj->refreshTime = refreshTime * 1000;
-  pCacheObj->tmrCtrl = tmrCtrl;
-  
-  taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
 
   if (__cache_lock_init(pCacheObj) != 0) {
-    taosTmrStopA(&pCacheObj->pTimer);
     taosHashCleanup(pCacheObj->pHashTable);
     free(pCacheObj);
     
@@ -419,12 +254,18 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
     return NULL;
   }
 
-  T_REF_INC(pCacheObj);
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+  pthread_create(&pCacheObj->refreshWorker, &thattr, taosCacheRefresh, pCacheObj);
+
+  pthread_attr_destroy(&thattr);
   return pCacheObj;
 }
 
-SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
+SCacheObj *taosCacheInit(int64_t refreshTime) {
-  return taosCacheInitWithCb(tmrCtrl, refreshTime, NULL);
+  return taosCacheInitWithCb(refreshTime, NULL);
 }
 
 void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) {
@@ -600,7 +441,7 @@ void taosCacheEmpty(SCacheObj *pCacheObj) {
   __cache_unlock(pCacheObj);
   
   taosHashDestroyIter(pIter);
-  taosTrashEmpty(pCacheObj, false);
+  taosTrashCanEmpty(pCacheObj, false);
 }
 
 void taosCacheCleanup(SCacheObj *pCacheObj) {
@@ -608,8 +449,180 @@ void taosCacheCleanup(SCacheObj *pCacheObj) {
     return;
   }
 
-  int32_t ref = T_REF_DEC(pCacheObj);
+  pCacheObj->deleting = 1;
-  if (ref == 0) {
+  pthread_join(pCacheObj->refreshWorker, NULL);
+
   doCleanupDataCache(pCacheObj);
-  }
+}
+
+SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
+                                           uint64_t duration) {
+  size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1;
+
+  SCacheDataNode *pNewNode = calloc(1, totalSize);
+  if (pNewNode == NULL) {
+    uError("failed to allocate memory, reason:%s", strerror(errno));
+    return NULL;
+  }
+
+  memcpy(pNewNode->data, pData, size);
+
+  pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
+  pNewNode->keySize = keyLen;
+
+  memcpy(pNewNode->key, key, keyLen);
+
+  pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
+  pNewNode->expiredTime = pNewNode->addedTime + duration;
+
+  pNewNode->signature = (uint64_t)pNewNode;
+  pNewNode->size = (uint32_t)totalSize;
+
+  return pNewNode;
+}
+
+void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
+  if (pNode->inTrashCan) { /* node is already in trash */
+    return;
+  }
+
+  STrashElem *pElem = calloc(1, sizeof(STrashElem));
+  pElem->pData = pNode;
+
+  pElem->next = pCacheObj->pTrash;
+  if (pCacheObj->pTrash) {
+    pCacheObj->pTrash->prev = pElem;
+  }
+
+  pElem->prev = NULL;
+  pCacheObj->pTrash = pElem;
+
+  pNode->inTrashCan = true;
+  pCacheObj->numOfElemsInTrash++;
+
+  uTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash);
+}
+
+void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) {
+  if (pElem->pData->signature != (uint64_t)pElem->pData) {
+    uError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData);
+    return;
+  }
+
+  pCacheObj->numOfElemsInTrash--;
+  if (pElem->prev) {
+    pElem->prev->next = pElem->next;
+  } else { /* pnode is the header, update header */
+    pCacheObj->pTrash = pElem->next;
+  }
+
+  if (pElem->next) {
+    pElem->next->prev = pElem->prev;
+  }
+
+  pElem->pData->signature = 0;
+  if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data);
+  free(pElem->pData);
+  free(pElem);
+}
+
+void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
+  __cache_wr_lock(pCacheObj);
+
+  if (pCacheObj->numOfElemsInTrash == 0) {
+    if (pCacheObj->pTrash != NULL) {
+      uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash);
+    }
+    pCacheObj->pTrash = NULL;
+
+    __cache_unlock(pCacheObj);
+    return;
+  }
+
+  STrashElem *pElem = pCacheObj->pTrash;
+
+  while (pElem) {
+    T_REF_VAL_CHECK(pElem->pData);
+    if (pElem->next == pElem) {
+      pElem->next = NULL;
+    }
+
+    if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
+      uTrace("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData,
+             pCacheObj->numOfElemsInTrash - 1);
+      STrashElem *p = pElem;
+
+      pElem = pElem->next;
+      taosRemoveFromTrashCan(pCacheObj, p);
+    } else {
+      pElem = pElem->next;
+    }
+  }
+
+  assert(pCacheObj->numOfElemsInTrash >= 0);
+  __cache_unlock(pCacheObj);
+}
+
+void doCleanupDataCache(SCacheObj *pCacheObj) {
+  __cache_wr_lock(pCacheObj);
+  taosHashCleanup(pCacheObj->pHashTable);
+  __cache_unlock(pCacheObj);
+
+  taosTrashCanEmpty(pCacheObj, true);
+  __cache_lock_destroy(pCacheObj);
+
+  memset(pCacheObj, 0, sizeof(SCacheObj));
+  free(pCacheObj);
+}
+
+void* taosCacheRefresh(void *handle) {
+  SCacheObj *pCacheObj = (SCacheObj *)handle;
+  if (pCacheObj == NULL) {
+    uTrace("object is destroyed. no refresh retry");
+    return NULL;
+  }
+
+  const int32_t SLEEP_DURATION = 500; //500 ms
+  int64_t totalTick = pCacheObj->refreshTime / SLEEP_DURATION;
+
+  int64_t count = 0;
+  while(1) {
+    taosMsleep(500);
+
+    // check if current cache object will be deleted every 500ms.
+    if (pCacheObj->deleting) {
+      break;
+    }
+
+    if (++count < totalTick) {
+      continue;
+    }
+
+    // reset the count value
+    count = 0;
+    size_t num = taosHashGetSize(pCacheObj->pHashTable);
+    if (num == 0) {
+      continue;
+    }
+
+    uint64_t expiredTime = taosGetTimestampMs();
+    pCacheObj->statistics.refreshCount++;
+
+    SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
+
+    __cache_wr_lock(pCacheObj);
+    while (taosHashIterNext(pIter)) {
+      SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
+      if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
+        taosCacheReleaseNode(pCacheObj, pNode);
+      }
+    }
+
+    __cache_unlock(pCacheObj);
+
+    taosHashDestroyIter(pIter);
+    taosTrashCanEmpty(pCacheObj, false);
+  }
+
+  return NULL;
 }

src/util/src/tdes.c

@@ -18,6 +18,7 @@ void generate_key(unsigned char* key);
 void generate_sub_keys(unsigned char* main_key, key_set* key_sets);
 void process_message(unsigned char* message_piece, unsigned char* processed_piece, key_set* key_sets, int mode);
 
+#if 0
 int64_t taosDesGenKey() {
   unsigned int iseed = (unsigned int)time(NULL);
   srand(iseed);
@@ -27,6 +28,7 @@ int64_t taosDesGenKey() {
 
   return *((int64_t*)key);
 }
+#endif
 
 char* taosDesImp(unsigned char* key, char* src, unsigned int len, int process_mode) {
   unsigned int  number_of_blocks = len / 8;

src/util/src/tkvstore.c

@@ -0,0 +1,292 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "hash.h"
+#include "os.h"
+#include "taoserror.h"
+#include "tchecksum.h"
+#include "tcoding.h"
+#include "tkvstore.h"
+#include "tulog.h"
+
+#define TD_KVSTORE_HEADER_SIZE 512
+#define TD_KVSTORE_MAJOR_VERSION 1
+#define TD_KVSTORE_MAINOR_VERSION 0
+#define TD_KVSTORE_SNAP_SUFFIX ".snap"
+#define TD_KVSTORE_NEW_SUFFIX ".new"
+
+static int       tdInitKVStoreHeader(int fd, char *fname);
+static void *    tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo);
+// static void *    tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo);
+static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
+static char *    tdGetKVStoreSnapshotFname(char *fdata);
+static char *    tdGetKVStoreNewFname(char *fdata);
+static void      tdFreeKVStore(SKVStore *pStore);
+static int       tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo);
+
+int tdCreateKVStore(char *fname) {
+  char *tname = strdup(fname);
+  if (tname == NULL) return TSDB_CODE_COM_OUT_OF_MEMORY;
+
+  int fd = open(fname, O_RDWR | O_CREAT, 0755);
+  if (fd < 0) {
+    uError("failed to open file %s since %s", fname, strerror(errno));
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  int code = tdInitKVStoreHeader(fd, fname);
+  if (code != TSDB_CODE_SUCCESS) return code;
+
+  if (fsync(fd) < 0) {
+    uError("failed to fsync file %s since %s", fname, strerror(errno));
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  if (close(fd) < 0) {
+    uError("failed to close file %s since %s", fname, strerror(errno));
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
+  SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH);
+  if (pStore == NULL) return NULL;
+
+  pStore->fd = open(pStore->fname, O_RDWR);
+  if (pStore->fd < 0) {
+    uError("failed to open file %s since %s", pStore->fname, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (access(pStore->fsnap, F_OK) == 0) {
+    uTrace("file %s exists, try to recover the KV store", pStore->fsnap);
+    pStore->sfd = open(pStore->fsnap, O_RDONLY);
+    if (pStore->sfd < 0) {
+      uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+
+    // TODO: rewind the file
+
+    close(pStore->sfd);
+    pStore->sfd = -1;
+    remove(pStore->fsnap);
+  }
+
+  // TODO: Recover from the file
+
+  return pStore;
+
+_err:
+  if (pStore->fd > 0) {
+    close(pStore->fd);
+    pStore->fd = -1;
+  }
+  if (pStore->sfd > 0) {
+    close(pStore->sfd);
+    pStore->sfd = -1;
+  }
+  tdFreeKVStore(pStore);
+  return NULL;
+}
+
+int tdKVStoreStartCommit(SKVStore *pStore) {
+  pStore->fd = open(pStore->fname, O_RDWR);
+  if (pStore->fd < 0) {
+    uError("failed to open file %s since %s", pStore->fname, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT, 0755);
+  if (pStore->sfd < 0) {
+    uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (tsendfile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+    uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (fsync(pStore->sfd) < 0) {
+    uError("failed to fsync file %s since %s", pStore->fsnap, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (close(pStore->sfd) < 0) {
+    uError("failed to close file %s since %s", pStore->fsnap, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+  pStore->sfd = -1;
+
+  return 0;
+
+_err:
+  if (pStore->sfd > 0) {
+    close(pStore->sfd);
+    pStore->sfd = -1;
+    remove(pStore->fsnap);
+  }
+  if (pStore->fd > 0) {
+    close(pStore->fd);
+    pStore->fd = -1;
+  }
+  return -1;
+}
+
+int tdKVStoreEndCommit(SKVStore *pStore) {
+  ASSERT(pStore->fd > 0);
+
+  terrno = tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &(pStore->info));
+  if (terrno != TSDB_CODE_SUCCESS) return -1;
+
+  if (fsync(pStore->fd) < 0) {
+    uError("failed to fsync file %s since %s", pStore->fname, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (close(pStore->fd) < 0) {
+    uError("failed to close file %s since %s", pStore->fname, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  remove(pStore->fsnap);
+  return 0;
+}
+
+static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
+  char buf[TD_KVSTORE_HEADER_SIZE] = "\0";
+
+  if (lseek(fd, 0, SEEK_SET) < 0) {
+    uError("failed to lseek file %s since %s", fname, strerror(errno));
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  tdEncodeStoreInfo(buf, pInfo);
+  taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
+  if (twrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+    uError("failed to write file %s %d bytes since %s", fname, TD_KVSTORE_HEADER_SIZE, strerror(errno));
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int tdInitKVStoreHeader(int fd, char *fname) {
+  SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0};
+
+  return tdUpdateKVStoreHeader(fd, fname, &info);
+}
+
+static void *tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo) {
+  buf = taosEncodeVariantI64(buf, pInfo->size);
+  buf = taosEncodeVariantI64(buf, pInfo->tombSize);
+  buf = taosEncodeVariantI64(buf, pInfo->nRecords);
+  buf = taosEncodeVariantI64(buf, pInfo->nDels);
+
+  return buf;
+}
+
+// static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) {
+//   buf = taosDecodeVariantI64(buf, &(pInfo->size));
+//   buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
+//   buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
+//   buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
+
+//   return buf;
+// }
+
+static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
+  SKVStore *pStore = (SKVStore *)malloc(sizeof(SKVStore));
+  if (pStore == NULL) goto _err;
+
+  pStore->fname = strdup(fname);
+  if (pStore->map == NULL) goto _err;
+
+  pStore->fsnap = tdGetKVStoreSnapshotFname(fname);
+  if (pStore->fsnap == NULL) goto _err;
+
+  pStore->fnew = tdGetKVStoreNewFname(fname);
+  if (pStore->fnew == NULL) goto _err;
+
+  pStore->fd = -1;
+  pStore->sfd = -1;
+  pStore->nfd = -1;
+  pStore->iFunc = iFunc;
+  pStore->aFunc = aFunc;
+  pStore->appH = appH;
+  pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+  if (pStore->map == NULL) {
+    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  return pStore;
+
+_err:
+  terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
+  tdFreeKVStore(pStore);
+  return NULL;
+}
+
+static void tdFreeKVStore(SKVStore *pStore) {
+  if (pStore) {
+    tfree(pStore->fname);
+    tfree(pStore->fsnap);
+    tfree(pStore->fnew);
+    taosHashCleanup(pStore->map);
+    free(pStore);
+  }
+}
+
+static char *tdGetKVStoreSnapshotFname(char *fdata) {
+  size_t size = strlen(fdata) + strlen(TD_KVSTORE_SNAP_SUFFIX) + 1;
+  char * fname = malloc(size);
+  if (fname == NULL) {
+    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
+    return NULL;
+  }
+  sprintf(fname, "%s%s", fdata, TD_KVSTORE_SNAP_SUFFIX);
+  return fname;
+}
+
+static char *tdGetKVStoreNewFname(char *fdata) {
+  size_t size = strlen(fdata) + strlen(TD_KVSTORE_NEW_SUFFIX) + 1;
+  char * fname = malloc(size);
+  if (fname == NULL) {
+    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
+    return NULL;
+  }
+  sprintf(fname, "%s%s", fdata, TD_KVSTORE_NEW_SUFFIX);
+  return fname;
+}

src/util/src/tlog.c

@@ -191,16 +191,15 @@ void taosResetLog() {
 }
 
 static bool taosCheckFileIsOpen(char *logFileName) {
-  int32_t exist = access(logFileName, F_OK);
+  int32_t fd = open(logFileName, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
-  if (exist != 0) {
-    return false;
-  }
-
-  int32_t fd = open(logFileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
   if (fd < 0) {
+    if (errno == ENOENT) {
+      return false;
+    } else {
       printf("\nfailed to open log file:%s, reason:%s\n", logFileName, strerror(errno));
       return true;
     }
+  }
 
   if (taosLockFile(fd)) {
     taosUnLockFile(fd);

src/util/src/tnote.c

@@ -87,6 +87,10 @@ void *taosThreadToOpenNewNote(void *param)
     umask(0);
 
     int fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+    if (fd < 0) {
+      return NULL;
+    }
+
     taosLockNote(fd, pNote);
     lseek(fd, 0, SEEK_SET);
 

src/util/src/tsched.c

@@ -15,6 +15,7 @@
 
 #include "os.h"
 #include "taosdef.h"
+#include "tutil.h"
 #include "tulog.h"
 #include "tsched.h"
 #include "ttimer.h"
@@ -62,8 +63,7 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) {
   }
 
   pSched->queueSize = queueSize;
-  strncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow
+  tstrncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow
-  pSched->label[sizeof(pSched->label)-1] = '\0';
 
   pSched->fullSlot = 0;
   pSched->emptySlot = 0;

src/util/src/ttimer.c

@@ -552,8 +552,7 @@ void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* lab
     return NULL;
   }
 
-  strncpy(ctrl->label, label, sizeof(ctrl->label));
+  tstrncpy(ctrl->label, label, sizeof(ctrl->label));
-  ctrl->label[sizeof(ctrl->label) - 1] = 0;
   tmrTrace("%s timer controller is initialized, number of timer controllers: %d.", label, numOfTmrCtrl);
   return ctrl;
 }

src/util/src/tutil.c

@@ -582,13 +582,13 @@ bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
 }
 
 int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) {
-  char client_version[64] = {0};
+  char client_version[TSDB_VERSION_LEN] = {0};
-  char server_version[64] = {0};
+  char server_version[TSDB_VERSION_LEN] = {0};
   int clientVersionNumber[4] = {0};
   int serverVersionNumber[4] = {0};
 
-  strcpy(client_version, input_client_version);
+  tstrncpy(client_version, input_client_version, sizeof(client_version));
-  strcpy(server_version, input_server_version);
+  tstrncpy(server_version, input_server_version, sizeof(server_version));
 
   if (!taosGetVersionNumber(client_version, clientVersionNumber)) {
     uError("invalid client version:%s", client_version);

src/util/tests/cacheTest.cpp

@@ -19,8 +19,7 @@ int32_t tsMaxMeterConnections = 200;
 // test cache
 TEST(testCase, client_cache_test) {
   const int32_t REFRESH_TIME_IN_SEC = 2;
-  void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC");
+  SCacheObj* tscCacheHandle = taosCacheInit(REFRESH_TIME_IN_SEC);
-  SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
 
   const char* key1 = "test1";
   char data1[] = "test11";
@@ -106,9 +105,7 @@ TEST(testCase, client_cache_test) {
 
 TEST(testCase, cache_resize_test) {
   const int32_t REFRESH_TIME_IN_SEC = 2;
-  void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC");
+  auto* pCache = taosCacheInit(REFRESH_TIME_IN_SEC);
-
-  auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
 
   char key[256] = {0};
   char data[1024] = "abcdefghijk";

src/vnode/inc/vnodeInt.h

@@ -51,7 +51,7 @@ typedef struct {
   SSyncCfg     syncCfg;
   SWalCfg      walCfg;
   char        *rootDir;
-  char         db[TSDB_DB_NAME_LEN + 1];
+  char         db[TSDB_DB_NAME_LEN];
 } SVnodeObj;
 
 int  vnodeWriteToQueue(void *param, void *pHead, int type);

src/vnode/src/vnodeMain.c

@@ -75,19 +75,29 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
     return TSDB_CODE_SUCCESS;
   }
 
-  mkdir(tsVnodeDir, 0755);
+  if (mkdir(tsVnodeDir, 0755) != 0 && errno != EEXIST) {
-
+    vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), tsVnodeDir);
-  char rootDir[TSDB_FILENAME_LEN] = {0};
+    if (errno == EACCES) {
-  sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
+      return TSDB_CODE_VND_NO_DISK_PERMISSIONS;
-  if (mkdir(rootDir, 0755) != 0) {
+    } else if (errno == ENOSPC) {
-    vPrint("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir);
+      return TSDB_CODE_VND_NO_DISKSPACE;
+    } else if (errno == ENOENT) {
+      return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR;
+    } else {
+      return TSDB_CODE_VND_INIT_FAILED;
+    }
+  }
+
+  char rootDir[TSDB_FILENAME_LEN] = {0};
+  sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
+  if (mkdir(rootDir, 0755) != 0 && errno != EEXIST) {
+    vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir);
     if (errno == EACCES) {
       return TSDB_CODE_VND_NO_DISK_PERMISSIONS;
     } else if (errno == ENOSPC) {
       return TSDB_CODE_VND_NO_DISKSPACE;
     } else if (errno == ENOENT) {
       return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR;
-    } else if (errno == EEXIST) {
     } else {
       return TSDB_CODE_VND_INIT_FAILED;
     }
@@ -371,13 +381,15 @@ void *vnodeGetWal(void *pVnode) {
 static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
   if (pVnode->status == TAOS_VN_STATUS_DELETING) return;
   if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
+  int64_t totalStorage, compStorage, pointsWritten = 0;
+  tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
 
   SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
   pLoad->vgId = htonl(pVnode->vgId);
   pLoad->cfgVersion = htonl(pVnode->cfgVersion);
-  pLoad->totalStorage = htobe64(pLoad->totalStorage);
+  pLoad->totalStorage = htobe64(totalStorage);
-  pLoad->compStorage = htobe64(pLoad->compStorage);
+  pLoad->compStorage = htobe64(compStorage);
-  pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
+  pLoad->pointsWritten = htobe64(pointsWritten);
   pLoad->status = pVnode->status;
   pLoad->role = pVnode->role;
   pLoad->replica = pVnode->syncCfg.replica;  

tests/examples/c/demo.c

@@ -21,14 +21,16 @@
 #include <stdlib.h>
 #include <string.h>
 #include <taos.h>  // TAOS header file
-#include <unistd.h>
+#include <sys/time.h>
-
+#include <inttypes.h>
-void taosMsleep(int mseconds);
 
 static int32_t doQuery(TAOS* taos, const char* sql) {
+  struct timeval t1 = {0};
+  gettimeofday(&t1, NULL);
+  
   TAOS_RES* res = taos_query(taos, sql);
   if (taos_errno(res) != 0) {
-    printf("failed to execute query, reason:%s\n", taos_errstr(taos));
+    printf("failed to execute query, reason:%s\n", taos_errstr(res));
     return -1;
   }
   
@@ -38,13 +40,19 @@ static int32_t doQuery(TAOS* taos, const char* sql) {
   int32_t numOfFields = taos_num_fields(res);
   TAOS_FIELD* pFields = taos_fetch_fields(res);
   
+  int32_t i = 0;
   while((row = taos_fetch_row(res)) != NULL) {
     taos_print_row(buf, row, pFields, numOfFields);
-    printf("%s\n", buf);
+    printf("%d:%s\n", ++i, buf);
     memset(buf, 0, 512);
   }
   
   taos_free_result(res);
+  
+  struct timeval t2 = {0};
+  gettimeofday(&t2, NULL);
+  
+  printf("elapsed time:%"PRId64 " ms\n", ((t2.tv_sec*1000000 + t2.tv_usec) - (t1.tv_sec*1000000 + t1.tv_usec))/1000);
   return 0;
 }
 
@@ -101,14 +109,18 @@ int main(int argc, char *argv[]) {
   
   taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
   if (taos == NULL) {
-    printf("failed to connect to server, reason:%s\n", taos_errstr(taos));
+    printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));
     exit(1);
   }
+  
   printf("success to connect to server\n");
+//  doQuery(taos, "select c1,count(*) from group_db0.group_mt0 where c1<8 group by c1");
+  doQuery(taos, "select * from test.m1");
 
 //  multiThreadTest(1, taos);
-  doQuery(taos, "use test");
+//  doQuery(taos, "select tbname from test.m1");
-  doQuery(taos, "alter table tm99 set tag a=99");
+//   doQuery(taos, "select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_db0.lm2_stb0 where ts >= 1537146000000 and ts <= 1543145400000 and tbname in ('lm2_tb0') interval(1s) group by t1");
+//   doQuery(taos, "select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_db0.lm2_stb0 where ts >= 1537146000000 and ts <= 1543145400000 and tbname in ('lm2_tb0', 'lm2_tb1', 'lm2_tb2') interval(1s)");
 //  for(int32_t i = 0; i < 100000; ++i) {
 //    doQuery(taos, "insert into t1 values(now, 2)");
 //  }

tests/pytest/crash_gen.py

@@ -730,13 +730,15 @@ class DbState():
     # when we re-run the test in 3 minutes (180 seconds), basically we should expand time duration
     # by a factor of 500.
     # TODO: what if it goes beyond 10 years into the future
+    # TODO: fix the error as result of above: "tsdb timestamp is out of range"
     def setupLastTick(self):
-        t1 = datetime.datetime(2020, 5, 30)
+        t1 = datetime.datetime(2020, 6, 1)
         t2 = datetime.datetime.now()
-        elSec = t2.timestamp() - t1.timestamp()
+        elSec = int(t2.timestamp() - t1.timestamp()) # maybe a very large number, takes 69 years to exceed Python int range
+        elSec2 = (  elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500 ) ) * 500 # a number representing seconds within 10 years
         # print("elSec = {}".format(elSec))
         t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years
-        t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec * 500) # see explanation above
+        t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above
         logger.info("Setting up TICKS to start from: {}".format(t4))
         return t4
 
@@ -963,7 +965,7 @@ class Task():
         try:
             self._executeInternal(te, wt) # TODO: no return value?
         except taos.error.ProgrammingError as err:
-            self.logDebug("[=] Taos library exception: errno={}, msg: {}".format(err.errno, err))
+            self.logDebug("[=] Taos library exception: errno={:X}, msg: {}".format(err.errno, err))
             self._err = err
         except:
             self.logDebug("[=] Unexpected exception")

tests/pytest/fulltest.sh

@@ -134,3 +134,7 @@ python3 ./test.py -f query/filter.py
 python3 ./test.py -f query/filterCombo.py
 python3 ./test.py -f query/queryNormal.py
 python3 ./test.py -f query/queryError.py
+
+#stream
+python3 ./test.py -f stream/stream1.py
+python3 ./test.py -f stream/stream2.py

tests/pytest/random-test/random-test-multi-threading-3.py

@@ -205,7 +205,8 @@ class Test (Thread):
         global written
 
         dnodesDir = tdDnodes.getDnodesRootDir()
-        dataDir = dnodesDir + '/dnode1/*'
+        tdDnodes.forcestop(1)
+        dataDir = dnodesDir + '/dnode1/data/*'
         deleteCmd = 'rm -rf %s' % dataDir
         os.system(deleteCmd)
 
@@ -261,7 +262,7 @@ class Test (Thread):
             while True:
                 self.queryEvent.wait()
                 tdLog.notice("third thread")
-                randQueryOp = random.randint(1, 9)
+                randQueryOp = random.randint(1, 2)
                 queryOp.get(randQueryOp, lambda: "ERROR")()
                 self.queryEvent.clear()
                 self.dbEvent.clear()

tests/pytest/random-test/random-test-multi-threading.py

@@ -208,12 +208,12 @@ class Test (threading.Thread):
         global written
 
         dnodesDir = tdDnodes.getDnodesRootDir()
-        dataDir = dnodesDir + '/dnode1/*'
+        tdDnodes.forcestop(1)
+        dataDir = dnodesDir + '/dnode1/data/*'
         deleteCmd = 'rm -rf %s' % dataDir
         os.system(deleteCmd)
 
         tdDnodes.start(1)
-#        tdLog.sleep(10)
         tdSql.prepare()
         last_tb = ""
         last_stb = ""

tests/pytest/random-test/random-test.py

@@ -166,7 +166,8 @@ class Test:
     def delete_datafiles(self):
         tdLog.info("delete_datafiles")
         dnodesDir = tdDnodes.getDnodesRootDir()
-        dataDir = dnodesDir + '/dnode1/*'
+        tdDnodes.forcestop(1)
+        dataDir = dnodesDir + '/dnode1/data/*'
         deleteCmd = 'rm -rf %s' % dataDir
         os.system(deleteCmd)
 

tests/pytest/regressiontest.sh

@@ -131,6 +131,11 @@ python3 ./test.py -f user/pass_len.py
 
 #query
 python3 ./test.py -f query/filter.py
-# python3 ./test.py -f query/filterCombo.py
+
-# python3 ./test.py -f query/queryNormal.py
+
-# python3 ./test.py -f query/queryError.py
+#stream
+python3 ./test.py -f stream/stream1.py
+python3 ./test.py -f stream/stream2.py
+
+
+

tests/pytest/stream/stream1.py

@@ -31,21 +31,26 @@ class TDTestCase:
         tdSql.prepare()
 
         tdLog.info("===== step1 =====")
-        tdSql.execute("create table stb0(ts timestamp, col1 int, col2 float) tags(tgcol int)")
+        tdSql.execute(
+            "create table stb0(ts timestamp, col1 int, col2 float) tags(tgcol int)")
         for i in range(tbNum):
             tdSql.execute("create table tb%d using stb0 tags(%d)" % (i, i))
             for j in range(rowNum):
-                tdSql.execute("insert into tb%d values (now - %dm, %d, %d)" % (i, 1440 - j, j, j))
+                tdSql.execute(
+                    "insert into tb%d values (now - %dm, %d, %d)" %
+                    (i, 1440 - j, j, j))
         time.sleep(0.1)
 
         tdLog.info("===== step2 =====")
-        tdSql.query("select count(*), count(col1), count(col2) from tb0 interval(1d)")
+        tdSql.query(
+            "select count(*), count(col1), count(col2) from tb0 interval(1d)")
         tdSql.checkData(0, 1, rowNum)
         tdSql.checkData(0, 2, rowNum)
         tdSql.checkData(0, 3, rowNum)
         tdSql.query("show tables")
         tdSql.checkRows(tbNum)
-        tdSql.execute("create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)")
+        tdSql.execute(
+            "create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)")
         tdSql.query("show tables")
         tdSql.checkRows(tbNum + 1)
 
@@ -67,7 +72,8 @@ class TDTestCase:
 
         tdLog.info("===== step6 =====")
         time.sleep(0.1)
-        tdSql.execute("create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)")
+        tdSql.execute(
+            "create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)")
         tdSql.query("show tables")
         tdSql.checkRows(tbNum + 1)
 
@@ -81,14 +87,16 @@ class TDTestCase:
         tdSql.checkData(0, 3, rowNum)
 
         tdLog.info("===== step8 =====")
-        tdSql.query("select count(*), count(col1), count(col2) from stb0 interval(1d)")
+        tdSql.query(
+            "select count(*), count(col1), count(col2) from stb0 interval(1d)")
         tdSql.checkData(0, 1, rowNum * tbNum)
         tdSql.checkData(0, 2, rowNum * tbNum)
         tdSql.checkData(0, 3, rowNum * tbNum)
         tdSql.query("show tables")
         tdSql.checkRows(tbNum + 1)
 
-        tdSql.execute("create table s1 as select count(*), count(col1), count(col2) from stb0 interval(1d)")
+        tdSql.execute(
+            "create table s1 as select count(*), count(col1), count(col2) from stb0 interval(1d)")
         tdSql.query("show tables")
         tdSql.checkRows(tbNum + 2)
 
@@ -110,7 +118,8 @@ class TDTestCase:
         tdSql.error("select * from s1")
 
         tdLog.info("===== step12 =====")
-        tdSql.execute("create table s1 as select count(*), count(col1), count(col2) from stb0 interval(1d)")
+        tdSql.execute(
+            "create table s1 as select count(*), count(col1), count(col2) from stb0 interval(1d)")
         tdSql.query("show tables")
         tdSql.checkRows(tbNum + 2)