Merge branch 'develop' into feature/changshuaiqiang/testcase

* develop: (48 commits)
  fix tsdb log format
  [TD-276] add user auth
  fix drop table coredump
  [td-225]
  [td-225] FIX BUGS IN DIFF
  add log in tsdb
  Additional fix for TD-225
  forward to peer only the data packet is from client or CQ
  [td-225] fix bug in diff query
  [td-225]
  [td-225] fix bugs in tags filter query
  scripts
  scripts
  [TD-271]
  support RpcIpSet change
  [TD-271] fix bug while drop stable
  integer division yields float in python3.
  change tables to maxtables according to Dr. Liao
  fixup: fix make command to make coverity_scan work.
  add log for database create
  ...

.travis.yml

@@ -49,7 +49,7 @@ matrix:
           ./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $?
 
           cd ${TRAVIS_BUILD_DIR}/tests/pytest
-          ./smoketest.sh -g 2>&1 | tee mem-error-out.txt
+          ./valgrind-test.sh -g 2>&1 | tee mem-error-out.txt
           sleep 1
 
           # Color setting
@@ -59,12 +59,12 @@ matrix:
           GREEN_UNDERLINE='\033[4;32m'
           NC='\033[0m'
 
-          grep 'ERROR SUMMARY' mem-error-out.txt | uniq | tee uniq-mem-error-out.txt
+          grep 'ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt
 
           for memError in `cat uniq-mem-error-out.txt | awk '{print $4}'` 
           do
             if [ -n "$memError" ]; then
-              if [ "$memError" -gt 16 ]; then
+              if [ "$memError" -gt 5 ]; then
                 echo -e "${RED} ## Memory errors number valgrind reports is $memError.\
                          More than our threshold! ## ${NC}"
                 travis_terminate $memError
@@ -72,11 +72,11 @@ matrix:
             fi
           done
 
-          grep 'definitely lost:' mem-error-out.txt | uniq | tee uniq-definitely-lost-out.txt
+          grep 'definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt
-          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $4}'`
+          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'`
           do
             if [ -n "$defiMemError" ]; then
-              if [ "$defiMemError" -gt 16 ]; then
+              if [ "$defiMemError" -gt 3 ]; then
                 echo -e "${RED} ## Memory errors number valgrind reports \
                          Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
                 travis_terminate $defiMemError
@@ -115,7 +115,7 @@ matrix:
 
         # The command that will be added as an argument to "cov-build" to compile your project for analysis,
         # ** likely specific to your build **
-        build_command: make > /dev/null
+        build_command: make
 
         # Pattern to match selecting branches that will run analysis. We recommend leaving this set to 'coverity_scan'.
         # Take care in resource usage, and consider the build frequency allowances per
@@ -163,11 +163,9 @@ matrix:
 
           cd ${TRAVIS_BUILD_DIR}/tests
 
-          ./test-all.sh
+          ./test-all.sh $TRAVIS_EVENT_TYPE COVER
 
-          if [ "$?" -ne "0" ]; then
+          TEST_RESULT=$?
-            travis_terminate $?
-          fi
 
           pkill taosd
           sleep 1
@@ -199,6 +197,9 @@ matrix:
             echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} "
           fi
 
+          if [ "$TEST_RESULT" -ne "0" ]; then
+            travis_terminate $?
+          fi
           ;;
         esac
 

README.md

@@ -121,7 +121,7 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo
 
 1.  sudo apt install
     build-essential cmake net-tools python-pip python-setuptools python3-pip
-    python3-setuptools valgrind
+    python3-setuptools valgrind psmisc curl
 
 2.  git clone <https://github.com/taosdata/TDengine>; cd TDengine
 

src/client/inc/tscLog.h

@@ -22,25 +22,25 @@ extern "C" {
 
 #include "tlog.h"
 
-extern int32_t cdebugFlag;
+extern int32_t cDebugFlag;
 
 #define tscError(...)                        \
-  if (cdebugFlag & DEBUG_ERROR) {            \
+  if (cDebugFlag & DEBUG_ERROR) {            \
     taosPrintLog("ERROR TSC ", 255, __VA_ARGS__); \
   }
 #define tscWarn(...)                                \
-  if (cdebugFlag & DEBUG_WARN) {                    \
+  if (cDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  TSC ", cdebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN  TSC ", cDebugFlag, __VA_ARGS__); \
   }
 #define tscTrace(...)                         \
-  if (cdebugFlag & DEBUG_TRACE) {             \
+  if (cDebugFlag & DEBUG_TRACE) {             \
-    taosPrintLog("TSC ", cdebugFlag, __VA_ARGS__); \
+    taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); \
   }
 #define tscPrint(...) \
   { taosPrintLog("TSC ", 255, __VA_ARGS__); }
 #define tscDump(...)                                      \
-  if (cdebugFlag & DEBUG_TRACE) {                         \
+  if (cDebugFlag & DEBUG_TRACE) {                         \
-    taosPrintLongString("TSC ", cdebugFlag, __VA_ARGS__); \
+    taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); \
   }
 
 #ifdef __cplusplus

src/client/inc/tsclient.h

@@ -365,7 +365,7 @@ void    tscInitMsgsFp();
 
 int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
 
-void tscProcessMsgFromServer(SRpcMsg *rpcMsg);
+void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet);
 int  tscProcessSql(SSqlObj *pSql);
 
 int  tscRenewMeterMeta(SSqlObj *pSql, char *tableId);

src/client/src/TSDBJNIConnector.c

@@ -23,16 +23,16 @@
 #include "ttime.h"
 
 #define jniError(...)                                 \
-  if (jnidebugFlag & DEBUG_ERROR) {                   \
+  if (jniDebugFlag & DEBUG_ERROR) {                   \
-    taosPrintLog("ERROR JNI ", jnidebugFlag, __VA_ARGS__); \
+    taosPrintLog("ERROR JNI ", jniDebugFlag, __VA_ARGS__); \
   }
 #define jniWarn(...)                                  \
-  if (jnidebugFlag & DEBUG_WARN) {                    \
+  if (jniDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  JNI ", jnidebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN  JNI ", jniDebugFlag, __VA_ARGS__); \
   }
 #define jniTrace(...)                           \
-  if (jnidebugFlag & DEBUG_TRACE) {             \
+  if (jniDebugFlag & DEBUG_TRACE) {             \
-    taosPrintLog("JNI ", jnidebugFlag, __VA_ARGS__); \
+    taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \
   }
 #define jniPrint(...) \
   { taosPrintLog("JNI ", 255, __VA_ARGS__); }

src/client/src/tscFunctionImpl.c

@@ -1691,10 +1691,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
 }
 
 static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  assert(pCtx->order != TSDB_ORDER_ASC);
-    return;
-  }
-  
   void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
@@ -2912,7 +2909,7 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 static void date_col_output_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->scanFlag == SUPPLEMENTARY_SCAN) {
+  if (pCtx->scanFlag == REVERSE_SCAN) {
     return;
   }
   
@@ -2969,11 +2966,12 @@ static void tag_project_function(SQLFunctionCtx *pCtx) {
     char* output = pCtx->aOutputBuf;
   
     if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) {
-      *(int16_t*) output = pCtx->tag.nLen;
+      varDataSetLen(output, pCtx->tag.nLen);
-      output += VARSTR_HEADER_SIZE;
+      tVariantDump(&pCtx->tag, varDataVal(output), pCtx->outputType);
+    } else {
+      tVariantDump(&pCtx->tag, output, pCtx->outputType);
     }
     
-    tVariantDump(&pCtx->tag, output, pCtx->outputType);
     pCtx->aOutputBuf += pCtx->outputBytes;
   }
 }

src/client/src/tscSQLParser.c

@@ -5563,8 +5563,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       }
       
-      *(VarDataLenT*)tagVal = pList->a[i].pVar.nLen;
+      ret = tVariantDump(&(pList->a[i].pVar), varDataVal(tagVal), pTagSchema[i].type);
-      ret = tVariantDump(&(pList->a[i].pVar), tagVal + VARSTR_HEADER_SIZE, pTagSchema[i].type);
+      varDataSetLen(tagVal, pList->a[i].pVar.nLen);
     } else {
       ret = tVariantDump(&(pList->a[i].pVar), tagVal, pTagSchema[i].type);
     }
@@ -5911,7 +5911,7 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
   }
   
   if (pSqlExpr->pLeft == NULL) {
-    if (pSqlExpr->nSQLOptr >= TK_TINYINT && pSqlExpr->nSQLOptr <= TK_DOUBLE) {
+    if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) {
       *pExpr = calloc(1, sizeof(tExprNode));
       (*pExpr)->nodeType = TSQL_NODE_VALUE;
       (*pExpr)->pVal = calloc(1, sizeof(tVariant));

src/client/src/tscServer.c

@@ -221,7 +221,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 }
 
-void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
+void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   SSqlObj *pSql = (SSqlObj *)rpcMsg->handle;
   if (pSql == NULL) {
     tscError("%p sql is already released", pSql->signature);
@@ -245,6 +245,12 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
     return;
   }
 
+  if (pCmd->command < TSDB_SQL_MGMT) {
+    if (pIpSet) pSql->ipList = *pIpSet;
+  } else {
+    if (pIpSet) tscMgmtIpSet = *pIpSet;
+  }
+
   if (rpcMsg->pCont == NULL) {
     rpcMsg->code = TSDB_CODE_NETWORK_UNAVAIL;
   } else {
@@ -492,13 +498,15 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pMsg += sizeof(pQueryInfo->type);
 
   // todo valid the vgroupId at the client side
-  if (UTIL_TABLE_IS_SUPERTABLE(pQueryInfo->pTableMetaInfo[0])) {
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-    SVgroupsInfo* pVgroupInfo = pQueryInfo->pTableMetaInfo[0]->vgroupList;
+  
-    assert(pVgroupInfo->numOfVgroups == 1); // todo fix me
+  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    int32_t vgIndex = pTableMetaInfo->vgroupIndex;
     
-    pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[0].vgId);
+    SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList;
+    pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
   } else {
-    STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta;
+    STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
     pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
   }
   

src/client/src/tscSql.c

@@ -172,7 +172,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) {
 }
 
 TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
-  tscTrace("try to create a connection to %s", ip);
+  tscTrace("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db);
 
   STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL);
   if (pObj != NULL) {
@@ -191,7 +191,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
       return NULL;
     }
     
-    tscTrace("%p DB connection is opening", pObj);
+    tscTrace("%p DB connection is opening, dnodeConn:%p", pObj, pObj->pDnodeConn);
     
     // version compare only requires the first 3 segments of the version string
     int code = taosCheckVersion(version, taos_get_server_info(pObj), 3);

src/client/src/tscSubquery.c

@@ -1436,8 +1436,8 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
       return;
     } else {  // reach the maximum retry count, abort
       atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows);
-      tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql,
+      tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pPObj, pSql,
-               numOfRows, subqueryIndex, tstrerror(pState->code));
+               tstrerror(numOfRows), subqueryIndex, tstrerror(pState->code));
     }
   }
   
@@ -1450,7 +1450,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
   }
   
   // all subqueries are failed
-  tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, pState->numOfTotal, pState->code);
+  tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pPObj, pState->numOfTotal, tstrerror(pState->code));
   pPObj->res.code = pState->code;
   
   // release allocated resource

src/client/src/tscSystem.c

@@ -63,12 +63,15 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) {
     rpcInit.user = (char*)user;
     rpcInit.idleTime = 2000;
     rpcInit.ckey = "key";
+    rpcInit.spi = 1;
     rpcInit.secret = secretEncrypt;
 
     *pDnodeConn = rpcOpen(&rpcInit);
     if (*pDnodeConn == NULL) {
       tscError("failed to init connection to TDengine");
       return -1;
+    } else {
+      tscTrace("dnodeConn:%p is created, user:%s", *pDnodeConn, user);
     }
   }
 

src/client/src/tscUtil.c

@@ -766,7 +766,7 @@ void tscCloseTscObj(STscObj* pObj) {
     rpcClose(pObj->pDnodeConn);
   }
   
-  tscTrace("%p DB connection is closed", pObj);
+  tscTrace("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn);
   tfree(pObj);
 }
 
@@ -1144,23 +1144,21 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) {
 }
 
 SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) {
-  SColumnFilterInfo* pFilter = NULL;
+  if (numOfFilters == 0) {
-  if (numOfFilters > 0) {
-    pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo));
-  } else {
     assert(src == NULL);
     return NULL;
   }
   
+  SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo));
+  
   memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters);
   for (int32_t j = 0; j < numOfFilters; ++j) {
+    
     if (pFilter[j].filterstr) {
       size_t len = (size_t) pFilter[j].len + 1;
-  
+      pFilter[j].pz = (int64_t) calloc(1, len);
-      char*  pTmp   = calloc(1, len);
-      pFilter[j].pz = (int64_t) pTmp;
       
-      memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len);
+      memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t)len);
     }
   }
   
@@ -1675,10 +1673,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
   pTableMetaInfo->pTableMeta = pTableMeta;
   
   if (vgroupList != NULL) {
-    assert(vgroupList->numOfVgroups == 1);  // todo fix me
-    
     size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
-    
     pTableMetaInfo->vgroupList = malloc(size);
     memcpy(pTableMetaInfo->vgroupList, vgroupList, size);
   }

src/common/inc/tdataformat.h

@@ -115,10 +115,8 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
       return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
-      break;
     default:
       return POINTER_SHIFT(row, offset);
-      break;
   }
 }
 

src/common/inc/tglobal.h

@@ -141,9 +141,9 @@ extern int32_t tsAsyncLog;
 extern int32_t tsNumOfLogLines;
 extern int32_t dDebugFlag;
 extern int32_t vDebugFlag;
-extern int32_t mdebugFlag;
+extern int32_t mDebugFlag;
-extern int32_t cdebugFlag;
+extern int32_t cDebugFlag;
-extern int32_t jnidebugFlag;
+extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
@@ -151,8 +151,8 @@ extern int32_t monitorDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t debugFlag;
-extern int32_t odbcdebugFlag;
+extern int32_t odbcDebugFlag;
-extern int32_t qdebugFlag;
+extern int32_t qDebugFlag;
 
 extern uint32_t taosMaxTmrCtrl;
 

src/common/src/tglobal.c

@@ -127,16 +127,16 @@ int32_t tsRestRowLimit = 10240;
 int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
 
 int32_t tsNumOfLogLines = 10000000;
-int32_t mdebugFlag = 135;
+int32_t mDebugFlag = 135;
 int32_t sdbDebugFlag = 135;
 int32_t dDebugFlag = 135;
 int32_t vDebugFlag = 135;
-int32_t cdebugFlag = 135;
+int32_t cDebugFlag = 135;
-int32_t jnidebugFlag = 131;
+int32_t jniDebugFlag = 131;
-int32_t odbcdebugFlag = 131;
+int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
 int32_t monitorDebugFlag = 131;
-int32_t qdebugFlag = 131;
+int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 135;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
@@ -204,19 +204,19 @@ static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
 
 void taosSetAllDebugFlag() {
   for (int32_t i = 0; i < tsGlobalConfigNum; ++i) {
-    mdebugFlag = debugFlag;
+    mDebugFlag = debugFlag;
     sdbDebugFlag = debugFlag;
     dDebugFlag = debugFlag;
     vDebugFlag = debugFlag;
-    cdebugFlag = debugFlag;
+    cDebugFlag = debugFlag;
-    jnidebugFlag = debugFlag;
+    jniDebugFlag = debugFlag;
-    odbcdebugFlag = debugFlag;
+    odbcDebugFlag = debugFlag;
     httpDebugFlag = debugFlag;
     monitorDebugFlag = debugFlag;
     rpcDebugFlag = debugFlag;
     uDebugFlag = debugFlag;
     sDebugFlag = debugFlag;
-    //qdebugFlag = debugFlag;    
+    //qDebugFlag = debugFlag;    
   }
   uPrint("all debug flag are set to %d", debugFlag);
 }
@@ -1002,7 +1002,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "mDebugFlag";
-  cfg.ptr = &mdebugFlag;
+  cfg.ptr = &mDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
   cfg.minValue = 0;
@@ -1062,7 +1062,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "cDebugFlag";
-  cfg.ptr = &cdebugFlag;
+  cfg.ptr = &cDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1072,7 +1072,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "jniDebugFlag";
-  cfg.ptr = &jnidebugFlag;
+  cfg.ptr = &jniDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1082,7 +1082,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "odbcDebugFlag";
-  cfg.ptr = &odbcdebugFlag;
+  cfg.ptr = &odbcDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1122,7 +1122,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "qDebugFlag";
-  cfg.ptr = &qdebugFlag;
+  cfg.ptr = &qDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;

src/dnode/src/dnodeMain.c

@@ -167,9 +167,9 @@ static int32_t dnodeInitSystem() {
   if (dnodeInitRead() != 0) return -1;
   if (dnodeInitWrite() != 0) return -1;
   if (dnodeInitClient() != 0) return -1;
-  if (dnodeInitModules() != 0) return -1;
   if (dnodeInitServer() != 0) return -1;
   if (dnodeInitMgmt() != 0) return -1;
+  if (dnodeInitModules() != 0) return -1;
   if (dnodeInitShell() != 0) return -1;
 
   dnodeStartModules();
@@ -184,14 +184,14 @@ static void dnodeCleanUpSystem() {
   if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
     dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
     dnodeCleanupShell();
-    dnodeCleanupServer();
+    dnodeCleanUpModules();
     dnodeCleanupMgmt();
+    dnodeCleanupServer();
     dnodeCleanupClient();
     dnodeCleanupWrite();
     dnodeCleanupRead();
-    dnodeCleanUpModules();
-    taos_cleanup();
     dnodeCleanupStorage();
+    taos_cleanup();
     taosCloseLog();
   }
 }

src/dnode/src/dnodeMgmt.c

@@ -266,9 +266,12 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
   return taosCfgDynamicOptions(pCfg->config);
 }
 
+void dnodeUpdateIpSet(SRpcIpSet *pIpSet) {
+  dPrint("mnode IP list is changed, numOfIps:%d inUse:%d", pIpSet->numOfIps, pIpSet->inUse);
+  for (int i = 0; i < pIpSet->numOfIps; ++i) {
+    dPrint("mnode index:%d %s:%u", i, pIpSet->fqdn[i], pIpSet->port[i])
+  }
 
-void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
-  dTrace("mgmt IP list is changed for ufp is called");
   tsMnodeIpSet = *pIpSet;
 }
 

src/dnode/src/dnodePeer.c

@@ -29,11 +29,11 @@
 #include "dnodeVWrite.h"
 #include "mnode.h"
 
-extern void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet);
+extern void dnodeUpdateIpSet(SRpcIpSet *pIpSet);
 static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
-static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg);
+static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *);
 static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
-static void dnodeProcessRspFromDnode(SRpcMsg *pMsg);
+static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet);
 static void *tsDnodeServerRpc = NULL;
 static void *tsDnodeClientRpc = NULL;
 
@@ -52,7 +52,8 @@ int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mgmtProcessReqMsgFromDnode;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = mgmtProcessReqMsgFromDnode;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = mgmtProcessReqMsgFromDnode;
-
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = mgmtProcessReqMsgFromDnode;
+  
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort    = tsDnodeDnodePort;
@@ -61,7 +62,7 @@ int32_t dnodeInitServer() {
   rpcInit.cfp          = dnodeProcessReqMsgFromDnode;
   rpcInit.sessions     = 100;
   rpcInit.connType     = TAOS_CONN_SERVER;
-  rpcInit.idleTime     = tsShellActivityTimer * 2000;
+  rpcInit.idleTime     = tsShellActivityTimer * 1000;
 
   tsDnodeServerRpc = rpcOpen(&rpcInit);
   if (tsDnodeServerRpc == NULL) {
@@ -81,7 +82,7 @@ void dnodeCleanupServer() {
   }
 }
 
-static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg) {
+static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
   SRpcMsg rspMsg;
   rspMsg.handle  = pMsg->handle;
   rspMsg.pCont   = NULL;
@@ -119,10 +120,9 @@ int32_t dnodeInitClient() {
   rpcInit.label        = "DND-C";
   rpcInit.numOfThreads = 1;
   rpcInit.cfp          = dnodeProcessRspFromDnode;
-  rpcInit.ufp          = dnodeUpdateIpSet;
   rpcInit.sessions     = 100;
   rpcInit.connType     = TAOS_CONN_CLIENT;
-  rpcInit.idleTime     = tsShellActivityTimer * 2000;
+  rpcInit.idleTime     = tsShellActivityTimer * 1000;
   rpcInit.user         = "t";
   rpcInit.ckey         = "key";
   rpcInit.secret       = "secret";
@@ -145,9 +145,10 @@ void dnodeCleanupClient() {
   }
 }
 
-static void dnodeProcessRspFromDnode(SRpcMsg *pMsg) {
+static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
 
   if (dnodeProcessRspMsgFp[pMsg->msgType]) {
+    if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pIpSet) dnodeUpdateIpSet(pIpSet);
     (*dnodeProcessRspMsgFp[pMsg->msgType])(pMsg);
   } else {
     dError("RPC %p, msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]);
@@ -163,3 +164,9 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
 void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) {
   rpcSendRequest(tsDnodeClientRpc, ipSet, rpcMsg);
 }
+
+void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
+  SRpcIpSet ipSet = {0};
+  dnodeGetMnodeDnodeIpSet(&ipSet);
+  rpcSendRecv(tsDnodeClientRpc, &ipSet, rpcMsg, rpcRsp);
+}

src/dnode/src/dnodeShell.c

@@ -21,6 +21,7 @@
 #include "trpc.h"
 #include "tglobal.h"
 #include "http.h"
+#include "mnode.h"
 #include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeVRead.h"
@@ -28,7 +29,7 @@
 #include "dnodeShell.h"
 
 static void  (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
-static void    dnodeProcessMsgFromShell(SRpcMsg *pMsg);
+static void    dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *);
 static int     dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey);
 static void  * tsDnodeShellRpc = NULL;
 static int32_t tsDnodeQueryReqNum  = 0;
@@ -86,7 +87,7 @@ int32_t dnodeInitShell() {
   rpcInit.cfp          = dnodeProcessMsgFromShell;
   rpcInit.sessions     = TSDB_SESSIONS_PER_DNODE;
   rpcInit.connType     = TAOS_CONN_SERVER;
-  rpcInit.idleTime     = tsShellActivityTimer * 1500;
+  rpcInit.idleTime     = tsShellActivityTimer * 1000;
   rpcInit.afp          = dnodeRetrieveUserAuthInfo;
 
   tsDnodeShellRpc = rpcOpen(&rpcInit);
@@ -106,7 +107,7 @@ void dnodeCleanupShell() {
   }
 }
 
-void dnodeProcessMsgFromShell(SRpcMsg *pMsg) {
+void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
   SRpcMsg rpcMsg;
   rpcMsg.handle = pMsg->handle;
   rpcMsg.pCont = NULL;
@@ -138,7 +139,34 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg) {
 }
 
 static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  return TSDB_CODE_SUCCESS;
+  int code = mgmtRetriveAuth(user, spi, encrypt, secret, ckey);
+  if (code != TSDB_CODE_NOT_READY) return code;
+
+  SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
+  strcpy(pMsg->user, user);
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pMsg;
+  rpcMsg.contLen = sizeof(SDMAuthMsg);
+  rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH;
+  
+  dTrace("user:%s, send auth msg to mnode", user);
+  SRpcMsg rpcRsp = {0};
+  dnodeSendMsgToDnodeRecv(&rpcMsg, &rpcRsp);
+
+  if (rpcRsp.code != 0) {
+    dError("user:%s, auth msg received from mnode, error:%s", user, tstrerror(rpcRsp.code));
+  } else {
+    dTrace("user:%s, auth msg received from mnode", user);
+    SDMAuthRsp *pRsp = rpcRsp.pCont;
+    memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
+    memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
+    *spi = pRsp->spi;
+    *encrypt = pRsp->encrypt;
+  }
+
+  rpcFreeCont(rpcRsp.pCont);
+  return rpcRsp.code;
 }
 
 SDnodeStatisInfo dnodeGetStatisInfo() {

src/inc/dnode.h

@@ -52,6 +52,7 @@ int32_t  dnodeGetDnodeId();
 void     dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void     dnodeAddServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void     dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg);
+void     dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
 
 #ifdef __cplusplus
 }

src/inc/mnode.h

@@ -26,8 +26,10 @@ void    mgmtCleanUpSystem();
 void    mgmtStopSystem();
 void    sdbUpdateSync();
 
-void   mgmtProcessMsgFromShell(SRpcMsg *rpcMsg);
+int32_t mgmtRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-void   mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg);
+void    mgmtProcessMsgFromShell(SRpcMsg *rpcMsg);
+void    mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg);
+
 
 #ifdef __cplusplus
 }

src/inc/taosdef.h

@@ -36,6 +36,11 @@ extern "C" {
 typedef int32_t VarDataOffsetT;
 typedef int16_t VarDataLenT;
 
+typedef struct tstr {
+  VarDataLenT len;
+  char        data[];
+} tstr;
+
 #define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
 
 #define varDataLen(v)       ((VarDataLenT *)(v))[0]
@@ -74,10 +79,10 @@ extern const int32_t TYPE_BYTES[11];
 #define TSDB_DATA_NULL_STR              "NULL"
 #define TSDB_DATA_NULL_STR_L            "null"
 
-#define TSDB_TRUE 1
+#define TSDB_TRUE   1
-#define TSDB_FALSE 0
+#define TSDB_FALSE  0
-#define TSDB_OK 0
+#define TSDB_OK     0
-#define TSDB_ERR -1
+#define TSDB_ERR   -1
 
 #define TS_PATH_DELIMITER "."
 

src/inc/taosmsg.h

@@ -100,6 +100,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_TABLE, "config-table" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_VNODE, "config-vnode" )	
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_STATUS, "status" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_GRANT, "grant" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_AUTH, "auth" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
@@ -737,6 +738,14 @@ typedef struct {
   char     tableId[TSDB_TABLE_ID_LEN + 1];
 } SMDAlterStreamMsg;
 
+typedef struct {
+  char user[TSDB_USER_LEN + 1];
+  char spi;
+  char encrypt;
+  char secret[TSDB_KEY_LEN + 1];
+  char ckey[TSDB_KEY_LEN + 1];
+} SDMAuthMsg, SDMAuthRsp;
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

src/inc/trpc.h

@@ -66,10 +66,7 @@ typedef struct {
   char *ckey;         // ciphering key
 
   // call back to process incoming msg, code shall be ignored by server app
-  void (*cfp)(SRpcMsg *);  
+  void (*cfp)(SRpcMsg *, SRpcIpSet *);  
-
-  // call back to process notify the ipSet changes, for client app only
-  void (*ufp)(void *ahandle, SRpcIpSet *pIpSet);
 
   // call back to retrieve the client auth info, for server app only 
   int  (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);

src/inc/tsdb.h

@@ -102,13 +102,15 @@ int  tsdbTableSetSName(STableCfg *config, char *sname, bool dup);
 void tsdbClearTableCfg(STableCfg *config);
 
 int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId* id, int32_t colId, int16_t *type, int16_t *bytes, char **val);
-int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name);
+char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes);
 
 int   tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
 int   tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
 int   tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
 TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid);
 
+uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size);
+
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
   STsdbCfg tsdbCfg;

src/mnode/inc/mgmtDef.h

@@ -85,8 +85,7 @@ typedef struct SSuperTableObj {
   int32_t    numOfTables;
   int16_t    nextColId;
   SSchema *  schema;
-  int32_t    vgLen;
+  void *     vgHash;
-  int32_t *  vgList;
 } SSuperTableObj;
 
 typedef struct {

src/mnode/inc/mgmtInt.h

@@ -23,13 +23,13 @@ extern "C" {
 #include "tlog.h"
 #include "monitor.h"
 
-extern int32_t mdebugFlag;
+extern int32_t mDebugFlag;
 extern int32_t sdbDebugFlag;
 
 // mnode log function
-#define mError(...) if (mdebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND ", 255, __VA_ARGS__); }
+#define mError(...) if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND ", 255, __VA_ARGS__); }
-#define mWarn(...)  if (mdebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  MND ", mdebugFlag, __VA_ARGS__); }
+#define mWarn(...)  if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  MND ", mDebugFlag, __VA_ARGS__); }
-#define mTrace(...) if (mdebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mdebugFlag, __VA_ARGS__); }
+#define mTrace(...) if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }
 #define mPrint(...)                               { taosPrintLog("MND ", 255, __VA_ARGS__); }
 
 #define mLError(...) monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__)

src/mnode/src/mgmtDb.c

@@ -297,8 +297,10 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
   if (pDb != NULL) {
     mgmtDecDbRef(pDb); 
     if (pCreate->ignoreExist) {
+      mTrace("db:%s, already exist, ignore exist is set", pCreate->db);
       return TSDB_CODE_SUCCESS;
     } else {
+      mError("db:%s, is already exist, ignore exist not set", pCreate->db);
       return TSDB_CODE_DB_ALREADY_EXIST;
     }
   }
@@ -751,6 +753,8 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
     code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate);
     if (code == TSDB_CODE_SUCCESS) {
       mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user);
+    } else {
+      mError("db:%s, failed to create, reason:%s", pCreate->db, tstrerror(code));
     }
   }
 

src/mnode/src/mgmtSdb.c

@@ -278,7 +278,7 @@ void sdbUpdateSync() {
     sdbPrint("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort);
   }
 
-  SSyncInfo syncInfo;
+  SSyncInfo syncInfo = {0};
   syncInfo.vgId = 1;
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;

src/mnode/src/mgmtServer.c

@@ -88,10 +88,12 @@ void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg) {
     
     SRpcIpSet ipSet = {0};
     dnodeGetMnodeDnodeIpSet(&ipSet);
+    for (int i = 0; i < ipSet.numOfIps; ++i) 
+      ipSet.port[i] = htons(ipSet.port[i]);
     
     mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
     for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
-      mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
+      mTrace("mnode index:%d %s:%d", i, ipSet.fqdn[i], htons(ipSet.port[i]));
     }
     rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
     return;

src/mnode/src/mgmtShell.c

@@ -41,7 +41,6 @@
 typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-//static int  mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
 static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg);
 static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg);
 static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg);
@@ -137,7 +136,7 @@ void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
     mgmtGetMnodeIpSet(&ipSet);
     mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
     for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
-      mTrace("index:%d ip:%s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
+      mTrace("mnode index:%d ip:%s:%d", i, ipSet.fqdn[i], htons(ipSet.port[i]));
     }
 
     rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
@@ -343,29 +342,6 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
-/*
-static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  *spi = 1;
-  *encrypt = 0;
-  *ckey = 0;
-
-  if (!sdbIsMaster()) {
-    *secret = 0;
-    return TSDB_CODE_NOT_READY;
-  }
-
-  SUserObj *pUser = mgmtGetUser(user);
-  if (pUser == NULL) {
-    *secret = 0;
-    return TSDB_CODE_INVALID_USER;
-  } else {
-    memcpy(secret, pUser->pass, TSDB_KEY_LEN);
-    mgmtDecUserRef(pUser);
-    return TSDB_CODE_SUCCESS;
-  }
-}
-*/
-
 static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   SCMConnectMsg *pConnectMsg = pMsg->pCont;

src/mnode/src/mgmtTable.c

@@ -24,6 +24,7 @@
 #include "tname.h"
 #include "tidpool.h"
 #include "tglobal.h"
+#include "hash.h"
 #include "dnode.h"
 #include "mgmtDef.h"
 #include "mgmtInt.h"
@@ -363,39 +364,35 @@ static void mgmtCleanUpChildTables() {
 }
 
 static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
-  if (pStable->vgLen == 0) {
-    pStable->vgLen = 8;
-    pStable->vgList = calloc(pStable->vgLen, sizeof(int32_t));
-  }
-  
-  bool find = false;
-  int32_t pos = 0;
-  for (pos = 0; pos < pStable->vgLen; ++pos) {
-    if (pStable->vgList[pos] == 0) break;
-    if (pStable->vgList[pos] == pCtable->vgId) {
-      find = true;
-      break;
-    }
-  }
-
-  if (!find) {
-    if (pos >= pStable->vgLen) {
-      pStable->vgLen *= 2;
-      pStable->vgList = realloc(pStable->vgList, pStable->vgLen * sizeof(int32_t));
-    }
-    pStable->vgList[pos] = pCtable->vgId;
-  }
-
   pStable->numOfTables++;
+
+  if (pStable->vgHash == NULL) {
+    pStable->vgHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
+  }
+
+  if (pStable->vgHash != NULL) {
+    taosHashPut(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId));
+  }
 }
 
 static void mgmtRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
   pStable->numOfTables--;
+
+  if (pStable->vgHash == NULL) return;
+
+  SVgObj *pVgroup = mgmtGetVgroup(pCtable->vgId);
+  if (pVgroup != NULL) {
+    taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId));
+  }
+  mgmtDecVgroupRef(pVgroup);
 }
 
 static void mgmtDestroySuperTable(SSuperTableObj *pStable) {
+  if (pStable->vgHash != NULL) {
+    taosHashCleanup(pStable->vgHash);
+    pStable->vgHash = NULL;
+  }
   tfree(pStable->schema);
-  tfree(pStable->vgList)
   tfree(pStable);
 }
 
@@ -434,7 +431,7 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
     void *oldSchema = pTable->schema;
     memcpy(pTable, pNew, pOper->rowSize);
     pTable->schema = pNew->schema;
-    free(pNew->vgList);
+    free(pNew->vgHash);
     free(pNew);
     free(oldSchema);
   }
@@ -797,26 +794,26 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
 static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
   SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
   if (pStable->numOfTables != 0) {
-    mgmtDropAllChildTablesInStable(pStable);
+    SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
-    for (int32_t vg = 0; vg < pStable->vgLen; ++vg) {
+    while (taosHashIterNext(pIter)) {
-      int32_t vgId = pStable->vgList[vg];
+      int32_t *pVgId = taosHashIterGet(pIter);
-      if (vgId == 0) break;
+      SVgObj *pVgroup = mgmtGetVgroup(*pVgId);
-
-      SVgObj *pVgroup = mgmtGetVgroup(vgId);
       if (pVgroup == NULL) break;
-      
+
       SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg));
       pDrop->contLen = htonl(sizeof(SMDDropSTableMsg));
-      pDrop->vgId = htonl(vgId);
+      pDrop->vgId = htonl(pVgroup->vgId);
       pDrop->uid = htobe64(pStable->uid);
       mgmtExtractTableName(pStable->info.tableId, pDrop->tableId);
         
-      mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, vgId);
+      mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, pVgroup->vgId);
       SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
       SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
       dnodeSendMsgToDnode(&ipSet, &rpcMsg);
       mgmtDecVgroupRef(pVgroup);
     }
+
+    mgmtDropAllChildTablesInStable(pStable);
   } 
   
   SSdbOper oper = {
@@ -1243,59 +1240,58 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
 static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
   SCMSTableVgroupMsg *pInfo = pMsg->pCont;
   int32_t numOfTable = htonl(pInfo->numOfTables);
-  
+
-  char* name = (char*) pInfo + sizeof(struct SCMSTableVgroupMsg);
+  // reserve space
-  SCMSTableVgroupRspMsg *pRsp = NULL;
+  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo); 
-  
+  for (int32_t i = 0; i < numOfTable; ++i) {
-  // todo set the initial size to be 10, fix me
+    char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i;
-  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + (sizeof(SCMVgroupInfo) * 10 + sizeof(SVgroupsInfo))*numOfTable;
+    SSuperTableObj *pTable = mgmtGetSuperTable(stableName);
-  
+    if (pTable->vgHash != NULL) {
-  pRsp = rpcMallocCont(contLen);
+      contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo));
+    }
+    mgmtDecTableRef(pTable);
+  }
+
+  SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
     return;
   }
-  
+
   pRsp->numOfTables = htonl(numOfTable);
   char* msg = (char*) pRsp + sizeof(SCMSTableVgroupRspMsg);
-  
+
-  for(int32_t i = 0; i < numOfTable; ++i) {
+  for (int32_t i = 0; i < numOfTable; ++i) {
-    SSuperTableObj *pTable = mgmtGetSuperTable(name);
+    char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i;
-  
+    SSuperTableObj *pTable = mgmtGetSuperTable(stableName);
-    pMsg->pTable = (STableObj *)pTable;
+    SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg;
-    if (pMsg->pTable == NULL) {
+
-      mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
+    SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash);
-      return;
+    int32_t vgSize = 0;
-    }
+    while (taosHashIterNext(pIter)) {
-    
+      int32_t *pVgId = taosHashIterGet(pIter);
-    SVgroupsInfo* pVgroup = (SVgroupsInfo*) msg;
+      SVgObj * pVgroup = mgmtGetVgroup(*pVgId);
-    
+      if (pVgroup == NULL) continue;
-    int32_t vg = 0;
+
-    for (; vg < pTable->vgLen; ++vg) {
+      pVgroupInfo->vgroups[vgSize].vgId = htonl(pVgroup->vgId);
-      int32_t vgId = pTable->vgList[vg];
+      for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) {
-      if (vgId == 0) break;
+        SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode;
-    
-      SVgObj *vgItem = mgmtGetVgroup(vgId);
-      if (vgItem == NULL) break;
-  
-      pVgroup->vgroups[vg].vgId = htonl(vgId);
-      for (int32_t vn = 0; vn < vgItem->numOfVnodes; ++vn) {
-        SDnodeObj *pDnode = vgItem->vnodeGid[vn].pDnode;
         if (pDnode == NULL) break;
-  
+
-        strncpy(pVgroup->vgroups[vg].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
+        strncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
-        pVgroup->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort);
+        pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(tsDnodeShellPort);
-        
+
-        pVgroup->vgroups[vg].numOfIps++;
+        pVgroupInfo->vgroups[vgSize].numOfIps++;
       }
-    
+
-      mgmtDecVgroupRef(vgItem);
+      vgSize++;
+      mgmtDecVgroupRef(pVgroup);
     }
-  
+
-    pVgroup->numOfVgroups = htonl(vg);
+    pVgroupInfo->numOfVgroups = htonl(vgSize);
-    
+
     // one table is done, try the next table
-    msg += sizeof(SVgroupsInfo) + vg * sizeof(SCMVgroupInfo);
+    msg += sizeof(SVgroupsInfo) + vgSize * sizeof(SCMVgroupInfo);
   }
 
   SRpcMsg rpcRsp = {0};

src/mnode/src/mgmtUser.c

@@ -37,6 +37,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
 static void    mgmtProcessCreateUserMsg(SQueuedMsg *pMsg);
 static void    mgmtProcessAlterUserMsg(SQueuedMsg *pMsg);
 static void    mgmtProcessDropUserMsg(SQueuedMsg *pMsg);
+static void    mgmtProcessAuthMsg(SRpcMsg *rpcMsg);
 
 static int32_t mgmtUserActionDestroy(SSdbOper *pOper) {
   tfree(pOper->pObj);
@@ -140,7 +141,8 @@ int32_t mgmtInitUsers() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mgmtProcessDropUserMsg);
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_USER, mgmtGetUserMeta);
   mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mgmtRetrieveUsers);
-  
+  dnodeAddServerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mgmtProcessAuthMsg);
+   
   mTrace("table:%s, hash is created", tableDesc.tableName);
   return 0;
 }
@@ -529,3 +531,40 @@ void  mgmtDropAllUsers(SAcctObj *pAcct)  {
 
   mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);
 }
+
+int32_t mgmtRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  if (!sdbIsMaster()) {
+    *secret = 0;
+    mTrace("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_NOT_READY));
+    return TSDB_CODE_NOT_READY;
+  }
+
+  SUserObj *pUser = mgmtGetUser(user);
+  if (pUser == NULL) {
+    *secret = 0;
+    mError("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_INVALID_USER));
+    return TSDB_CODE_INVALID_USER;
+  } else {
+    *spi = 1;
+    *encrypt = 0;
+    *ckey = 0;
+
+    memcpy(secret, pUser->pass, TSDB_KEY_LEN);
+    mgmtDecUserRef(pUser);
+    mTrace("user:%s, auth info is returned", user);
+    return TSDB_CODE_SUCCESS;
+  }
+}
+
+static void mgmtProcessAuthMsg(SRpcMsg *rpcMsg) {
+  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  
+  SDMAuthMsg *pAuthMsg = rpcMsg->pCont;
+  SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp));
+  
+  rpcRsp.code = mgmtRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey);
+  rpcRsp.pCont = pAuthRsp;
+  rpcRsp.contLen = sizeof(SDMAuthRsp);
+  
+  rpcSendResponse(&rpcRsp);
+}

src/query/inc/qast.h

@@ -50,11 +50,11 @@ typedef struct tQueryInfo {
   SSchema       sch;      // schema of tags
   char*         q;
   __compar_fn_t compare;  // filter function
-  void*         param;    // STSchema,
+  void*         param;    // STSchema
 } tQueryInfo;
 
 typedef struct SExprTraverseSupp {
-  __result_filter_fn_t   fp;
+  __result_filter_fn_t   nodeFilterFn;
   __do_filter_suppl_fn_t setupInfoFn;
   void *                 pExtInfo;
 } SExprTraverseSupp;

src/query/inc/queryExecutor.h

@@ -110,6 +110,14 @@ typedef struct STableQueryInfo {  // todo merge with the STableQueryInfo struct
   SWindowResInfo windowResInfo;
 } STableQueryInfo;
 
+typedef struct SQueryCostSummary {
+} SQueryCostSummary;
+
+typedef struct SGroupItem {
+  STableId id;
+  STableQueryInfo* info;
+} SGroupItem;
+
 typedef struct SQuery {
   int16_t           numOfCols;
   int16_t           numOfTags;
@@ -131,17 +139,15 @@ typedef struct SQuery {
   SColumnInfo*      tagColList;
   int32_t           numOfFilterCols;
   int64_t*          defaultVal;
-  TSKEY             lastKey;
+//  TSKEY             lastKey;
   uint32_t          status;  // query status
   SResultRec        rec;
   int32_t           pos;
   SData**           sdata;
+  STableQueryInfo*  current;
   SSingleColumnFilterInfo* pFilterInfo;
 } SQuery;
 
-typedef struct SQueryCostSummary {
-} SQueryCostSummary;
-
 typedef struct SQueryRuntimeEnv {
   SResultInfo*       resultInfo;  // todo refactor to merge with SWindowResInfo
   SQuery*            pQuery;

src/query/inc/queryLog.h

@@ -22,20 +22,20 @@ extern "C" {
 
 #include "tlog.h"
 
-extern int32_t qdebugFlag;
+extern int32_t qDebugFlag;
 
 #define qTrace(...)                               \
-  if (qdebugFlag & DEBUG_TRACE) {                 \
+  if (qDebugFlag & DEBUG_TRACE) {                 \
-    taosPrintLog("DND QRY ", qdebugFlag, __VA_ARGS__); \
+    taosPrintLog("DND QRY ", qDebugFlag, __VA_ARGS__); \
   }
 
 #define qError(...)                                   \
-  if (qdebugFlag & DEBUG_ERROR) {                   \
+  if (qDebugFlag & DEBUG_ERROR) {                   \
-    taosPrintLog("ERROR QRY ", qdebugFlag, __VA_ARGS__); \
+    taosPrintLog("ERROR QRY ", qDebugFlag, __VA_ARGS__); \
   }
 #define qWarn(...)                                    \
-  if (qdebugFlag & DEBUG_WARN) {                    \
+  if (qDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  QRY ", qdebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN  QRY ", qDebugFlag, __VA_ARGS__); \
   }  
 
 #ifdef __cplusplus

src/query/inc/tsqlfunction.h

@@ -104,7 +104,7 @@ extern "C" {
 
 enum {
   MASTER_SCAN           = 0x0u,
-  SUPPLEMENTARY_SCAN    = 0x1u,
+  REVERSE_SCAN          = 0x1u,
   REPEAT_SCAN           = 0x2u,  //repeat scan belongs to the master scan
   FIRST_STAGE_MERGE     = 0x10u,
   SECONDARY_STAGE_MERGE = 0x20u,

src/query/src/qast.c

@@ -540,8 +540,11 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
 
 static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) {
   SSkipListIterator* iter = NULL;
+  
   SQueryCond cond = {0};
-  setQueryCond(pQueryInfo, &cond);
+  if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) {
+    //todo handle error
+  }
 
   if (cond.start != NULL) {
     iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
@@ -552,18 +555,18 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
   if (cond.start != NULL) {
     int32_t optr = cond.start->optr;
     
-    if (optr == TSDB_RELATION_EQUAL) {
+    if (optr == TSDB_RELATION_EQUAL) {   // equals
       while(tSkipListIterNext(iter)) {
         SSkipListNode* pNode = tSkipListIterGet(iter);
 
         int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v);
-        if (ret == 0) {
+        if (ret != 0) {
-          taosArrayPush(result, SL_GET_NODE_DATA(pNode));
-        } else {
           break;
         }
+        
+        taosArrayPush(result, SL_GET_NODE_DATA(pNode));
       }
-    } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) {
+    } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
       bool comp = true;
       int32_t ret = 0;
       
@@ -582,8 +585,33 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
           comp = false;
         }
       }
-    } else if (optr == TSDB_RELATION_NOT_EQUAL) {
+    } else if (optr == TSDB_RELATION_NOT_EQUAL) {   // not equal
-      assert(0);
+      bool comp = true;
+      
+      while(tSkipListIterNext(iter)) {
+        SSkipListNode* pNode = tSkipListIterGet(iter);
+        comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
+        if (comp) {
+          continue;
+        }
+        
+        taosArrayPush(result, SL_GET_NODE_DATA(pNode));
+      }
+      
+      tSkipListDestroyIter(iter);
+  
+      comp = true;
+      iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+      while(tSkipListIterNext(iter)) {
+        SSkipListNode* pNode = tSkipListIterGet(iter);
+        comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
+        if (comp) {
+          continue;
+        }
+  
+        taosArrayPush(result, SL_GET_NODE_DATA(pNode));
+      }
+  
     } else {
       assert(0);
     }
@@ -732,7 +760,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p
   assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE);
   param->setupInfoFn(pExpr, param->pExtInfo);
 
-  return param->fp(pItem, pExpr->_node.info);
+  return param->nodeFilterFn(pItem, pExpr->_node.info);
 }
 
 /**
@@ -773,35 +801,33 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki
   tSkipListDestroyIter(iter);
 }
 
-
+static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) {
-
-static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) {
   SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
 
   while (tSkipListIterNext(iter)) {
     bool addToResult = false;
 
     SSkipListNode *pNode = tSkipListIterGet(iter);
-    char* pTable = SL_GET_NODE_DATA(pNode);
+    char *         pData = SL_GET_NODE_DATA(pNode);
-    
+
-    //todo refactor:
+    // todo refactor:
-    char* name = (*(STable**) pTable)->name;
+    tstr *name = ((STableIndexElem *)pData)->pTable->name;
-//    char* name = NULL;
+    //    char* name = NULL;
-//    tsdbGetTableName(tsdb, pTable, &name);
+//        tsdbGetTableName(pQueryInfo->, pTable, &name);
 
     // todo speed up by using hash
     if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
       if (pQueryInfo->optr == TSDB_RELATION_IN) {
         addToResult = pQueryInfo->compare(name, pQueryInfo->q);
-      } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) {
+      } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) {
         addToResult = !pQueryInfo->compare(name, pQueryInfo->q);
       }
     } else {
-      // TODO: other columns
+      addToResult = filterFp(pNode, pQueryInfo);
     }
 
     if (addToResult) {
-      taosArrayPush(result, pTable);
+      taosArrayPush(res, pData);
     }
   }
 
@@ -825,7 +851,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
 
     param->setupInfoFn(pExpr, param->pExtInfo);
     if (pSkipList == NULL) {
-      tArrayTraverse(pExpr, param->fp, result);
+      tArrayTraverse(pExpr, param->nodeFilterFn, result);
       return;
     }
 
@@ -833,7 +859,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
     if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) {
       tQueryIndexColumn(pSkipList, pQueryInfo, result);
     } else {
-      tQueryIndexlessColumn(pSkipList, pQueryInfo, result);
+      tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn);
     }
 
     return;

src/query/src/qparserImpl.c

@@ -820,7 +820,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI
 
   pInfo->pDCLInfo->dbOpt = *pDB;
   pInfo->pDCLInfo->dbOpt.dbname = *pToken;
-  pInfo->pDCLInfo->dbOpt.ignoreExists = (pIgExists->z != NULL);
+  pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS.   {X.n = 1;}
 }
 
 void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) {

src/query/src/queryFilterFunc.c

@@ -183,33 +183,33 @@ bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
 
 bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
   // query condition string is greater than the max length of string, not qualified data
-  if (pFilter->filterInfo.len > pFilter->bytes) {
+  if (pFilter->filterInfo.len != varDataLen(minval)) {
     return false;
   }
 
-  return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0;
+  return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) == 0;
 }
 
 bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) {
   // query condition string is greater than the max length of string, not qualified data
-  if (pFilter->filterInfo.len > pFilter->bytes) {
+  if (pFilter->filterInfo.len != varDataLen(minval)) {
     return false;
   }
 
-  return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0;
+  return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) == 0;
 }
 
 ////////////////////////////////////////////////////////////////
 bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
   SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
 
-  return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH;
+  return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH;
 }
 
 bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) {
   SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
-
+  
-  return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH;
+  return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH;
 }
 
 ////////////////////////////////////////////////////////////////
@@ -270,11 +270,11 @@ bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) {
 }
 
 bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) {
-  if (pFilter->filterInfo.len > pFilter->bytes) {
+  if (pFilter->filterInfo.len != varDataLen(minval)) {
     return true;
   }
 
-  return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0;
+  return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) != 0;
 }
 
 bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
@@ -282,7 +282,7 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
     return true;
   }
 
-  return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0;
+  return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0;
 }
 
 ////////////////////////////////////////////////////////////////

src/query/src/tvariant.c

@@ -389,6 +389,7 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   return 0;
 }
 
+// todo handle the error
 static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   char tmpBuf[40] = {0};
   
@@ -424,7 +425,12 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
     
     pVariant->wpz = (wchar_t *)tmp;
   } else {
-    taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
+    size_t output = -1;
+    taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
+    
+    if (pDestSize != NULL) {
+      *pDestSize = output;
+    }
   }
   
   return 0;
@@ -779,7 +785,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) {
     }
     case TSDB_DATA_TYPE_NCHAR: {
       if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
-        *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL;
+        *(uint32_t *) payload = TSDB_DATA_NCHAR_NULL;
       } else {
         if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
           toNchar(pVariant, &payload, &pVariant->nLen);

src/query/tests/patternMatchTest.cpp

@@ -76,4 +76,8 @@ TEST(testCase, patternMatchTest) {
   str = "carzero";
   ret = patternMatch("%o", str, strlen(str), &info);
   EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+  
+  str = "19";
+  ret = patternMatch("%9", str, 2, &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
 }

src/rpc/src/rpcMain.c

@@ -55,9 +55,8 @@ typedef struct {
   char     secret[TSDB_KEY_LEN]; // secret for the link
   char     ckey[TSDB_KEY_LEN];   // ciphering key 
 
-  void   (*cfp)(SRpcMsg *);
+  void   (*cfp)(SRpcMsg *, SRpcIpSet *);
   int    (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); 
-  void   (*ufp)(void *ahandle, SRpcIpSet *pIpSet);
 
   void     *idPool;   // handle to ID pool
   void     *tmrCtrl;  // handle to timer
@@ -222,7 +221,6 @@ void *rpcOpen(const SRpcInit *pInit) {
   if (pInit->secret) strcpy(pRpc->secret, pInit->secret);
   if (pInit->ckey) strcpy(pRpc->ckey, pInit->ckey);
   pRpc->spi = pInit->spi;
-  pRpc->ufp = pInit->ufp;
   pRpc->cfp = pInit->cfp;
   pRpc->afp = pInit->afp;
 
@@ -793,7 +791,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
 
   pConn->chandle = pRecv->chandle;
   pConn->peerIp = pRecv->ip; 
-  if (pConn->peerPort == 0) pConn->peerPort = pRecv->port;
+  pConn->peerPort = pRecv->port;
   if (pHead->port) pConn->peerPort = htons(pHead->port); 
 
   terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen);
@@ -900,10 +898,11 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
     memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg));
   } else {
     // for asynchronous API 
-    if (pRpc->ufp && (pContext->ipSet.inUse != pContext->oldInUse || pContext->redirect)) 
+    SRpcIpSet *pIpSet = NULL;
-      (*pRpc->ufp)(pContext->ahandle, &pContext->ipSet);  // notify the update of ipSet
+    if (pContext->ipSet.inUse != pContext->oldInUse || pContext->redirect) 
+      pIpSet = &pContext->ipSet;  
 
-    (*pRpc->cfp)(pMsg);  
+    (*pRpc->cfp)(pMsg, pIpSet);  
   }
 
   // free the request message
@@ -924,7 +923,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
   if ( rpcIsReq(pHead->msgType) ) {
     rpcMsg.handle = pConn;
     taosTmrReset(rpcProcessProgressTimer, tsRpcTimer/2, pConn, pRpc->tmrCtrl, &pConn->pTimer);
-    (*(pRpc->cfp))(&rpcMsg);
+    (*(pRpc->cfp))(&rpcMsg, NULL);
   } else {
     // it's a response
     SRpcReqContext *pContext = pConn->pContext;
@@ -1094,8 +1093,8 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
   } else {
     if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
     if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
-      tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
+      tTrace( "%s %p, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
-          pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, 
+          pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, 
           htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 

src/rpc/test/rclient.c

@@ -31,22 +31,16 @@ typedef struct {
   void     *pRpc;
 } SInfo;
 
-static void processResponse(SRpcMsg *pMsg) {
+static void processResponse(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
   SInfo *pInfo = (SInfo *)pMsg->handle;
   tTrace("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code);
 
+  if (pIpSet) pInfo->ipSet = *pIpSet;
+
   rpcFreeCont(pMsg->pCont);
-
   sem_post(&pInfo->rspSem); 
 }
 
-static void processUpdateIpSet(void *handle, SRpcIpSet *pIpSet) {
-  SInfo *pInfo = (SInfo *)handle;
-
-  tTrace("thread:%d, ip set is changed, index:%d", pInfo->index, pIpSet->inUse);
-  pInfo->ipSet = *pIpSet;
-}
-
 static int tcount = 0;
 
 static void *sendRequest(void *param) {
@@ -99,7 +93,6 @@ int main(int argc, char *argv[]) {
   rpcInit.label        = "APP";
   rpcInit.numOfThreads = 1;
   rpcInit.cfp          = processResponse;
-  rpcInit.ufp          = processUpdateIpSet;
   rpcInit.sessions     = 100;
   rpcInit.idleTime     = tsShellActivityTimer*1000;
   rpcInit.user         = "michael";

src/rpc/test/rsclient.c

@@ -32,12 +32,6 @@ typedef struct {
   void     *pRpc;
 } SInfo;
 
-static void processUpdateIpSet(void *handle, SRpcIpSet *pIpSet) {
-  SInfo *pInfo = (SInfo *)handle;
-
-  tTrace("thread:%d, ip set is changed, index:%d", pInfo->index, pIpSet->inUse);
-  pInfo->ipSet = *pIpSet;
-}
 
 static int tcount = 0;
 static int terror = 0;
@@ -100,8 +94,6 @@ int main(int argc, char *argv[]) {
   rpcInit.localPort    = 0;
   rpcInit.label        = "APP";
   rpcInit.numOfThreads = 1;
-  // rpcInit.cfp          = processResponse;
-  rpcInit.ufp          = processUpdateIpSet;
   rpcInit.sessions     = 100;
   rpcInit.idleTime     = tsShellActivityTimer*1000;
   rpcInit.user         = "michael";

src/rpc/test/rserver.c

@@ -113,7 +113,7 @@ int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char
   return ret;
 }
 
-void processRequestMsg(SRpcMsg *pMsg) {
+void processRequestMsg(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
   SRpcMsg *pTemp;
 
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));

src/tsdb/inc/tsdbMain.h

@@ -81,7 +81,6 @@ typedef struct {
 // ---------- TSDB TABLE DEFINITION
 typedef struct STable {
   int8_t         type;
-  char *         name;
   STableId       tableId;
   int64_t        superUid;  // Super table UID
   int32_t        sversion;
@@ -96,9 +95,10 @@ typedef struct STable {
   TSKEY          lastKey;        // lastkey inserted in this table, initialized as 0, TODO: make a structure
   struct STable *next;           // TODO: remove the next
   struct STable *prev;
+  tstr *         name;           // NOTE: there a flexible string here
 } STable;
 
-#define TSDB_GET_TABLE_LAST_KEY(pTable) ((pTable)->lastKey)
+#define TSDB_GET_TABLE_LAST_KEY(tb) ((tb)->lastKey)
 
 void *  tsdbEncodeTable(STable *pTable, int *contLen);
 STable *tsdbDecodeTable(void *cont, int contLen);
@@ -121,6 +121,12 @@ typedef struct {
   int        maxCols;
 } STsdbMeta;
 
+// element put in skiplist for each table
+typedef struct STableIndexElem {
+  STsdbMeta* pMeta;
+  STable*    pTable;
+} STableIndexElem;
+
 STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables);
 int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
 STSchema * tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable);
@@ -145,12 +151,10 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable);
 
 STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo);
 
-int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg);
-int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId);
 STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);
 // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable);
 STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid);
-char *  getTupleKey(const void *data);
+char *getTSTupleKey(const void * data);
 
 typedef struct {
   int  blockId;
@@ -490,9 +494,10 @@ int tsdbWriteCompInfo(SRWHelper *pHelper);
 int tsdbWriteCompIdx(SRWHelper *pHelper);
 
 // --------- Other functions need to further organize
-void tsdbFitRetention(STsdbRepo *pRepo);
+void    tsdbFitRetention(STsdbRepo *pRepo);
-int  tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
+int     tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
-void tsdbAdjustCacheBlocks(STsdbCache *pCache);
+void    tsdbAdjustCacheBlocks(STsdbCache *pCache);
+int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
 
 #ifdef __cplusplus
 }

src/tsdb/src/tsdbCache.c

@@ -164,7 +164,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
   }
 
   tsdbUnLockRepo((TsdbRepoT *)pRepo);
-  tsdbTrace("tsdbId %d: tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);
+  tsdbTrace("vgId: %d tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks);
   return 0;
 }
 

src/tsdb/src/tsdbMain.c

@@ -7,6 +7,7 @@
 #include "tscompression.h"
 #include "tchecksum.h"
 #include "ttime.h"
+#include <sys/stat.h>
 
 int tsdbDebugFlag = 135;
 
@@ -133,6 +134,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */)
  */
 int32_t tsdbDropRepo(TsdbRepoT *repo) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
+  int id = pRepo->config.tsdbId;
 
   pRepo->state = TSDB_REPO_STATE_CLOSED;
 
@@ -148,6 +150,8 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) {
   free(pRepo->rootDir);
   free(pRepo);
 
+  tsdbTrace("vgId: %d tsdb repository is dropped!", id);
+
   return 0;
 }
 
@@ -238,6 +242,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
 
   pRepo->state = TSDB_REPO_STATE_ACTIVE;
 
+  tsdbTrace("vgId: %d open tsdb repository successfully!", pRepo->config.tsdbId);
   return (TsdbRepoT *)pRepo;
 }
 
@@ -256,6 +261,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
 int32_t tsdbCloseRepo(TsdbRepoT *repo) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   if (pRepo == NULL) return 0;
+  int id = pRepo->config.tsdbId;
 
   pRepo->state = TSDB_REPO_STATE_CLOSED;
   tsdbLockRepo(repo);
@@ -289,6 +295,8 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) {
   tfree(pRepo->rootDir);
   tfree(pRepo);
 
+  tsdbTrace("vgId: %d repository is closed!", id);
+
   return 0;
 }
 
@@ -349,6 +357,7 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
   pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo);
+  tsdbTrace("vgId: %d start to commit!", pRepo->config.tsdbId);
 
   return 0;
 }
@@ -376,11 +385,6 @@ STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo) {
   return NULL;
 }
 
-int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
-  return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg);
-}
-
 int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
   // TODO
   return 0;
@@ -395,13 +399,6 @@ TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) {
   return TSDB_GET_TABLE_LAST_KEY(pTable);
 }
 
-int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
-  if (repo == NULL) return -1;
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
-
-  return tsdbDropTableImpl(pRepo->tsdbMeta, tableId);
-}
-
 STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) {
   // TODO
   return NULL;
@@ -756,12 +753,12 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
   if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1;
 
   if (mkdir(dirName, 0755) < 0) {
-    tsdbError("id %d: failed to create repository directory! reason %s", pRepo->config.tsdbId, strerror(errno));
+    tsdbError("vgId: %d failed to create repository directory! reason %s", pRepo->config.tsdbId, strerror(errno));
     return -1;
   }
 
-  tsdbError(
+  tsdbTrace(
-      "id %d: set up tsdb environment succeed! cacheBlockSize %d, totalBlocks %d, maxTables %d, daysPerFile %d, keep "
+      "vgId: %d set up tsdb environment succeed! cacheBlockSize %d, totalBlocks %d, maxTables %d, daysPerFile %d, keep "
       "%d, minRowsPerFileBlock %d, maxRowsPerFileBlock %d, precision %d, compression%d",
       pRepo->config.tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep,
       pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression);
@@ -808,7 +805,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   if (pTable->mem == NULL) {
     pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable));
     if (pTable->mem == NULL) return -1;
-    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey);
+    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey);
     pTable->mem->keyFirst = INT64_MAX;
     pTable->mem->keyLast = 0;
   }
@@ -831,7 +828,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   if (pTable->mem == NULL) {
     pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable));
     if (pTable->mem == NULL) return -1;
-    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey);
+    pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey);
     pTable->mem->keyFirst = INT64_MAX;
     pTable->mem->keyLast = 0;
   }
@@ -842,6 +839,9 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   
   pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData);
 
+  tsdbTrace("vgId: %d, tid: %d, uid: " PRId64 "a row is inserted to table! key" PRId64,
+            pRepo->config.tsdbId, pTable->tableId.tid, pTable->tableId.uid, dataRowKey(row));
+
   return 0;
 }
 
@@ -1104,11 +1104,14 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK
 }
 
 static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) {
+  int8_t oldCompRession = pRepo->config.compression;
   pRepo->config.compression = compression;
+  tsdbTrace("vgId: %d tsdb compression is changed from %d to %d", oldCompRession, compression);
 }
 
 static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
   STsdbCfg *pCfg = &pRepo->config;
+  int oldKeep = pCfg->keep;
 
   int maxFiles = keep / pCfg->maxTables + 3;
   if (pRepo->config.keep > keep) {
@@ -1120,8 +1123,57 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
     }
     pRepo->tsdbFileH->maxFGroups = maxFiles;
   }
+  tsdbTrace("vgId: %d keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep);
 }
 
 static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
   // TODO
+  int oldMaxTables = pRepo->config.maxTables;
+  tsdbTrace("vgId: %d tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
+}
+
+uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size) {
+  // TODO: need to refactor this function
+
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  // STsdbMeta *pMeta = pRepo->tsdbMeta;
+  STsdbFileH *pFileH = pRepo->tsdbFileH;
+  uint32_t   magic = 0;
+  char       fname[256] = "\0";
+
+  struct stat fState;
+  char *spath = strdup(pRepo->rootDir);
+  char *prefixDir = dirname(spath);
+
+  if (name[0] == 0) {
+    // Map index to the file name
+    int fid = (*index) / 3;
+
+    if (fid > pFileH->numOfFGroups) {
+      // return meta data file
+      if ((*index) % 3 > 0) { // it is finished
+        tfree(spath);
+        return 0;
+      } else {
+        tsdbGetMetaFileName(pRepo->rootDir, fname);
+      }
+    } else {
+      // return data file name
+      strcpy(fname, pFileH->fGroup[fid].files[(*index) % 3].fname);
+    }
+    strcpy(name, fname + strlen(spath));
+  } else {
+    // Name is provided, need to get the file info
+    sprintf(fname, "%s/%s", prefixDir, name);
+  }
+
+  if (stat(fname, &fState) < 0) {
+    tfree(spath);
+    return 0;
+  }
+
+  *size = fState.st_size;
+  magic = *size;
+
+  return magic;
 }

src/tsdb/src/tsdbMeta.c

@@ -1,6 +1,4 @@
 #include <stdlib.h>
-
-// #include "taosdef.h"
 #include "tskiplist.h"
 #include "tsdb.h"
 #include "taosdef.h"
@@ -10,6 +8,8 @@
 #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here
 #define TSDB_META_FILE_NAME "META"
 
+const int32_t DEFAULT_TAG_INDEX_COLUMN = 0;   // skip list built based on the first column of tags
+
 static int     tsdbFreeTable(STable *pTable);
 static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
 static int     tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx);
@@ -39,11 +39,12 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) {
 
   void *ptr = ret;
   T_APPEND_MEMBER(ptr, pTable, STable, type);
-  // Encode name
+  // Encode name, todo refactor
-  *(int *)ptr = strlen(pTable->name);
+  *(int *)ptr = varDataLen(pTable->name);
   ptr = (char *)ptr + sizeof(int);
-  memcpy(ptr, pTable->name, strlen(pTable->name));
+  memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name));
-  ptr = (char *)ptr + strlen(pTable->name);
+  ptr = (char *)ptr + varDataLen(pTable->name);
+  
   T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid);
   T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid);
   T_APPEND_MEMBER(ptr, pTable, STable, superUid);
@@ -79,9 +80,12 @@ STable *tsdbDecodeTable(void *cont, int contLen) {
   T_READ_MEMBER(ptr, int8_t, pTable->type);
   int len = *(int *)ptr;
   ptr = (char *)ptr + sizeof(int);
-  pTable->name = calloc(1, len + 1);
+  pTable->name = calloc(1, len + VARSTR_HEADER_SIZE);
   if (pTable->name == NULL) return NULL;
-  memcpy(pTable->name, ptr, len);
+  
+  varDataSetLen(pTable->name, len);
+  memcpy(pTable->name->data, ptr, len);
+  
   ptr = (char *)ptr + len;
   T_READ_MEMBER(ptr, int64_t, pTable->tableId.uid);
   T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid);
@@ -105,8 +109,13 @@ void tsdbFreeEncode(void *cont) {
 }
 
 static char* getTagIndexKey(const void* pData) {
-  STable* table = *(STable**) pData;
+  STableIndexElem* elem = (STableIndexElem*) pData;
-  return getTupleKey(table->tagVal);
+  
+  SDataRow row = elem->pTable->tagVal;
+  STSchema* pSchema = tsdbGetTableTagSchema(elem->pMeta, elem->pTable);
+  STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
+  
+  return tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
 }
 
 int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
@@ -230,31 +239,23 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t
   STable* pTable = tsdbGetTableByUid(pMeta, id->uid);
   
   STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable);
-  
   STColumn* pCol = NULL;
-  int32_t offset = 0;
+  
+  // todo binary search
   for(int32_t col = 0; col < schemaNCols(pSchema); ++col) {
     STColumn* p = schemaColAt(pSchema, col);
     if (p->colId == colId) {
       pCol = p;
       break;
     }
-  
-    if (p->type == TSDB_DATA_TYPE_BINARY || p->type == TSDB_DATA_TYPE_NCHAR) {
-      offset += sizeof(int32_t);
-    } else {
-      offset += p->bytes;
-    }
   }
   
   if (pCol == NULL) {
     return -1;  // No matched tags. Maybe the modification of tags has not been done yet.
   }
   
-  assert(pCol != NULL);
-  
   SDataRow row = (SDataRow)pTable->tagVal;
-  char* d = tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + offset);
+  char* d = tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
   
   *val = d;
   *type  = pCol->type;
@@ -263,19 +264,29 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name) {
+char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes) {
   STsdbMeta* pMeta = tsdbGetMeta(repo);
   STable* pTable = tsdbGetTableByUid(pMeta, id->uid);
   
-  *name = strndup(pTable->name, TSDB_TABLE_NAME_LEN);
+  if (pTable == NULL) {
-  if (*name == NULL) {
+    if (bytes != NULL) {
-    return -1;
+      *bytes = 0;
+    }
+    
+    return NULL;
   } else {
-    return 0;
+    if (bytes != NULL) {
+      *bytes = varDataLen(pTable->name);
+    }
+    
+    return (char*) pTable->name;
   }
 }
 
-int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
+int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
+
   if (tsdbCheckTableCfg(pCfg) < 0) return -1;
 
   STable *super = NULL;
@@ -296,7 +307,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
       super->schema = tdDupSchema(pCfg->schema);
       super->tagSchema = tdDupSchema(pCfg->tagSchema);
       super->tagVal = NULL;
-      super->name = strdup(pCfg->sname);
+      
+      // todo refactor extract method
+      size_t size = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN);
+      super->name = malloc(size + VARSTR_HEADER_SIZE);
+      STR_WITH_SIZE_TO_VARSTR(super->name, pCfg->sname, size);
 
       // index the first tag column
       STColumn* pColSchema = schemaColAt(super->tagSchema, 0);
@@ -322,7 +337,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
   }
 
   table->tableId = pCfg->tableId;
-  table->name = strdup(pCfg->name);
+  
+  size_t size = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN);
+  table->name = malloc(size + VARSTR_HEADER_SIZE);
+  STR_WITH_SIZE_TO_VARSTR(table->name, pCfg->name, size);
+  
   table->lastKey = 0;
   if (IS_CREATE_STABLE(pCfg)) { // TSDB_CHILD_TABLE
     table->type = TSDB_CHILD_TABLE;
@@ -335,8 +354,14 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
   }
 
   // Register to meta
-  if (newSuper) tsdbAddTableToMeta(pMeta, super, true);
+  if (newSuper) {
+    tsdbAddTableToMeta(pMeta, super, true);
+    tsdbTrace("vgId: %d super table is created! uid " PRId64, pRepo->config.tsdbId,
+              super->tableId.uid);
+  }
   tsdbAddTableToMeta(pMeta, table, true);
+  tsdbTrace("vgId: %d table is created! tid %d, uid " PRId64, pRepo->config.tsdbId, table->tableId.tid,
+            table->tableId.uid);
 
   // Write to meta file
   int bufLen = 0;
@@ -369,13 +394,24 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId) {
   return pTable;
 }
 
-int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) {
+// int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) {
+int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  if (pRepo == NULL) return -1;
+
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
   if (pMeta == NULL) return -1;
 
   STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid);
-  if (pTable == NULL) return -1;
+  if (pTable == NULL) {
+    tsdbError("vgId %d: failed to drop table since table not exists! tid %d, uid " PRId64, pRepo->config.tsdbId,
+              tableId.tid, tableId.uid);
+    return -1;
+  }
 
+  tsdbTrace("vgId: %d table is dropped! tid %d, uid " PRId64, pRepo->config.tsdbId, tableId.tid, tableId.uid);
   if (tsdbRemoveTableFromMeta(pMeta, pTable) < 0) return -1;
+
   return 0;
 
 }
@@ -467,7 +503,9 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
   if (pTable->type == TSDB_SUPER_TABLE) {
     SSkipListIterator  *pIter = tSkipListCreateIter(pTable->pIndex);
     while (tSkipListIterNext(pIter)) {
-      STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+      STableIndexElem *pEle = (STableIndexElem *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+      STable *tTable = pEle->pTable;
+
       ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE);
 
       pMeta->tables[tTable->tableId.tid] = NULL;
@@ -513,11 +551,14 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) {
   
   // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
   // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
-  SSkipListNode* pNode = calloc(1, headSize + POINTER_BYTES);
+  SSkipListNode* pNode = calloc(1, headSize + sizeof(STableIndexElem));
   pNode->level = level;
   
   SSkipList* list = pSTable->pIndex;
-  memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES);
+  STableIndexElem* elem = (STableIndexElem*) (SL_GET_NODE_DATA(pNode));
+  
+  elem->pTable = pTable;
+  elem->pMeta = pMeta;
   
   tSkipListPut(list, pNode);
   return 0;
@@ -529,7 +570,10 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
   STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid);
   assert(pSTable != NULL);
   
-  char* key = dataRowTuple(pTable->tagVal); // key
+  STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable);
+  STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
+  
+  char* key = tdGetRowDataOfCol(pTable->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
   bool ret = tSkipListRemove(pSTable->pIndex, key);
   
   assert(ret);
@@ -539,7 +583,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
 static int tsdbEstimateTableEncodeSize(STable *pTable) {
   int size = 0;
   size += T_MEMBER_SIZE(STable, type);
-  size += sizeof(int) + strlen(pTable->name);
+  size += sizeof(int) + varDataLen(pTable->name);
   size += T_MEMBER_SIZE(STable, tableId);
   size += T_MEMBER_SIZE(STable, superUid);
   size += T_MEMBER_SIZE(STable, sversion);
@@ -556,8 +600,7 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) {
   return size;
 }
 
-char *getTupleKey(const void * data) {
+char *getTSTupleKey(const void * data) {
   SDataRow row = (SDataRow)data;
-
   return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE);
 }

src/tsdb/src/tsdbMetaFile.c

@@ -28,7 +28,7 @@ typedef struct {
   int64_t uid;
 } SRecordInfo;
 
-static int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
+// static int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
 // static int32_t tsdbCheckMetaHeader(int fd);
 static int32_t tsdbWriteMetaHeader(int fd);
 static int     tsdbCreateMetaFile(char *fname);
@@ -180,7 +180,7 @@ void tsdbCloseMetaFile(SMetaFile *mfh) {
   tfree(mfh);
 }
 
-static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) {
+int32_t tsdbGetMetaFileName(char *rootDir, char *fname) {
   if (rootDir == NULL) return -1;
   sprintf(fname, "%s/%s", rootDir, TSDB_META_FILE_NAME);
   return 0;

src/tsdb/src/tsdbRead.c

@@ -620,9 +620,13 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
         if (pCol->info.type != TSDB_DATA_TYPE_BINARY && pCol->info.type != TSDB_DATA_TYPE_NCHAR) {
           memmove(pCol->pData, src->pData + bytes * start, bytes * pQueryHandle->realNumOfRows);
         } else { // handle the var-string
+          char* dst = pCol->pData;
+  
+          // todo refactor, only copy one-by-one
           for(int32_t k = start; k < pQueryHandle->realNumOfRows + start; ++k) {
             char* p = tdGetColDataOfRow(src, k);
-            memcpy(pCol->pData + k * bytes, p, varDataTLen(p));  // todo refactor
+            memcpy(dst, p, varDataTLen(p));
+            dst += bytes;
           }
         }
         
@@ -1076,17 +1080,13 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY
       }
       
       assert(offset != -1); // todo handle error
+      void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
       
       if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
-        void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
         memcpy(pData, value, varDataTLen(value));
-  
-        offset += sizeof(int32_t);
       } else {
-        memcpy(pData, dataRowTuple(row) + offset, pColInfo->info.bytes);
+        memcpy(pData, value, pColInfo->info.bytes);
-        offset += pColInfo->info.bytes;
       }
-      
     }
 
     numOfRows++;
@@ -1225,8 +1225,8 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
   while (tSkipListIterNext(iter)) {
     SSkipListNode* pNode = tSkipListIterGet(iter);
     
-    STable* t = *(STable**)SL_GET_NODE_DATA(pNode);
+    STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*) pNode));
-    taosArrayPush(list, &t->tableId);
+    taosArrayPush(list, &elem->pTable->tableId);
   }
   
   tSkipListDestroyIter(iter);
@@ -1235,6 +1235,7 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
 
 /**
  * convert the result pointer to table id instead of table object pointer
+ * todo remove it by using callback function to change the final result in-time.
  * @param pRes
  */
 static void convertQueryResult(SArray* pRes, SArray* pTableList) {
@@ -1244,8 +1245,8 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) {
 
   size_t size = taosArrayGetSize(pTableList);
   for (int32_t i = 0; i < size; ++i) {  // todo speedup  by using reserve space.
-    STable* pTable = taosArrayGetP(pTableList, i);
+    STableIndexElem* elem = taosArrayGet(pTableList, i);
-    taosArrayPush(pRes, &pTable->tableId);
+    taosArrayPush(pRes, &elem->pTable->tableId);
   }
 }
 
@@ -1309,7 +1310,12 @@ void filterPrepare(void* expr, void* param) {
     pInfo->q = (char*) pCond->arr;
   } else {
     pInfo->q = calloc(1, pSchema->bytes);
-    tVariantDump(pCond, pInfo->q, pSchema->type);
+    if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
+      tVariantDump(pCond, varDataVal(pInfo->q), pSchema->type);
+      varDataSetLen(pInfo->q, pCond->nLen);  // the length may be changed after dump, so assign its value after dump
+    } else {
+      tVariantDump(pCond, pInfo->q, pSchema->type);
+    }
   }
 }
 
@@ -1341,16 +1347,16 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
     int32_t bytes = 0;
     
     if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
-      f1 = pTable1->name;
+      f1 = (char*) pTable1->name;
-      f2 = pTable2->name;
+      f2 = (char*) pTable2->name;
       type = TSDB_DATA_TYPE_BINARY;
-      bytes = TSDB_TABLE_NAME_LEN;
+      bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
     } else {
-      f1 = dataRowTuple(pTable1->tagVal);
+      STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex);
-      f2 = dataRowTuple(pTable2->tagVal);
+      bytes = pCol->bytes;
-
+  
-      type = schemaColAt(pTableGroupSupp->pTagSchema, colIndex)->type;
+      f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
-      bytes = schemaColAt(pTableGroupSupp->pTagSchema, colIndex)->bytes;
+      f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
     }
     
     int32_t ret = doCompare(f1, f2, type, bytes);
@@ -1426,26 +1432,22 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
   return pTableGroup;
 }
 
-bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
+bool indexedNodeFilterFp(const void* pNode, void* param) {
   tQueryInfo* pInfo = (tQueryInfo*) param;
-
+  
-  STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
+  STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
 
   char*  val = NULL;
   int8_t type = pInfo->sch.type;
 
   if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
-    val = pTable->name;
+    val = (char*) elem->pTable->name;
     type = TSDB_DATA_TYPE_BINARY;
   } else {
     STSchema* pTSchema = (STSchema*) pInfo->param; // todo table schema is identical to stable schema??
     
     int32_t offset = pTSchema->columns[pInfo->colIndex].offset;
-    if (pInfo->sch.type == TSDB_DATA_TYPE_BINARY || pInfo->sch.type == TSDB_DATA_TYPE_NCHAR) {
+    val = tdGetRowDataOfCol(elem->pTable->tagVal, pInfo->sch.type, TD_DATA_ROW_HEAD_SIZE + offset);
-      val = tdGetRowDataOfCol(pTable->tagVal, pInfo->sch.type, TD_DATA_ROW_HEAD_SIZE + offset);
-    } else {
-      val = dataRowTuple(pTable->tagVal) + offset;
-    }
   }
 
   int32_t ret = 0;
@@ -1456,8 +1458,6 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
       ret = pInfo->compare(val, pInfo->q);
     }
   } else {
-//    tVariant t = {0};
-//    tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type);
     ret = pInfo->compare(val, pInfo->q);
   }
 
@@ -1497,12 +1497,12 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
 static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) {
   // query according to the expression tree
   SExprTraverseSupp supp = {
-      .fp = (__result_filter_fn_t) tSkipListNodeFilterCallback,
+      .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp,
       .setupInfoFn = filterPrepare,
       .pExtInfo = pSTable->tagSchema,
       };
 
-  SArray* pTableList = taosArrayInit(8, POINTER_BYTES);
+  SArray* pTableList = taosArrayInit(8, sizeof(STableIndexElem));
 
   tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp);
   tExprTreeDestroy(&pExpr, destroyHelper);

src/util/src/talgo.c

@@ -144,11 +144,11 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const
   }
   
   if (leftPartEnd > start) {
-    tqsortImpl(src, size, start, leftPartEnd, param, comparFn, buf);
+    tqsortImpl(src, start, leftPartEnd, size, param, comparFn, buf);
   }
   
   if (rightPartStart < end) {
-    tqsortImpl(src, size, rightPartStart, end, param, comparFn, buf);
+    tqsortImpl(src, rightPartStart, end, size, param, comparFn, buf);
   }
 }
 

src/util/src/tcompare.c

@@ -51,42 +51,54 @@ int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) {
   }
 }
 
+int32_t compareFloatVal(const void *pLeft, const void *pRight) {
+  float ret = GET_FLOAT_VAL(pLeft) - GET_FLOAT_VAL(pRight);
+  if (fabs(ret) < FLT_EPSILON) {
+    return 0;
+  } else {
+    return ret > 0? 1 : -1;
+  }
+}
+
 int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
   double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight);
   if (fabs(ret) < FLT_EPSILON) {
     return 0;
   } else {
-    return ret > 0 ? 1 : -1;
+    return ret > 0? 1 : -1;
   }
 }
 
-int32_t compareStrVal(const void *pLeft, const void *pRight) {
+int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
-  return (int32_t)strcmp(pLeft, pRight);
+  int32_t len1 = varDataLen(pLeft);
+  int32_t len2 = varDataLen(pRight);
+  
+  if (len1 != len2) {
+    return len1 > len2? 1:-1;
+  } else {
+    int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
+    if (ret == 0) {
+      return 0;
+    } else {
+      return ret > 0 ? 1:-1;
+    }
+  }
 }
 
-int32_t compareWStrVal(const void *pLeft, const void *pRight) {
+int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
-  //  SSkipListKey *pL = (SSkipListKey *)pLeft;
+  int32_t len1 = varDataLen(pLeft);
-  //  SSkipListKey *pR = (SSkipListKey *)pRight;
+  int32_t len2 = varDataLen(pRight);
-  //
+  
-  //  if (pL->nLen == 0 && pR->nLen == 0) {
+  if (len1 != len2) {
-  //    return 0;
+    return len1 > len2? 1:-1;
-  //  }
+  } else {
-  //
+    int32_t ret = wcsncmp(varDataVal(pLeft), varDataVal(pRight), len1);
-  //  // handle only one-side bound compare situation, there is only lower bound or only upper bound
+    if (ret == 0) {
-  //  if (pL->nLen == -1) {
+      return 0;
-  //    return 1;  // no lower bound, lower bound is minimum, always return -1;
+    } else {
-  //  } else if (pR->nLen == -1) {
+      return ret > 0 ? 1 : -1;
-  //    return -1;  // no upper bound, upper bound is maximum situation, always return 1;
+    }
-  //  }
+  }
-  //
-  //  int32_t ret = wcscmp(((SSkipListKey *)pLeft)->wpz, ((SSkipListKey *)pRight)->wpz);
-  //
-  //  if (ret == 0) {
-  //    return 0;
-  //  } else {
-  //    return ret > 0 ? 1 : -1;
-  //  }
-  return 0;
 }
 
 /*
@@ -124,7 +136,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat
         size_t n = strcspn(str, next);
         str += n;
         
-        if (str[0] == 0 || (n >= size - 1)) {
+        if (str[0] == 0 || (n >= size)) {
           break;
         }
         
@@ -172,10 +184,10 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
       
       wchar_t accept[3] = {towupper(c), towlower(c), 0};
       while (1) {
-        size_t n = wcsspn(str, accept);
+        size_t n = wcscspn(str, accept);
         
         str += n;
-        if (str[0] == 0 || (n >= size - 1)) {
+        if (str[0] == 0 || (n >= size)) {
           break;
         }
         
@@ -254,7 +266,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
     }
 
     case TSDB_DATA_TYPE_FLOAT: {
-      comparFn = compareDoubleVal; break;
+      comparFn = compareFloatVal; break;
     }
     
     case TSDB_DATA_TYPE_DOUBLE: {
@@ -267,7 +279,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = compareFindStrInArray;
       } else { /* normal relational comparFn */
-        comparFn = compareStrVal;
+        comparFn = compareLenPrefixedStr;
       }
     
       break;
@@ -277,7 +289,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
       if (optr == TSDB_RELATION_LIKE) {
         comparFn = compareWStrPatternComp;
       } else {
-        comparFn = compareWStrVal;
+        comparFn = compareLenPrefixedWStr;
       }
     
       break;
@@ -296,6 +308,7 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) {
   
   switch (keyType) {
     case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_BOOL:
       comparFn = compareInt8Val;
       break;
     case TSDB_DATA_TYPE_SMALLINT:
@@ -308,21 +321,19 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) {
     case TSDB_DATA_TYPE_TIMESTAMP:
       comparFn = compareInt64Val;
       break;
-    case TSDB_DATA_TYPE_BOOL:
-      comparFn = compareInt32Val;
-      break;
-    
     case TSDB_DATA_TYPE_FLOAT:
+      comparFn = compareFloatVal;
+      break;
     case TSDB_DATA_TYPE_DOUBLE:
       comparFn = compareDoubleVal;
       break;
     
     case TSDB_DATA_TYPE_BINARY:
-      comparFn = compareStrVal;
+      comparFn = compareLenPrefixedStr;
       break;
     
     case TSDB_DATA_TYPE_NCHAR:
-      comparFn = compareWStrVal;
+      comparFn = compareLenPrefixedWStr;
       break;
     
     default:
@@ -349,13 +360,20 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
       }
       return (ret < 0) ? -1 : 1;
     }
-    default: {
+    default: {  // todo refactor
-      int32_t ret = strncmp(f1, f2, (size_t)size);
+      tstr* t1 = (tstr*) f1;
-      if (ret == 0) {
+      tstr* t2 = (tstr*) f2;
-        return ret;
-      }
       
-      return (ret < 0) ? -1 : 1;
+      if (t1->len != t2->len) {
+        return t1->len > t2->len? 1:-1;
+      } else {
+        int32_t ret = strncmp(t1->data, t2->data, t1->len);
+        if (ret == 0) {
+          return 0;
+        } else {
+          return ret < 0? -1:1;
+        }
+      }
     }
   }
 }

src/util/src/tconfig.c

@@ -236,7 +236,7 @@ void taosReadGlobalLogCfg() {
   int    olen, vlen;
   char   fileName[PATH_MAX] = {0};
 
-  mdebugFlag = 135;
+  mDebugFlag = 135;
   sdbDebugFlag = 135;
 
   wordexp_t full_path;

src/vnode/src/vnodeMain.c

@@ -267,7 +267,7 @@ void vnodeRelease(void *pVnodeRaw) {
   assert(refCount >= 0);
 
   if (refCount > 0) {
-    vTrace("vgId:%d, release vnode, refCount:%d", pVnode, vgId, refCount);
+    vTrace("vgId:%d, release vnode, refCount:%d", vgId, refCount);
     return;
   }
 
@@ -383,9 +383,8 @@ static int vnodeWalCallback(void *arg) {
 }
 
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) {
-  // SVnodeObj *pVnode = ahandle;
+  SVnodeObj *pVnode = ahandle;
-  //tsdbGetFileInfo(pVnode->tsdb, name, index, size);
+  return tsdbGetFileInfo(pVnode->tsdb, name, index, size);
-  return 0;
 }
 
 static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) {
@@ -405,9 +404,9 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
 
 static void vnodeNotifyFileSynced(void *ahandle) {
   SVnodeObj *pVnode = ahandle;
-  vTrace("pVnode:%p vgId:%d, data file is synced", pVnode, pVnode->vgId);
+  vTrace("vgId:%d, data file is synced", pVnode->vgId);
 
-  // clsoe tsdb, then open tsdb
+  // close tsdb, then open tsdb
 }
 
 static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
@@ -693,8 +692,9 @@ static bool vnodeReadVersion(SVnodeObj *pVnode) {
   sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
   FILE *fp = fopen(versionFile, "r");
   if (!fp) {
-    vTrace("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId,
+    if (errno != ENOENT) {
-           versionFile, strerror(errno));
+      vError("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, versionFile, strerror(errno));
+    }
     return false;
   }
 

src/vnode/src/vnodeRead.c

@@ -65,7 +65,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
     pRet->len = sizeof(SQueryTableRsp);
     pRet->rsp = pRsp;
     
-    vTrace("vgId:%d QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo);
+    vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo);
   } else {
     pQInfo = pCont;
     code = TSDB_CODE_ACTION_IN_PROGRESS;
@@ -83,7 +83,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
 
   int32_t code = TSDB_CODE_SUCCESS;
 
-  vTrace("vgId:%d QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo);
+  vTrace("vgId:%d, QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo);
   
   pRet->code = qRetrieveQueryResultInfo(pQInfo);
   if (pRet->code != TSDB_CODE_SUCCESS) {
@@ -104,6 +104,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
     }
   }
   
-  vTrace("vgId:%d QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo);
+  vTrace("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo);
   return code;
 }

src/vnode/src/vnodeWrite.c

@@ -51,10 +51,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
   if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) 
     return TSDB_CODE_MSG_NOT_PROCESSED; 
 
-  if (pVnode->status != TAOS_VN_STATUS_READY && qtype == TAOS_QTYPE_RPC) 
+  if (pHead->version == 0) { // from client or CQ 
-    return TSDB_CODE_NOT_ACTIVE_VNODE; 
-
-  if (pHead->version == 0) { // from client 
     if (pVnode->status != TAOS_VN_STATUS_READY) 
       return TSDB_CODE_NOT_ACTIVE_VNODE;
 
@@ -64,12 +61,10 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
     // assign version
     pVnode->version++;
     pHead->version = pVnode->version;
-  } else {  
+  } else { // from wal or forward 
     // for data from WAL or forward, version may be smaller
     if (pHead->version <= pVnode->version) return 0;
   }
-   
-  // more status and role checking here
 
   pVnode->version = pHead->version;
 
@@ -77,9 +72,13 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
   code = walWrite(pVnode->wal, pHead);
   if (code < 0) return code;
 
-  int32_t syncCode = syncForwardToPeer(pVnode->sync, pHead, item);
+  // forward to peers if data is from RPC or CQ
+  int32_t syncCode = 0;
+  if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_CQ) 
+    syncCode = syncForwardToPeer(pVnode->sync, pHead, item);
   if (syncCode < 0) return syncCode;
 
+  // write data locally 
   code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
   if (code < 0) return code;
 
@@ -239,7 +238,7 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
 
   code = tsdbDropTable(pVnode->tsdb, stableId);
   
-  vTrace("vgId:%d, stable:%s, drop stable result:%s", pVnode, pTable->tableId, tstrerror(code));
+  vTrace("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableId, tstrerror(code));
  
   return code;
 }

tests/pytest/import_merge/importDataH2.py

@@ -46,16 +46,16 @@ class TDTestCase:
             self.maxrows)
 
         tdLog.info("================= step2")
-        tdLog.info("import %d sequential data" % (self.maxrows / 2))
+        tdLog.info("import %d sequential data" % (self.maxrows // 2))
         startTime = self.startTime
         sqlcmd = ['import into tb1 values']
-        for rid in range(1, self.maxrows / 2 + 1):
+        for rid in range(1, self.maxrows // 2 + 1):
             sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
         tdSql.execute(" ".join(sqlcmd))
 
         tdLog.info("================= step3")
         tdSql.query('select * from tb1')
-        tdSql.checkRows(self.maxrows / 2)
+        tdSql.checkRows(self.maxrows // 2)
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
@@ -70,7 +70,7 @@ class TDTestCase:
         tdLog.info("================= step7")
         tdSql.execute('reset query cache')
         tdSql.query('select * from tb1 order by ts desc')
-        tdSql.checkRows(self.maxrows / 2 + 1)
+        tdSql.checkRows(self.maxrows // 2 + 1)
 
         tdLog.info("================= step8")
         tdLog.info("import 10 data in batch before")
@@ -83,7 +83,7 @@ class TDTestCase:
         tdLog.info("================= step9")
         tdSql.execute('reset query cache')
         tdSql.query('select * from tb1 order by ts desc')
-        tdSql.checkRows(self.maxrows / 2 + 11)
+        tdSql.checkRows(self.maxrows // 2 + 11)
 
     def stop(self):
         tdSql.close()

tests/pytest/import_merge/importDataSub.py

@@ -46,16 +46,17 @@ class TDTestCase:
             self.maxrows)
 
         tdLog.info("================= step2")
-        tdLog.info("import %d sequential data" % (self.maxrows / 2))
+        tdLog.info("import %d sequential data" % (self.maxrows // 2))
         startTime = self.startTime
         sqlcmd = ['import into tb1 values']
-        for rid in range(1, self.maxrows / 2 + 1):
+        for rid in range(1, self.maxrows // 2 + 1):
             sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
+
         tdSql.execute(" ".join(sqlcmd))
 
         tdLog.info("================= step3")
         tdSql.query('select * from tb1')
-        tdSql.checkRows(self.maxrows / 2)
+        tdSql.checkRows(self.maxrows // 2)
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
@@ -73,7 +74,7 @@ class TDTestCase:
         tdLog.info("================= step9")
         tdSql.execute('reset query cache')
         tdSql.query('select * from tb1 order by ts desc')
-        tdSql.checkRows(self.maxrows / 2)
+        tdSql.checkRows(self.maxrows // 2)
 
     def stop(self):
         tdSql.close()

tests/pytest/import_merge/importToCommit.py

@@ -33,7 +33,7 @@ class TDTestCase:
         tdDnodes.start(1)
         tdSql.execute('reset query cache')
         tdSql.execute('drop database if exists db')
-        tdSql.execute('create database db cache 512 tables 10')
+        tdSql.execute('create database db cache 512 maxtables 10')
         tdSql.execute('use db')
 
         tdLog.info("================= step1")

tests/pytest/smoketest.sh

@@ -29,88 +29,17 @@ python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f table/db_table.py
 python3 ./test.py $1 -s && sleep 1
 
-python3 ./test.py $1 -f import_merge/importDataLastTO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importDataLastT.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importDataTO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importDataT.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importHeadOverlap.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importHORestart.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importHPORestart.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importHRestart.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importLastSub.py
-python3 ./test.py $1 -s && sleep 1
-
-python3 ./test.py $1 -f import_merge/importBlock1HO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock1HPO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock1H.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock1S.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock1Sub.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock1TO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock1TPO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock1T.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2HO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2HPO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2H.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2S.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2Sub.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2TO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2TPO.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlock2T.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importBlockbetween.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importCacheFileSub.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importCacheFileTO.py
-python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f import_merge/importCacheFileT.py
 python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f import_merge/importDataLastSub.py
 python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f import_merge/importHead.py
 python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importLastTO.py
-python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f import_merge/importLastT.py
 python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f import_merge/importSpan.py
 python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importSRestart.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importSubRestart.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importTailOverlap.py
-python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f import_merge/importTail.py
 python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importTORestart.py
-python3 ./test.py $1 -s && sleep 1
-python3 ./test.py $1 -f import_merge/importTPORestart.py
-python3 ./test.py $1 -s && sleep 1
 python3 ./test.py $1 -f import_merge/importTRestart.py
 python3 ./test.py $1 -s && sleep 1

tests/pytest/util/dnodes.py

@@ -201,8 +201,8 @@ class TDDnode:
         self.running = 1
         tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
 
-        tdLog.debug("wait 2 seconds for the dnode:%d to start." % (self.index))
+        tdLog.debug("wait 4 seconds for the dnode:%d to start." % (self.index))
-        time.sleep(2)
+        time.sleep(4)
 
     def stop(self):
         if self.valgrind == 0:

tests/pytest/valgrind-test.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+python3 ./test.py $1 -f insert/basic.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/int.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/float.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/bigint.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/bool.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/double.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/smallint.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/tinyint.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/binary.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/date.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f insert/nchar.py
+python3 ./test.py $1 -s && sleep 1
+
+python3 ./test.py $1 -f table/column_name.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f table/column_num.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f table/db_table.py
+python3 ./test.py $1 -s && sleep 1
+
+python3 ./test.py $1 -f import_merge/importCacheFileT.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f import_merge/importDataLastSub.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f import_merge/importHead.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f import_merge/importLastT.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f import_merge/importSpan.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f import_merge/importTail.py
+python3 ./test.py $1 -s && sleep 1
+python3 ./test.py $1 -f import_merge/importTRestart.py
+python3 ./test.py $1 -s && sleep 1

tests/script/basicSuite.sim

@@ -1,27 +1,22 @@
-################################# 
 run general/cache/new_metrics.sim
 run general/compress/compress.sim
 run general/compute/avg.sim
-run general/compute/bottom.sim
 run general/compute/count.sim
 run general/db/len.sim
 run general/db/basic4.sim
 run general/http/restful_insert.sim
 run general/import/basic.sim
 run general/import/commit.sim
-run general/insert/basic.sim
 run general/insert/query_file_memory.sim
 run general/parser/binary_escapeCharacter.sim
 run general/parser/columnValue_bigint.sim
 run general/parser/select_from_cache_disk.sim
 run general/table/autocreate.sim
-run general/table/basic3.sim
 run general/table/column_name.sim
 run general/table/int.sim
 run general/table/vgroup.sim
 run general/user/basic1.sim
-run general/user/user_len.sim
+run general/user/pass_alter.sim
 run general/vector/single.sim
 run general/connection/connection.sim
 run general/user/authority.sim
-################################## 

tests/script/general/cache/restart_metrics.sim

@@ -27,11 +27,15 @@ sql create table $tb using $mt tags( "1" )
 sql insert into $tb values (now, 1)
 
 sql select * from $tb
-print ===>rows $rows, data $data01
+#print ===>rows $rows, data $data01
+
 if $rows != 1 then
+  print expect 1, actual: $rows
   return -1
-endi  
+endi
+
 if $data01 != 1 then
+  print expect 1 actual: $data01
   return -1
 endi  
 
@@ -66,8 +70,10 @@ sql select * from $tb
 print ===>rows $rows, data $data01
 if $rows != 1 then
   return -1
-endi  
+endi
+
 if $data01 != 1 then
+  print expect 1, actual $data01
   return -1
 endi  
 
@@ -80,6 +86,7 @@ if $data01 != 1 then
   return -1
 endi  
 if $data02 != 3 then
+  print expect 3 actual: $data02
   return -1
 endi 
 

tests/script/general/compute/diff2.sim

@@ -81,7 +81,7 @@ sql_error select diff(c8) from $tb
 sql_error select diff(c9) from $tb
 sql_error select diff(ts) from $tb
 sql_error select diff(c1), diff(c2) from $tb
-sql_error select 2+diff(c1) from $tb
+#sql_error select 2+diff(c1) from $tb
 sql_error select diff(c1+2) from $tb
 sql_error select diff(c1) from $tb where ts > 0 and ts < now + 100m interval(10m)
 sql_error select diff(c1) from $mt

tests/script/general/db/delete_reuse2.sim

@@ -26,6 +26,7 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
 
 print ========= start dnodes
+sleep 2000
 system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect

tests/script/general/db/delete_reusevnode2.sim

@@ -1,7 +1,5 @@
 system sh/stop_dnodes.sh
-
 system sh/deploy.sh -n dnode1 -i 1
-
 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10
 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 10
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 10
@@ -36,7 +34,7 @@ while $i < 10
     sql create table $tb1 using st tags(1)
     sql insert into  $tb1 values(now, 1);
 
-    $tb1 = $tb . 5
+    $tb1 = $tb . 4
     sql create table $tb1 using st tags(1)
     sql insert into  $tb1 values(now, 1);
 
@@ -58,7 +56,7 @@ while $i < 10
 
   sql drop table st
 
-  sleep 1000
+  sleep 2000
   print times $i
   $i = $i + 1
 

tests/script/general/db/testSuite.sim

@@ -1,4 +1,4 @@
-run general//db/basic.sim
+run general/db/basic.sim
 run general/db/basic1.sim
 run general/db/basic2.sim
 run general/db/basic3.sim
@@ -13,3 +13,4 @@ run general/db/delete_writing2.sim
 run general/db/len.sim
 run general/db/repeat.sim
 run general/db/tables.sim
+run general/db/vnodes.sim

tests/script/general/db/vnodes.sim

@@ -0,0 +1,45 @@
+system sh/stop_dnodes.sh
+
+$totalVnodes = 10
+$maxTables = 4
+$totalRows = $totalVnodes * $maxTables
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel            -v 0
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode   -v $maxTables
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes    -v $totalVnodes
+system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000
+system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000
+system sh/cfg.sh -n dnode1 -c maxShellConns       -v 100000
+system sh/cfg.sh -n dnode1 -c maxMgmtConnections  -v 100000
+
+print ==========  prepare data
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect 
+sql create database db blocks 2 cache 1 maxTables $maxTables
+sql use db
+
+print ==========  step1
+sql create table mt (ts timestamp, tbcol int) TAGS(tgcol int)
+
+$x = 0
+while $x < $totalRows
+  $tb = t . $x
+  sql create table $tb using mt tags( $x )
+  sql insert into $tb values (now, $x ) 
+  $x = $x + 1
+endw 
+
+print ========== step2
+sql select * from mt
+if $rows != $totalRows then
+  return -1
+endi  
+
+sql select count(*) from mt
+if $data00 != $totalRows then
+  return -1
+endi  
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT

tests/script/general/field/bigint.sim

@@ -132,7 +132,9 @@ endi
 print =============== step6
 sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 
-if $data00 != 100 then 
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
+if $data00 != 100 then
+  print expect 100, actual $data00
   return -1
 endi
 

tests/script/general/parser/auto_create_tb.sim

@@ -96,25 +96,25 @@ $ts1 = $ts0 + 1000
 $ts2 = $ts0 + 2000
 sql insert into tb_1 using $stb tags (-1) values ( $ts1 , 1,1,1,1,'bin',1,1,1,'涛思数据') ( $ts2 , 2,2,2,2,'binar', 1,1,1,'nchar')
 sql select * from $stb
-if $rows != 3 then
+if $rows != 5 then
   return -1
 endi
-if $data19 != 涛思数据 then
+if $data09 != 涛思数据 then
   return -1
 endi
-if $data11 != 1 then
+if $data01 != 1 then
   return -1
 endi
-if $data22 != 2 then
+if $data42 != 2 then
   return -2
 endi
-if $data23 != 2.00000 then
+if $data43 != 2.00000 then
   return -1
 endi
-if $data25 != binar then
+if $data45 != binar then
   return -1
 endi
-if $data29 != nchar then
+if $data49 != nchar then
   return -1
 endi
 sql drop table tb_1
@@ -127,22 +127,22 @@ sql select * from $stb
 if $rows != 5 then
   return -1
 endi
-if $data19 != 涛思数据 then
+if $data09 != 涛思数据 then
   return -1
 endi
-if $data11 != 1 then
+if $data01 != 1 then
   return -1
 endi
-if $data22 != 2 then
+if $data42 != 2 then
   return -2
 endi
-if $data23 != 2.00000 then
+if $data43 != 2.00000 then
   return -1
 endi
-if $data25 != binar then
+if $data45 != binar then
   return -1
 endi
-if $data29 != nchar then
+if $data49 != nchar then
   return -1
 endi
 
@@ -154,13 +154,13 @@ sql show tables
 if $rows != 3 then
   return -1
 endi
-if $data00 != tb3 then
+if $data00 != tb1 then
   return -1
 endi
 if $data10 != tb2 then
   return -1
 endi
-if $data20 != tb1 then
+if $data20 != tb3 then
   return -1
 endi
 

tests/script/general/parser/lastrow_query.sim

@@ -43,9 +43,11 @@ if $data07 != 1 then
   return -1
 endi
 if $data08 != BINARY then
+  print expect BINARY actual: $data08
   return -1
 endi
 if $data09 != NCHAR then
+  print expect NCHAR actual: $data09
   return -1
 endi
 

tests/script/general/parser/nchar.sim

@@ -194,8 +194,10 @@ sql reset query cache
 sql select * from $mt where tgcol = '1'
 step2:
 if $rows != 100  then
+  print expect 100, actual: $rows
   return -1
 endi
+
 sql select * from $mt where tgcol > '0'
 #print rows = $rows
 if $rows != 100 then

tests/script/general/parser/testSuite.sim

@@ -1,113 +1,100 @@
-# run general/parser/alter.sim
+#run general/parser/alter.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/alter1.sim
+#run general/parser/alter1.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/alter_stable.sim
+#run general/parser/alter_stable.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/auto_create_tb.sim
+#run general/parser/auto_create_tb.sim
-# sleep 2000
+#sleep 2000
-# run general/parser/auto_create_tb_drop_tb.sim
+#run general/parser/auto_create_tb_drop_tb.sim
-# sleep 2000
 
-run general/parser/binary_escapeCharacter.sim
-sleep 2000
-run general/parser/bug.sim
 sleep 2000
 run general/parser/col_arithmetic_operation.sim
 sleep 2000
-run general/parser/columnValue_bigint.sim
+run general/parser/columnValue.sim
 sleep 2000
-run general/parser/columnValue_bool.sim
+run general/parser/commit.sim
 sleep 2000
-run general/parser/columnValue_double.sim
+run general/parser/create_db.sim
 sleep 2000
-run general/parser/columnValue_float.sim
+run general/parser/create_mt.sim
 sleep 2000
-run general/parser/columnValue_int.sim
+run general/parser/create_tb.sim
 sleep 2000
+run general/parser/dbtbnameValidate.sim
+sleep 2000
+run general/parser/import_commit1.sim
+sleep 2000
+run general/parser/import_commit2.sim
+sleep 2000
+run general/parser/import_commit3.sim
+sleep 2000
+run general/parser/insert_tb.sim
+sleep 2000
+run general/parser/first_last.sim
+sleep 2000
+run general/parser/import_file.sim
+sleep 2000
+run general/parser/lastrow.sim
+sleep 2000
+run general/parser/nchar.sim
 
-# sleep 2000
+sleep 2000
-# run general/parser/col_arithmetic_operation.sim
+run general/parser/null_char.sim
-# sleep 2000
+sleep 2000
-# run general/parser/columnValue.sim
+run general/parser/single_row_in_tb.sim
-# sleep 2000
+sleep 2000
-# run general/parser/commit.sim
-# run general/parser/create_db.sim
-# sleep 2000
-# run general/parser/create_mt.sim
-# sleep 2000
-# run general/parser/create_tb.sim
-# sleep 2000
-# run general/parser/dbtbnameValidate.sim
-# sleep 2000
-# run general/parser/fill.sim
-# sleep 2000
-# run general/parser/fill_stb.sim
-# sleep 2000
-# run general/parser/first_last.sim
-# sleep 2000
-# run general/parser/import_commit1.sim
-# sleep 2000
-# run general/parser/import_commit2.sim
-# sleep 2000
-# run general/parser/import_commit3.sim
-# sleep 2000
-# run general/parser/import_file.sim
-# sleep 2000
-# run general/parser/insert_tb.sim
-# sleep 2000
-# run general/parser/tags_dynamically_specifiy.sim
-# sleep 2000
-# run general/parser/interp.sim
-# run general/parser/lastrow.sim
-# sleep 2000
-# run general/parser/limit.sim
-# sleep 2000
-# run general/parser/limit1.sim
-# sleep 2000
-# run general/parser/limit1_tblocks100.sim
-# sleep 2000
-# run general/parser/limit2.sim
-# sleep 2000
-# run general/parser/mixed_blocks.sim
-# sleep 2000
-# run general/parser/nchar.sim
-# sleep 2000
-# run general/parser/null_char.sim
-# sleep 2000
-# run general/parser/selectResNum.sim
-# sleep 2000
-# run general/parser/select_across_vnodes.sim
-# sleep 2000
 run general/parser/select_from_cache_disk.sim
 sleep 2000
-# run general/parser/set_tag_vals.sim
+run general/parser/limit.sim
-# sleep 2000
+
-# run general/parser/single_row_in_tb.sim
+sleep 2000
-# sleep 2000
+run general/parser/fill.sim
-# run general/parser/slimit.sim
+sleep 2000
-# sleep 2000
+run general/parser/fill_stb.sim
+sleep 2000
+run general/parser/tags_dynamically_specifiy.sim
+sleep 2000
+run general/parser/interp.sim
+sleep 2000
+run general/parser/limit1.sim
+sleep 2000
+run general/parser/limit1_tblocks100.sim
+sleep 2000
+run general/parser/limit2.sim
+sleep 2000
+run general/parser/mixed_blocks.sim
+sleep 2000
+run general/parser/selectResNum.sim
+sleep 2000
+run general/parser/select_across_vnodes.sim
+sleep 2000
+run general/parser/set_tag_vals.sim
+sleep 2000
+run general/parser/slimit.sim
+sleep 2000
 run general/parser/slimit1.sim
 sleep 2000
-run general/parser/slimit1_query.sim
+run general/parser/slimit_alter_tags.sim
 sleep 2000
-# run general/parser/slimit_alter_tags.sim
+run general/parser/stream_on_sys.sim
-# sleep 2000
+sleep 2000
-# run general/parser/stream_on_sys.sim
+run general/parser/stream.sim
-# sleep 2000
+sleep 2000
-# run general/parser/stream.sim
+run general/parser/tbnameIn.sim
-# sleep 2000
+sleep 2000
-# run general/parser/tbnameIn.sim
+run general/parser/where.sim
-# sleep 2000
+sleep 2000
-# run general/parser/where.sim
+#run general/parser/repeatAlter.sim
-# sleep 2000
+sleep 2000
-# #run general/parser/repeatAlter.sim
+#run general/parser/repeatStream.sim
-# sleep 2000
+sleep 2000
-# #run general/parser/repeatStream.sim
+run general/parser/join.sim
-# sleep 2000
+sleep 2000
-# run general/parser/join.sim
+run general/parser/join_multivnode.sim
-# run general/parser/join_multivnode.sim
+sleep 2000
-# run general/parser/projection_limit_offset.sim
+run general/parser/projection_limit_offset.sim
-# sleep 2000
+sleep 2000
-# run general/parser/select_with_tags.sim
+run general/parser/select_with_tags.sim
-# run general/parser/groupby.sim
+sleep 2000
+run general/parser/groupby.sim

tests/script/general/table/fill.sim

@@ -42,7 +42,7 @@ sql select count(*), last(ts), min(k), max(k), avg(k) from db.mt where a=0 and t
 
 print =================== step2
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 10000
+sleep 5000
 system sh/exec.sh -n dnode1 -s start
 sleep 3000
 

tests/script/general/tag/bool.sim

@@ -122,7 +122,8 @@ if $rows != 100 then
   return -1
 endi
 sql select * from $mt where tgcol = 1
-if $rows != 100 then 
+if $rows != 100 then
+  print expect 100, actual:$rows
   return -1
 endi
 sql select * from $mt where tgcol <> 1

tests/script/general/tag/create.sim

@@ -134,7 +134,8 @@ if $data01 != 1 then
   return -1
 endi
 sql select * from $mt where tgcol = 0
-if $rows != 0 then 
+if $rows != 0 then
+  print expect 0, actual: $rows
   return -1
 endi
   
@@ -184,7 +185,8 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol bool, tgcol2 bool)
 sql create table $tb using $mt tags( 1, 2 )
 sql insert into $tb values(now, 1)  
 sql select * from $mt where tgcol2 = 2
-if $rows != 1 then 
+if $rows != 1 then
+  print expect 1, actual: $rows
   return -1
 endi
 if $data01 != 1 then 

tests/script/general/tag/filter.sim

@@ -122,7 +122,8 @@ endi
 print =============== step14
 sql select count(tbcol) as c from $mt where ts > 1000 group by tgcol
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 
-if $data00 != 100 then 
+if $data00 != 100 then
+  print expect 100, actual $data00
   return -1
 endi
 

tests/script/general/user/pass_alter.sim

@@ -19,28 +19,47 @@ endi
 
 print ============= step2
 sql close
+sleep 2500
+print user read login
 sql connect read
-sleep 2000
 sql alter user read pass 'taosdata'
 sql alter user write pass 'taosdata1' -x step2
   return -1
 step2:
 
+sql_error create user read pass 'taosdata1'
+sql_error create user write pass 'taosdata1'
+
+sql show users
+if $rows != 5 then 
+  return -1
+endi
 
 print ============= step3
 sql close
+sleep 2500
+print user write login
 sql connect write
-sleep 2000
+
+sql_error create user read pass 'taosdata1'
+sql_error create user write pass 'taosdata1'
 sql alter user write pass 'taosdata'
 sql alter user read pass 'taosdata' -x step3
   return -1
 step3:
 
+sql show users
+if $rows != 5 then 
+  return -1
+endi
+
 print ============= step4
 sql close 
+sleep 2500
+print root write login
 sql connect
-sleep 2000
 sql create user oroot pass 'taosdata'
+
 sql show users
 if $rows != 6 then 
   return -1

tests/script/jenkins/basic.txt

@@ -1,9 +1,6 @@
 cd ../../debug; cmake ..
-#cd ../../debug; make clean
 cd ../../debug; make
-
 cd ../../../debug; cmake ..
-#cd ../../../debug; make clean
 cd ../../../debug; make
 
 #./test.sh -f general/alter/cached_schema_after_alter.sim
@@ -15,12 +12,12 @@ cd ../../../debug; make
 #./test.sh -f general/alter/table.sim
 
 ./test.sh -f general/cache/new_metrics.sim
-#./test.sh -f general/cache/restart_metrics.sim
+./test.sh -f general/cache/restart_metrics.sim
 ./test.sh -f general/cache/restart_table.sim
 
-#./test.sh -f general/column/commit.sim
+#hongze ./test.sh -f general/column/commit.sim
-#./test.sh -f general/column/metrics.sim
+#hongze ./test.sh -f general/column/metrics.sim
-#./test.sh -f general/column/table.sim
+#hongze ./test.sh -f general/column/table.sim
 
 ./test.sh -f general/compress/commitlog.sim
 ./test.sh -f general/compress/compress.sim
@@ -31,11 +28,11 @@ cd ../../../debug; make
 ./test.sh -f general/compute/bottom.sim
 ./test.sh -f general/compute/count.sim
 ./test.sh -f general/compute/diff.sim
-#./test.sh -f general/compute/diff2.sim
+# liao./test.sh -f general/compute/diff2.sim
 ./test.sh -f general/compute/first.sim
-#./test.sh -f general/compute/interval.sim
+# liao./test.sh -f general/compute/interval.sim
-#./test.sh -f general/compute/last.sim
+# liao./test.sh -f general/compute/last.sim
-#./test.sh -f general/compute/leastsquare.sim
+# liao./test.sh -f general/compute/leastsquare.sim
 ./test.sh -f general/compute/max.sim
 ./test.sh -f general/compute/min.sim
 ./test.sh -f general/compute/null.sim
@@ -53,41 +50,42 @@ cd ../../../debug; make
 ./test.sh -f general/db/delete_reuse1.sim
 ./test.sh -f general/db/delete_reuse2.sim
 ./test.sh -f general/db/delete_reusevnode.sim
-#./test.sh -f general/db/delete_reusevnode2.sim
+./test.sh -f general/db/delete_reusevnode2.sim
 ./test.sh -f general/db/delete_writing1.sim
 ./test.sh -f general/db/delete_writing2.sim
 ./test.sh -f general/db/len.sim
+#./test.sh -u -f general/db/vnodes.sim
 ./test.sh -f general/db/repeat.sim
 ./test.sh -f general/db/tables.sim
 
-#./test.sh -f general/field/2.sim
+./test.sh -f general/field/2.sim
 #./test.sh -f general/field/3.sim
 #./test.sh -f general/field/4.sim
 #./test.sh -f general/field/5.sim
 #./test.sh -f general/field/6.sim
-##./test.sh -f general/field/bigint.sim
+./test.sh -f general/field/bigint.sim
-#./test.sh -f general/field/binary.sim
+# liao./test.sh -f general/field/binary.sim
-##./test.sh -f general/field/bool.sim
+./test.sh -f general/field/bool.sim
-#./test.sh -f general/field/single.sim
+./test.sh -f general/field/single.sim
-##./test.sh -f general/field/smallint.sim
+./test.sh -f general/field/smallint.sim
-##./test.sh -f general/field/tinyint.sim
+./test.sh -f general/field/tinyint.sim
 
-##./test.sh -f general/http/restful.sim
+# jeff ./test.sh -f general/http/restful.sim
 ./test.sh -f general/http/restful_insert.sim
 ./test.sh -f general/http/restful_limit.sim
-##./test.sh -f general/http/restful_full.sim
+# jeff ./test.sh -f general/http/restful_full.sim
 ./test.sh -f general/http/prepare.sim
 ./test.sh -f general/http/telegraf.sim
 ./test.sh -f general/http/grafana_bug.sim
-#./test.sh -f general/http/grafana.sim
+# jeff ./test.sh -f general/http/grafana.sim
 
 ./test.sh -f general/import/basic.sim
 ./test.sh -f general/import/commit.sim
 ./test.sh -f general/import/large.sim
-#./test.sh -f general/import/replica1.sim
+#hongze ./test.sh -f general/import/replica1.sim
 
 ./test.sh -f general/insert/basic.sim
-#./test.sh -f general/insert/insert_drop.sim
+#hongze ./test.sh -f general/insert/insert_drop.sim
 ./test.sh -f general/insert/query_block1_memory.sim
 ./test.sh -f general/insert/query_block2_memory.sim
 ./test.sh -f general/insert/query_block1_file.sim
@@ -96,8 +94,6 @@ cd ../../../debug; make
 ./test.sh -f general/insert/query_multi_file.sim
 ./test.sh -f general/insert/tcp.sim
 
-#parser
-
 # ./test.sh -f general/parser/alter.sim
 # ./test.sh -f general/parser/alter1.sim
 # ./test.sh -f general/parser/alter_stable.sim
@@ -112,8 +108,8 @@ cd ../../../debug; make
 ./test.sh -f general/parser/columnValue_float.sim
 ./test.sh -f general/parser/columnValue_int.sim
 # ./test.sh -f general/parser/col_arithmetic_operation.sim
-# ./test.sh -f general/parser/columnValue.sim
+./test.sh -f general/parser/columnValue.sim
-# ./test.sh -f general/parser/commit.sim
+./test.sh -f general/parser/commit.sim
 # ./test.sh -f general/parser/create_db.sim
 # ./test.sh -f general/parser/create_mt.sim
 # ./test.sh -f general/parser/create_tb.sim
@@ -121,9 +117,9 @@ cd ../../../debug; make
 # ./test.sh -f general/parser/fill.sim
 # ./test.sh -f general/parser/fill_stb.sim
 # ./test.sh -f general/parser/first_last.sim
-# ./test.sh -f general/parser/import_commit1.sim
+./test.sh -f general/parser/import_commit1.sim
-# ./test.sh -f general/parser/import_commit2.sim
+./test.sh -f general/parser/import_commit2.sim
-# ./test.sh -f general/parser/import_commit3.sim
+./test.sh -f general/parser/import_commit3.sim
 # ./test.sh -f general/parser/import_file.sim
 # ./test.sh -f general/parser/insert_tb.sim
 # ./test.sh -f general/parser/tags_dynamically_specifiy.sim
@@ -160,14 +156,12 @@ cd ../../../debug; make
 #./test.sh -f general/stable/disk.sim
 #./test.sh -f general/stable/metrics.sim
 #./test.sh -f general/stable/values.sim
-#./test.sh -f general/stable/vnode3.sim
+./test.sh -f general/stable/vnode3.sim
-
-#stream
 
 ./test.sh -f general/table/autocreate.sim
 ./test.sh -f general/table/basic1.sim
 ./test.sh -f general/table/basic2.sim
-./test.sh -f general/table/basic3.sim
+#hongze ./test.sh -f general/table/basic3.sim
 ./test.sh -f general/table/bigint.sim
 ./test.sh -f general/table/binary.sim
 ./test.sh -f general/table/bool.sim
@@ -177,17 +171,17 @@ cd ../../../debug; make
 ./test.sh -f general/table/column2.sim
 ./test.sh -f general/table/date.sim
 ./test.sh -f general/table/db.table.sim
-#./test.sh -f general/table/delete_reuse1.sim
+./test.sh -f general/table/delete_reuse1.sim
-#./test.sh -f general/table/delete_reuse2.sim
+./test.sh -f general/table/delete_reuse2.sim
-#./test.sh -f general/table/delete_writing.sim
+#hongze ./test.sh -f general/table/delete_writing.sim
-#./test.sh -f general/table/describe.sim
+./test.sh -f general/table/describe.sim
 ./test.sh -f general/table/double.sim
-#./test.sh -f general/table/fill.sim
+./test.sh -f general/table/fill.sim
 ./test.sh -f general/table/float.sim
 ./test.sh -f general/table/int.sim
 ./test.sh -f general/table/limit.sim
 ./test.sh -f general/table/smallint.sim
-#./test.sh -f general/table/table_len.sim
+./test.sh -f general/table/table_len.sim
 ./test.sh -f general/table/table.sim
 ./test.sh -f general/table/tinyint.sim
 ./test.sh -f general/table/vgroup.sim
@@ -197,63 +191,61 @@ cd ../../../debug; make
 #./test.sh -f general/tag/5.sim
 #./test.sh -f general/tag/6.sim
 #./test.sh -f general/tag/add.sim
-#./test.sh -f general/tag/bigint.sim
+./test.sh -f general/tag/bigint.sim
 #./test.sh -f general/tag/binary_binary.sim
 #./test.sh -f general/tag/binary.sim
 #./test.sh -f general/tag/bool_binary.sim
 #./test.sh -f general/tag/bool_int.sim
-#./test.sh -f general/tag/bool.sim
+./test.sh -f general/tag/bool.sim
 #./test.sh -f general/tag/change.sim
-#./test.sh -f general/tag/column.sim
+#liao ./test.sh -f general/tag/column.sim
 #./test.sh -f general/tag/commit.sim
-#./test.sh -f general/tag/create.sim
+#liao ./test.sh -f general/tag/create.sim
 #./test.sh -f general/tag/delete.sim
 #./test.sh -f general/tag/double.sim
-#./test.sh -f general/tag/filter.sim
+./test.sh -f general/tag/filter.sim
 #./test.sh -f general/tag/float.sim
 #./test.sh -f general/tag/int_binary.sim
 #./test.sh -f general/tag/int_float.sim
-#./test.sh -f general/tag/int.sim
+./test.sh -f general/tag/int.sim
 #./test.sh -f general/tag/set.sim
-#./test.sh -f general/tag/smallint.sim
+./test.sh -f general/tag/smallint.sim
-#./test.sh -f general/tag/tinyint.sim
+./test.sh -f general/tag/tinyint.sim
 
 ./test.sh -f general/user/basic1.sim
-#./test.sh -f general/user/monitor.sim
+./test.sh -f general/user/monitor.sim
-##./test.sh -f general/user/pass_alter.sim
+./test.sh -f general/user/pass_alter.sim
 ./test.sh -f general/user/pass_len.sim
-##./test.sh -f general/user/user_create.sim
+./test.sh -f general/user/user_create.sim
 ./test.sh -f general/user/user_len.sim
 
-#./test.sh -f general/vector/metrics_field.sim
+./test.sh -f general/vector/metrics_field.sim
-#./test.sh -f general/vector/metrics_mix.sim
+./test.sh -f general/vector/metrics_mix.sim
-#./test.sh -f general/vector/metrics_query.sim
+./test.sh -f general/vector/metrics_query.sim
-#./test.sh -f general/vector/metrics_tag.sim
+./test.sh -f general/vector/metrics_tag.sim
-#./test.sh -f general/vector/metrics_time.sim
+./test.sh -f general/vector/metrics_time.sim
-#./test.sh -f general/vector/multi.sim
+#liao ./test.sh -f general/vector/multi.sim
 ./test.sh -f general/vector/single.sim
-#./test.sh -f general/vector/table_field.sim
+./test.sh -f general/vector/table_field.sim
-#./test.sh -f general/vector/table_mix.sim
+./test.sh -f general/vector/table_mix.sim
-#./test.sh -f general/vector/table_query.sim
+./test.sh -f general/vector/table_query.sim
-#./test.sh -f general/vector/table_time.sim
+./test.sh -f general/vector/table_time.sim
-
-#################################  
 
 ./test.sh -u -f unique/account/account_create.sim
-##./test.sh -u -f unique/account/account_delete.sim
+./test.sh -u -f unique/account/account_delete.sim
 ./test.sh -u -f unique/account/account_len.sim
-##./test.sh -u -f unique/account/authority.sim
+./test.sh -u -f unique/account/authority.sim
 ./test.sh -u -f unique/account/basic.sim
 ./test.sh -u -f unique/account/paras.sim
-##./test.sh -u -f unique/account/pass_alter.sim
+./test.sh -u -f unique/account/pass_alter.sim
 ./test.sh -u -f unique/account/pass_len.sim
 ./test.sh -u -f unique/account/usage.sim
-##./test.sh -u -f unique/account/user_create.sim
+./test.sh -u -f unique/account/user_create.sim
 ./test.sh -u -f unique/account/user_len.sim
 
 #./test.sh -u -f unique/big/balance.sim
-#./test.sh -u -f unique/big/maxvnodes.sim
+#slguan ./test.sh -u -f unique/big/maxvnodes.sim
-#./test.sh -u -f unique/big/tcp.sim
+./test.sh -u -f unique/big/tcp.sim
 
 ##./test.sh -u -f unique/cluster/balance1.sim
 ##./test.sh -u -f unique/cluster/balance2.sim
@@ -298,27 +290,22 @@ cd ../../../debug; make
 #./test.sh -u -f unique/metrics/replica3_dnode6.sim
 #./test.sh -u -f unique/metrics/replica3_vnode3.sim
 
-##./test.sh -u -f unique/mnode/mgmt22.sim
+./test.sh -u -f unique/mnode/mgmt22.sim
-##./test.sh -u -f unique/mnode/mgmt23.sim
+./test.sh -u -f unique/mnode/mgmt23.sim
-##./test.sh -u -f unique/mnode/mgmt24.sim
+./test.sh -u -f unique/mnode/mgmt24.sim
-##./test.sh -u -f unique/mnode/mgmt25.sim
+./test.sh -u -f unique/mnode/mgmt25.sim
-##./test.sh -u -f unique/mnode/mgmt26.sim
+./test.sh -u -f unique/mnode/mgmt26.sim
-##./test.sh -u -f unique/mnode/mgmt33.sim
+./test.sh -u -f unique/mnode/mgmt33.sim
-##./test.sh -u -f unique/mnode/mgmt34.sim
+./test.sh -u -f unique/mnode/mgmt34.sim
 #./test.sh -u -f unique/mnode/mgmtr2.sim
 #./test.sh -u -f unique/mnode/secondIp.sim
 
-#stream
-
 ##./test.sh -u -f unique/table/delete_part.sim
 
-##./test.sh -u -f unique/vnode/replica2_basic2.sim
-##./test.sh -u -f unique/vnode/replica3_basic.sim
-
 #./test.sh -u -f unique/vnode/commit.sim
 #./test.sh -u -f unique/vnode/many.sim
 #./test.sh -u -f unique/vnode/replica2_basic.sim
-##./test.sh -u -f unique/vnode/replica2_basic2.sim
+./test.sh -u -f unique/vnode/replica2_basic2.sim
 #./test.sh -u -f unique/vnode/replica2_repeat.sim
 ##./test.sh -u -f unique/vnode/replica3_basic.sim
 #./test.sh -u -f unique/vnode/replica3_repeat.sim

tests/script/sh/deploy.sh

@@ -99,7 +99,7 @@ echo "logDir              $LOG_DIR"       >> $TAOS_CFG
 echo "dDebugFlag          199"            >> $TAOS_CFG
 echo "mDebugFlag          199"            >> $TAOS_CFG
 echo "sdbDebugFlag        199"            >> $TAOS_CFG
-echo "rpcDebugFlag        135"            >> $TAOS_CFG
+echo "rpcDebugFlag        151"            >> $TAOS_CFG
 echo "tmrDebugFlag        131"            >> $TAOS_CFG
 echo "cDebugFlag          135"            >> $TAOS_CFG
 echo "httpDebugFlag       135"            >> $TAOS_CFG

tests/script/test.sh

@@ -99,7 +99,8 @@ echo "cDebugFlag       135"                  >> $TAOS_CFG
 echo "httpDebugFlag    135"                  >> $TAOS_CFG
 echo "monitorDebugFlag 135"                  >> $TAOS_CFG
 echo "udebugFlag       135"                  >> $TAOS_CFG
-echo "clog             0"                    >> $TAOS_CFG
+echo "tablemetakeeptimer 5"                  >> $TAOS_CFG
+echo "wal              0"                    >> $TAOS_CFG
 echo "asyncLog         0"                    >> $TAOS_CFG
 echo "locale           en_US.UTF-8"          >> $TAOS_CFG
 echo " "                                     >> $TAOS_CFG

tests/script/unique/account/account_create.sim

@@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 print ============================ dnode1 start
 

tests/script/unique/account/account_delete.sim

@@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 print ============= step1
 sql create account oroot pass 'taosdata' 

tests/script/unique/account/account_len.sim

@@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 $i = 0
 $dbPrefix = aldb

tests/script/unique/account/authority.sim

@@ -4,8 +4,8 @@ system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 print ============= step1
 

tests/script/unique/account/basic.sim

@@ -1,6 +1,7 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
 system sh/exec_up.sh -n dnode1 -s start
+sleep 3000
 sql connect
 
 print =============== show accounts

tests/script/unique/account/paras.sim

@@ -1,6 +1,7 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
 system sh/exec_up.sh -n dnode1 -s start
+sleep 3000
 sql connect
 
 print =============== show accounts

tests/script/unique/account/pass_alter.sim

@@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 print ============= step1
 sql create user read pass 'taosdata1'

tests/script/unique/account/pass_len.sim

@@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 $i = 0
 $dbPrefix = apdb

tests/script/unique/account/usage.sim

@@ -1,6 +1,7 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
 system sh/exec_up.sh -n dnode1 -s start
+sleep 3000
 sql connect
 
 print =============== show accounts

tests/script/unique/account/user_create.sim

@@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 print =============== step1
 sql show users

tests/script/unique/account/user_len.sim

@@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/exec_up.sh -n dnode1 -s start
 
-sql connect
 sleep 3000
+sql connect
 
 $i = 0
 $dbPrefix = lm_us_db

tests/script/unique/big/balance.sim

@@ -20,7 +20,7 @@ system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 1000
 
 print =============== prepare data
-system sh/exec.sh -n dnode1 -s start
+system sh/exec_up.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
@@ -87,7 +87,7 @@ endi
 
 print ========== step1
 sql create dnode $hostname2
-system sh/exec.sh -n dnode2 -s start
+system sh/exec_up.sh -n dnode2 -s start
 
 $x = 0
 show1: 
@@ -98,12 +98,12 @@ show1:
 	endi
 	
 sql show dnodes
-print dnode1 freeVnodes $data3_192.168.0.1
+print dnode1 openvnodes $data2_1
-print dnode2 freeVnodes $data3_192.168.0.2
+print dnode2 openvnodes $data2_2
-if $data3_192.168.0.1 != 2 then
+if $data2_1 != 2 then
 	goto show1
 endi
-if $data3_192.168.0.2 != 2 then
+if $data2_2 != 2 then
 	goto show1
 endi
 
@@ -139,7 +139,7 @@ endi
 
 print ========== step2
 sql create dnode $hostname3
-system sh/exec.sh -n dnode3 -s start
+system sh/exec_up.sh -n dnode3 -s start
 sleep 10000
 
 print ========== step3
@@ -154,20 +154,20 @@ show3:
 	endi
 	
 sql show dnodes
-print dnode1 freeVnodes $data3_192.168.0.1
+print dnode1 freeVnodes $data2_1
-print dnode2 freeVnodes $data3_192.168.0.2
+print dnode2 freeVnodes $data2_2
-print dnode3 freeVnodes $data3_192.168.0.3
+print dnode3 freeVnodes $data2_3
-if $data3_192.168.0.1 != 2 then
+if $data2_1 != 2 then
 	goto show3
 endi
-if $data3_192.168.0.2 != NULL then
+if $data2_2 != NULL then
 	goto show3
 endi
-if $data3_192.168.0.3 != 2 then
+if $data2_3 != 2 then
 	goto show3
 endi
 
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec_up.sh -n dnode2 -s stop -x SIGINT
 
 sql select count(*) from t10
 print select count(*) from t10 $data00 expect $rowNum 
@@ -211,16 +211,16 @@ show4:
 	endi
 	
 sql show dnodes
-print dnode1 freeVnodes $data3_192.168.0.1
+print dnode1 freeVnodes $data2_1
-print dnode3 freeVnodes $data3_192.168.0.3
+print dnode3 freeVnodes $data2_3
-if $data3_192.168.0.1 != 0 then
+if $data2_1 != 0 then
 	goto show4
 endi
-if $data3_192.168.0.3 != NULL then
+if $data2_3 != NULL then
 	goto show4
 endi
 
-system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec_up.sh -n dnode3 -s stop -x SIGINT
 
 sql select count(*) from t10
 print select count(*) from t10 $data00 expect $rowNum 
@@ -255,7 +255,7 @@ endi
 print ========== step5
 sql alter database db replica 2
 sql create dnode $hostname4
-system sh/exec.sh -n dnode4 -s start
+system sh/exec_up.sh -n dnode4 -s start
 
 $x = 0
 show5: 
@@ -266,12 +266,12 @@ show5:
 	endi
 	
 sql show dnodes
-print dnode1 freeVnodes $data3_192.168.0.1
+print dnode1 freeVnodes $data2_1
-print dnode4 freeVnodes $data3_192.168.0.4
+print dnode4 freeVnodes $data2_4
-if $data3_192.168.0.1 != 0 then
+if $data2_1 != 0 then
 	goto show5
 endi
-if $data3_192.168.0.4 != 0 then
+if $data2_4 != 0 then
 	goto show5
 endi
 

tests/script/unique/big/maxvnodes.sim

@@ -1,25 +1,32 @@
 system sh/stop_dnodes.sh
 
+$totalVnodes = 100
+$minVnodes = 48
+$maxVnodes = 52
+$maxTables = 4
+$totalRows = $totalVnodes * $maxTables
+
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel            -v 0
-system sh/cfg.sh -n dnode1 -c maxtablesPerVnode   -v 100
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode   -v $maxTables
-system sh/cfg.sh -n dnode1 -c numOfTotalVnodes    -v 256
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes    -v $totalVnodes
 system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000
 system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000
 system sh/cfg.sh -n dnode1 -c maxShellConns       -v 100000
 system sh/cfg.sh -n dnode1 -c maxMgmtConnections  -v 100000
 
 print ==========  prepare data
-system sh/exec.sh -n dnode1 -s start
+system sh/exec_up.sh -n dnode1 -s start
+sleep 3000
 sql connect 
-sql create database db
+sql create database db blocks 2 cache 1 maxTables $maxTables
 sql use db
 
 print ==========  step1
 sql create table mt (ts timestamp, tbcol int) TAGS(tgcol int)
 
 $x = 0
-while $x < 25600
+while $x < $totalRows
   $tb = t . $x
   sql create table $tb using mt tags( $x )
   sql insert into $tb values (now, $x ) 
@@ -28,16 +35,15 @@ endw
 
 print ========== step2
 sql select * from mt
-if $rows != 25600 then
+if $rows != $totalRows then
   return -1
 endi  
 
 sql select count(*) from mt
-if $data00 != 25600 then
+if $data00 != $totalRows then
   return -1
 endi  
 
-
 system sh/deploy.sh -n dnode2 -i 2
 system sh/cfg.sh -n dnode2 -c walLevel            -v 0
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode   -v 100
@@ -49,7 +55,7 @@ system sh/cfg.sh -n dnode2 -c maxMgmtConnections  -v 100000
 
 print ==========  step2
 sql create dnode $hostname2
-system sh/exec.sh -n dnode2 -s start
+system sh/exec_up.sh -n dnode2 -s start
 
 $x = 0
 show3: 
@@ -60,23 +66,23 @@ show3:
 	endi
 	
 sql show dnodes
-print dnode1 freeVnodes $data3_192.168.0.1
+print dnode1 openvnodes $data2_1
-print dnode2 freeVnodes $data3_192.168.0.2
+print dnode2 openvnodes $data2_2
-if $data3_192.168.0.1 != 126 then
+if $data2_1 != $minVnodes then
 	goto show3
 endi
-if $data3_192.168.0.2 != 130 then
+if $data2_2 != $maxVnodes then
 	goto show3
 endi
 
 print ========== step4
 sql select * from mt
-if $rows != 25600 then
+if $rows != $totalRows then
   return -1
 endi  
 
 sql select count(*) from mt
-if $data00 != 25600 then
+if $data00 != $totalRows then
   return -1
 endi  
 

tests/script/unique/big/tcp.sim

@@ -14,8 +14,8 @@ system sh/cfg.sh -n dnode1 -c adminDebugFlag  -v 131
 system sh/cfg.sh -n dnode1 -c httpDebugFlag   -v 135
 system sh/cfg.sh -n dnode1 -c debugFlag       -v 131
 
-system sh/exec.sh -n dnode1 -s start
+system sh/exec_up.sh -n dnode1 -s start
-
+sleep 3000
 sql connect
 print ======================== dnode1 start
 
@@ -52,7 +52,7 @@ while $i < $tbNum
 endw 
 
 $i = 0
-while $i < 100
+while $i < 10
 	print =============== step3 $i
 	sql select count(*) from $mt
 	print ===> $data00 $data01
@@ -83,7 +83,7 @@ while $i < $tbNum
 endw 
 
 $i = 0
-while $i < 100
+while $i < 10
 	print =============== step5 $i
 	sql select count(*) from $mt where tgcol < 20200
 	print ===> $data00 $data01

tests/script/unique/mnode/mgmt22.sim

@@ -41,14 +41,21 @@ if $data2_2 != slave then
 	goto show2
 endi
 
+system sh/exec_up.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec_up.sh -n dnode8 -s stop  -x SIGINT
+
+return
+
 print ============== step3
 sql_error drop dnode $hostname1 -x error1
 print should not drop master
 
-system sh/exec_up.sh -n dnode1 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode2 -s stop  -x SIGINT
-return
-
 print ============== step4
 system sh/exec_up.sh -n dnode1 -s stop -x SIGINT
 sql_error show mnodes
@@ -83,7 +90,7 @@ if $data2_2 != slave then
 endi
 
 print ============== step7
-system sh/exec_up.sh -n dnode3 -s start -t
+system sh/exec_up.sh -n dnode3 -s start
 sql create dnode $hostname3
 sleep 5000
 
@@ -105,7 +112,7 @@ endi
 if $data2_2 != slave then
 	goto show7
 endi
-if $data3_3 != NULL then
+if $data3_3 != null then
 	goto show7
 endi