Merge branch 'develop' into feature/add-how-to-add-test-into-readme

.travis.yml

@@ -59,12 +59,12 @@ matrix:
           GREEN_UNDERLINE='\033[4;32m'
           NC='\033[0m'
 
-          grep 'ERROR SUMMARY' mem-error-out.txt | uniq | tee uniq-mem-error-out.txt
+          grep 'ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt
 
           for memError in `cat uniq-mem-error-out.txt | awk '{print $4}'` 
           do
             if [ -n "$memError" ]; then
-              if [ "$memError" -gt 16 ]; then
+              if [ "$memError" -gt 5 ]; then
                 echo -e "${RED} ## Memory errors number valgrind reports is $memError.\
                          More than our threshold! ## ${NC}"
                 travis_terminate $memError
@@ -72,11 +72,11 @@ matrix:
             fi
           done
 
-          grep 'definitely lost' mem-error-out.txt | uniq | tee uniq-definitely-lost-out.txt
+          grep 'definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt
-          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'` 
+          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'`
           do
             if [ -n "$defiMemError" ]; then
-              if [ "$defiMemError" -gt 16 ]; then
+              if [ "$defiMemError" -gt 3 ]; then
                 echo -e "${RED} ## Memory errors number valgrind reports \
                          Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
                 travis_terminate $defiMemError

packaging/cfg/taos.cfg

@@ -5,29 +5,16 @@
 #                                                      #
 ########################################################
 
-# master IP for TDengine system
+# first full-qualified domain name (FQDN) for TDengine system
-# masterIp              127.0.0.1
+# first                 127.0.0.1
 
-# second IP for TDengine system, for cluster version only 
+# second full-qualified domain name (FQDN) for TDengine system, for cluster edition only
-# secondIp              127.0.0.1
+# second              127.0.0.1
-
-# IP address of the server
-# privateIp             127.0.0.1
-
-# public IP of server, on which the tdengine are deployed
-# this IP is assigned by cloud service provider, for cluster version only 
-# publicIp              127.0.0.1   
-
-# network is bound to 0.0.0.0
-# anyIp                 1
 
 # set socket type ("udp" and "tcp")
 # the server and client should have the same socket type. Otherwise, connect will fail
 # sockettype            udp
 
-# client local IP
-# localIp               127.0.0.1
-
 # for the cluster version, data file's directory is configured this way 
 # option                mount_path           tier_level
 # dataDir               /mnt/disk1/taos      0
@@ -46,23 +33,8 @@
 # http service port, default tcp[6020]
 # httpPort              6020
 
-# port for MNode connect to Client, default udp[6030-6034] tcp[6030]
+# port for MNode connect to Client, default udp[6030-6055] tcp[6030]
-# mgmtShellPort         6030
+# serverPort            6030
-
-# port for DNode connect to Client, default udp[6035-6039] tcp[6035]
-# vnodeShellPort        6035
-
-# port for MNode connect to VNode, default udp[6040-6044] tcp[6040], for cluster version only 
-# mgmtVnodePort         6040
-
-# port for DNode connect to DNode, default tcp[6045], for cluster version only 
-# vnodeVnodePort        6045
-
-# port for MNode connect to MNode, default udp[6050], for cluster version only 
-# mgmtMgmtPort          6050
-
-# port sync file MNode and MNode, default tcp[6050], for cluster version only 
-# mgmtSyncPort          6050
 
 # number of threads per CPU core
 # numOfThreadsPerCore   1
@@ -73,20 +45,18 @@
 # number of total vnodes in DNode
 # numOfTotalVnodes      0
 
-# max number of sessions per vnode
+# max number of tables per vnode
-# tables                1024
+# maxtablesPerVnode     1000
 
 # cache block size
 # cache                 16384
 
-# row in file block
+# row of records in file block
-# rows                  4096
+# minRows                100
+# maxRows                4096
 
-# average cache blocks per meter
+# number of cache blocks per vnode
-# ablocks               4
+# blocks             2
-
-# max number of cache blocks per Meter
-# tblocks               512
 
 # interval of system monitor 
 # monitorInterval       60
@@ -118,11 +88,8 @@
 # interval of MNode send HB to MNode, unit is Second, for cluster version only 
 # mgmtPeerHBTimer       1
 
-# time to keep MeterMeta in Cache, seconds
+# duration of to keep tableMeta kept in Cache, seconds
-# meterMetaKeepTimer    7200
+# tableMetaKeepTimer    7200
-
-# time to keep MetricMeta in Cache, seconds
-# metricMetaKeepTimer   600 
 
 # max number of users
 # maxUsers              1000
@@ -148,8 +115,8 @@
 # system time zone
 # timezone              Asia/Shanghai (CST, +0800)
 
-# enable/disable commit log
+# set write ahead log (WAL) level
-# clog                  1
+# walLevel              1
 
 # enable/disable async log
 # asyncLog              1
@@ -268,6 +235,9 @@
 # debug flag for TDengine client 
 # cDebugFlag            131
 
+# debug flag for query
+# qDebugflag            131
+
 # debug flag for http server
 # httpDebugFlag         131
 

src/client/inc/tscLog.h

@@ -22,25 +22,25 @@ extern "C" {
 
 #include "tlog.h"
 
-extern int32_t cdebugFlag;
+extern int32_t cDebugFlag;
 
 #define tscError(...)                        \
-  if (cdebugFlag & DEBUG_ERROR) {            \
+  if (cDebugFlag & DEBUG_ERROR) {            \
     taosPrintLog("ERROR TSC ", 255, __VA_ARGS__); \
   }
 #define tscWarn(...)                                \
-  if (cdebugFlag & DEBUG_WARN) {                    \
+  if (cDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  TSC ", cdebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN  TSC ", cDebugFlag, __VA_ARGS__); \
   }
 #define tscTrace(...)                         \
-  if (cdebugFlag & DEBUG_TRACE) {             \
+  if (cDebugFlag & DEBUG_TRACE) {             \
-    taosPrintLog("TSC ", cdebugFlag, __VA_ARGS__); \
+    taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); \
   }
 #define tscPrint(...) \
   { taosPrintLog("TSC ", 255, __VA_ARGS__); }
 #define tscDump(...)                                      \
-  if (cdebugFlag & DEBUG_TRACE) {                         \
+  if (cDebugFlag & DEBUG_TRACE) {                         \
-    taosPrintLongString("TSC ", cdebugFlag, __VA_ARGS__); \
+    taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); \
   }
 
 #ifdef __cplusplus

src/client/inc/tscUtil.h

@@ -31,11 +31,13 @@ extern "C" {
 #include "tscSecondaryMerge.h"
 #include "tsclient.h"
 
-#define UTIL_TABLE_IS_SUPERTABLE(metaInfo) \
+#define UTIL_TABLE_IS_SUPERTABLE(metaInfo)  \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
-#define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo) (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo)))
 #define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
+  
+#define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo)\
+  (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
 
 #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0)
 
@@ -203,7 +205,6 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
 
 void tscSetFreeHeatBeat(STscObj* pObj);
 bool tscShouldFreeHeatBeat(SSqlObj* pHb);
-void tscCleanSqlCmd(SSqlCmd* pCmd);
 bool tscShouldBeFreed(SSqlObj* pSql);
 
 STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex);
@@ -220,7 +221,6 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
 STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo);
 int32_t tscAddSubqueryInfo(SSqlCmd *pCmd);
 
-void tscFreeQueryInfo(SSqlCmd* pCmd);
 void tscInitQueryInfo(SQueryInfo* pQueryInfo);
 
 void tscClearSubqueryInfo(SSqlCmd* pCmd);

src/client/inc/tsclient.h

@@ -88,7 +88,7 @@ typedef struct SSqlExpr {
   int16_t   functionId;     // function id in aAgg array
   int16_t   resType;        // return value type
   int16_t   resBytes;       // length of return value
-  int16_t   interResBytes;  // inter result buffer size
+  int16_t   interBytes;  // inter result buffer size
   int16_t   numOfParams;    // argument value of each function
   tVariant  param[3];       // parameters are not more than 3
   int32_t   offset;         // sub result column value of arithmetic expression.
@@ -195,7 +195,7 @@ typedef struct SDataBlockList {  // todo remove
 
 typedef struct SQueryInfo {
   int16_t  command;  // the command may be different for each subclause, so keep it seperately.
-  uint16_t type;     // query/insert/import type
+  uint32_t type;     // query/insert/import type
   char     slidingTimeUnit;
 
   STimeWindow     window;
@@ -283,6 +283,8 @@ typedef struct {
   int32_t*              length;  // length for each field for current row
   char **               buffer;  // Buffer used to put multibytes encoded using unicode (wchar_t)
   SColumnIndex *        pColumnIndex;
+  SArithmeticSupport*   pArithSup;   // support the arithmetic expression calculation on agg functions
+  
   struct SLocalReducer *pLocalReducer;
 } SSqlRes;
 
@@ -298,11 +300,11 @@ typedef struct STscObj {
   char               sversion[TSDB_VERSION_LEN];
   char               writeAuth : 1;
   char               superAuth : 1;
-  void*              pMgmtConn;
   struct SSqlObj *   pSql;
   struct SSqlObj *   pHb;
   struct SSqlObj *   sqlList;
   struct SSqlStream *streamList;
+  void*              pDnodeConn;
   pthread_mutex_t    mutex;
 } STscObj;
 
@@ -358,7 +360,7 @@ typedef struct SSqlStream {
   struct SSqlStream *prev, *next;
 } SSqlStream;
 
-int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn);
+int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn);
 void    tscInitMsgsFp();
 
 int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
@@ -424,7 +426,6 @@ void    tscQueueAsyncFreeResult(SSqlObj *pSql);
 int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo);
 void    tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column);
 
-extern void *    pVnodeConn;
 extern void *    tscCacheHandle;
 extern void *    tscTmr;
 extern void *    tscQhandle;

src/client/src/TSDBJNIConnector.c

@@ -23,16 +23,16 @@
 #include "ttime.h"
 
 #define jniError(...)                                 \
-  if (jnidebugFlag & DEBUG_ERROR) {                   \
+  if (jniDebugFlag & DEBUG_ERROR) {                   \
-    taosPrintLog("ERROR JNI ", jnidebugFlag, __VA_ARGS__); \
+    taosPrintLog("ERROR JNI ", jniDebugFlag, __VA_ARGS__); \
   }
 #define jniWarn(...)                                  \
-  if (jnidebugFlag & DEBUG_WARN) {                    \
+  if (jniDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  JNI ", jnidebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN  JNI ", jniDebugFlag, __VA_ARGS__); \
   }
 #define jniTrace(...)                           \
-  if (jnidebugFlag & DEBUG_TRACE) {             \
+  if (jniDebugFlag & DEBUG_TRACE) {             \
-    taosPrintLog("JNI ", jnidebugFlag, __VA_ARGS__); \
+    taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \
   }
 #define jniPrint(...) \
   { taosPrintLog("JNI ", 255, __VA_ARGS__); }

src/client/src/tscFunctionImpl.c

@@ -153,7 +153,7 @@ typedef struct SRateInfo {
 
 
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
-                          int16_t *bytes, int16_t *interResBytes, int16_t extLength, bool isSuperTable) {
+                          int16_t *bytes, int16_t *interBytes, int16_t extLength, bool isSuperTable) {
   if (!isValidDataType(dataType, dataBytes)) {
     tscError("Illegal data type %d or data type length %d", dataType, dataBytes);
     return TSDB_CODE_INVALID_SQL;
@@ -164,35 +164,36 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
       functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
-    *interResBytes = *bytes + sizeof(SResultInfo);
+    *interBytes = *bytes + sizeof(SResultInfo);
     return TSDB_CODE_SUCCESS;
   }
   
+  // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE
   if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct
     *type = TSDB_DATA_TYPE_BINARY;
-    *bytes = dataBytes + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t); // (uid, tid) + VGID + TAGSIZE
+    *bytes = dataBytes + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE;
-    *interResBytes = *bytes;
+    *interBytes = *bytes;
     return TSDB_CODE_SUCCESS;
   }
   
   if (functionId == TSDB_FUNC_COUNT) {
     *type = TSDB_DATA_TYPE_BIGINT;
     *bytes = sizeof(int64_t);
-    *interResBytes = *bytes;
+    *interBytes = *bytes;
     return TSDB_CODE_SUCCESS;
   }
   
   if (functionId == TSDB_FUNC_ARITHM) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interResBytes = *bytes;
+    *interBytes = *bytes;
     return TSDB_CODE_SUCCESS;
   }
   
   if (functionId == TSDB_FUNC_TS_COMP) {
     *type = TSDB_DATA_TYPE_BINARY;
     *bytes = sizeof(int32_t);  // this results is compressed ts data
-    *interResBytes = POINTER_BYTES;
+    *interBytes = POINTER_BYTES;
     return TSDB_CODE_SUCCESS;
   }
   
@@ -200,54 +201,54 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
     if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = dataBytes + DATA_SET_FLAG_SIZE;
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       
       return TSDB_CODE_SUCCESS;
     } else if (functionId == TSDB_FUNC_SUM) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = sizeof(SSumInfo);
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       
       return TSDB_CODE_SUCCESS;
     } else if (functionId == TSDB_FUNC_AVG) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = sizeof(SAvgInfo);
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       return TSDB_CODE_SUCCESS;
       
     } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) {
       *type = TSDB_DATA_TYPE_DOUBLE;
       *bytes = sizeof(SRateInfo);
-      *interResBytes = sizeof(SRateInfo);
+      *interBytes = sizeof(SRateInfo);
       return TSDB_CODE_SUCCESS;
     } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param;
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       
       return TSDB_CODE_SUCCESS;
     } else if (functionId == TSDB_FUNC_SPREAD) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = sizeof(SSpreadInfo);
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       
       return TSDB_CODE_SUCCESS;
     } else if (functionId == TSDB_FUNC_APERCT) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo);
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       
       return TSDB_CODE_SUCCESS;
     } else if (functionId == TSDB_FUNC_LAST_ROW) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = sizeof(SLastrowInfo) + dataBytes;
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       
       return TSDB_CODE_SUCCESS;
     } else if (functionId == TSDB_FUNC_TWA) {
       *type = TSDB_DATA_TYPE_DOUBLE;
       *bytes = sizeof(STwaInfo);
-      *interResBytes = *bytes;
+      *interBytes = *bytes;
       return TSDB_CODE_SUCCESS;
     }
   }
@@ -260,57 +261,57 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
     }
     
     *bytes = sizeof(int64_t);
-    *interResBytes = sizeof(SSumInfo);
+    *interBytes = sizeof(SSumInfo);
     return TSDB_CODE_SUCCESS;
   } else if (functionId == TSDB_FUNC_APERCT) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interResBytes =
+    *interBytes =
         sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1);
     return TSDB_CODE_SUCCESS;
   } else if (functionId == TSDB_FUNC_TWA) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interResBytes = sizeof(STwaInfo);
+    *interBytes = sizeof(STwaInfo);
     return TSDB_CODE_SUCCESS;
   }
   
   if (functionId == TSDB_FUNC_AVG) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interResBytes = sizeof(SAvgInfo);
+    *interBytes = sizeof(SAvgInfo);
   } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interResBytes = sizeof(SRateInfo);
+    *interBytes = sizeof(SRateInfo);
   } else if (functionId == TSDB_FUNC_STDDEV) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interResBytes = sizeof(SStddevInfo);
+    *interBytes = sizeof(SStddevInfo);
   } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
-    *interResBytes = dataBytes + DATA_SET_FLAG_SIZE;
+    *interBytes = dataBytes + DATA_SET_FLAG_SIZE;
   } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
-    *interResBytes = dataBytes + sizeof(SResultInfo);
+    *interBytes = dataBytes + sizeof(SResultInfo);
   } else if (functionId == TSDB_FUNC_SPREAD) {
     *type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interResBytes = sizeof(SSpreadInfo);
+    *interBytes = sizeof(SSpreadInfo);
   } else if (functionId == TSDB_FUNC_PERCT) {
     *type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
     *bytes = (int16_t)sizeof(double);
-    *interResBytes = (int16_t)sizeof(double);
+    *interBytes = (int16_t)sizeof(double);
   } else if (functionId == TSDB_FUNC_LEASTSQR) {
     *type = TSDB_DATA_TYPE_BINARY;
     *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE;  // string
-    *interResBytes = *bytes + sizeof(SResultInfo);
+    *interBytes = *bytes + sizeof(SResultInfo);
   } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) {
     *type = TSDB_DATA_TYPE_BINARY;
     *bytes = dataBytes + sizeof(SFirstLastInfo);
-    *interResBytes = *bytes;
+    *interBytes = *bytes;
   } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
@@ -318,11 +319,11 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
     size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param;
     
     // the output column may be larger than sizeof(STopBotInfo)
-    *interResBytes = size;
+    *interBytes = size;
   } else if (functionId == TSDB_FUNC_LAST_ROW) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
-    *interResBytes = dataBytes + sizeof(SLastrowInfo);
+    *interBytes = dataBytes + sizeof(SLastrowInfo);
   } else {
     return TSDB_CODE_INVALID_SQL;
   }
@@ -5285,10 +5286,10 @@ SQLAggFuncElem aAggs[] = {{
                           },
                           {
                               // 34
-                              "tid_tag",   // return table id and the corresponding tags for join match
+                              "tid_tag",   // return table id and the corresponding tags for join match and subscribe
                               TSDB_FUNC_TID_TAG,
                               TSDB_FUNC_TID_TAG,
-                              TSDB_FUNCSTATE_MO,
+                              TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE,
                               function_setup,
                               noop1,
                               noop2,

src/client/src/tscLocal.c

@@ -156,7 +156,8 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3);
     if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) {
       char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
-      STR_WITH_SIZE_TO_VARSTR(output, "TAG", 3);
+      const char *src = "TAG";
+      STR_WITH_SIZE_TO_VARSTR(output, src, strlen(src));
     }
   }
 
@@ -191,7 +192,8 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     // tag value
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3);
     char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_SIZE_TO_VARSTR(target, "TAG", 3);
+    const char *src = "TAG";
+    STR_WITH_SIZE_TO_VARSTR(target, src, strlen(src));
 
     pTagValue += pSchema[i].bytes;
   }

src/client/src/tscParseInsert.c

@@ -305,10 +305,8 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
     case TSDB_DATA_TYPE_BINARY:
       // binary data cannot be null-terminated char string, otherwise the last char of the string is lost
       if (pToken->type == TK_NULL) {
-        *(int16_t*) payload = sizeof(int8_t);
+        varDataSetLen(payload, sizeof(int8_t));
-        payload += VARSTR_HEADER_SIZE;
+        *(uint8_t*) varDataVal(payload) = TSDB_DATA_BINARY_NULL;
-        
-        *payload = TSDB_DATA_BINARY_NULL;
       } else { // too long values will return invalid sql, not be truncated automatically
         if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { //todo refactor
           return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z);
@@ -321,21 +319,18 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
 
     case TSDB_DATA_TYPE_NCHAR:
       if (pToken->type == TK_NULL) {
-        *(int16_t*) payload = sizeof(int32_t);
+        varDataSetLen(payload, sizeof(int32_t));
-        payload += VARSTR_HEADER_SIZE;
+        *(uint32_t*) varDataVal(payload) = TSDB_DATA_NCHAR_NULL;
-  
-        *(uint32_t*) payload = TSDB_DATA_NCHAR_NULL;
       } else {
         // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
-        int32_t resLen = -1;
+        size_t output = 0;
-        if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes - VARSTR_HEADER_SIZE, &resLen)) {
+        if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
           char buf[512] = {0};
-          snprintf(buf, 512, "%s", strerror(errno));
+          snprintf(buf, tListLen(buf), "%s", strerror(errno));
-          
           return tscInvalidSQLErrMsg(msg, buf, pToken->z);
         }
         
-        *(uint16_t*)payload = (uint16_t) (resLen * TSDB_NCHAR_SIZE);
+        varDataSetLen(payload, output);
       }
       break;
 
@@ -479,10 +474,19 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
     char *ptr = payload;
 
     for (int32_t i = 0; i < spd->numOfCols; ++i) {
+      
       if (!spd->hasVal[i]) {  // current column do not have any value to insert, set it to null
-        setNull(ptr, schema[i].type, schema[i].bytes);
+        if (schema[i].type == TSDB_DATA_TYPE_BINARY) {
+          varDataSetLen(ptr, sizeof(int8_t));
+          *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL;
+        } else if (schema[i].type == TSDB_DATA_TYPE_NCHAR) {
+          varDataSetLen(ptr, sizeof(int32_t));
+          *(uint32_t*) varDataVal(ptr) = TSDB_DATA_NCHAR_NULL;
+        } else {
+          setNull(ptr, schema[i].type, schema[i].bytes);
+        }
       }
-
+      
       ptr += schema[i].bytes;
     }
 
@@ -960,6 +964,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     *sqlstr = sql;
   }
 
+  if (*sqlstr == NULL) {
+    code = TSDB_CODE_INVALID_SQL;
+  }
+  
   return code;
 }
 
@@ -1283,6 +1291,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
 
   pCmd->count = 0;
   pCmd->command = TSDB_SQL_INSERT;
+  pSql->res.numOfRows = 0;
 
   SQueryInfo *pQueryInfo = NULL;
   tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
@@ -1295,7 +1304,6 @@ int tsParseInsertSql(SSqlObj *pSql) {
     return tscInvalidSQLErrMsg(pCmd->payload, "keyword INTO is expected", sToken.z);
   }
 
-  pSql->res.numOfRows = 0;
   return doParseInsertSql(pSql, pSql->sqlstr + index);
 }
 

src/client/src/tscSQLParser.c

@@ -370,10 +370,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
-      char* pMsg = pCmd->payload + tsRpcHeadSize;
+      char* pMsg = pCmd->payload;
-      pMsg += sizeof(SMgmtHead);
 
       SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg;
+      pDCL->a[0].n = strdequote(pDCL->a[0].z);
+      
       strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n);
 
       strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n);
@@ -1117,13 +1118,13 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
 
       // if the name of column is quoted, remove it and set the right information for later process
       extractColumnNameFromString(pItem);
-      pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
+      TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY);
 
       // select table_name1.field_name1, table_name2.field_name2  from table_name1, table_name2
       if (addProjectionExprAndResultField(pQueryInfo, pItem) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_INVALID_SQL;
       }
-    } else if (pItem->pNode->nSQLOptr >= TK_COUNT && pItem->pNode->nSQLOptr <= TK_AVG_IRATE) {
+    } else if (pItem->pNode->nSQLOptr >= TK_COUNT && pItem->pNode->nSQLOptr <= TK_TBID) {
       // sql function in selection clause, append sql function info in pSqlCmd structure sequentially
       if (addExprAndResultField(pQueryInfo, outputIndex, pItem, true) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_INVALID_SQL;
@@ -1198,48 +1199,26 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
         columnList.num = 0;
         columnList.ids[0] = (SColumnIndex) {0, 0};
         
-        insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, "abc", NULL);
+        insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, "dummy_column", NULL);
         
         int32_t slot = tscNumOfFields(pQueryInfo) - 1;
         SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot);
         
         if (pInfo->pSqlExpr == NULL) {
-          SExprInfo* pFuncExpr = calloc(1, sizeof(SExprInfo));
+          SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo));
-          pInfo->pArithExprInfo = pFuncExpr;
           
           // arithmetic expression always return result in the format of double float
-          pFuncExpr->bytes = sizeof(double);
+          pArithExprInfo->bytes = sizeof(double);
-          pFuncExpr->interResBytes = sizeof(double);
+          pArithExprInfo->interBytes = sizeof(double);
-          pFuncExpr->type = TSDB_DATA_TYPE_DOUBLE;
+          pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE;
 
-          tExprNode* pNode = NULL;
+          int32_t ret = exprTreeFromSqlExpr(&pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL);
-//          SArray* colList = taosArrayInit(10, sizeof(SColIndex));
-  
-          int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL);
           if (ret != TSDB_CODE_SUCCESS) {
-            tExprTreeDestroy(&pNode, NULL);
+            tExprTreeDestroy(&pArithExprInfo->pExpr, NULL);
             return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause");
           }
   
-          pFuncExpr->pExpr = pNode;
+          pInfo->pArithExprInfo = pArithExprInfo;
-          assert(0);
-//          pExprInfo->pReqColumns = pColIndex;
-          
-//          for(int32_t k = 0; k < pFuncExpr->numOfCols; ++k) {
-//            SColIndex* pCol = &pFuncExpr->colList[k];
-//            size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-//
-//            for(int32_t f = 0; f < size; ++f) {
-//              SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f);
-//              if (strcmp(pExpr->aliasName, pCol->name) == 0) {
-//                pCol->colIndex = f;
-//                break;
-//              }
-//            }
-//
-//            assert(pCol->colIndex >= 0 && pCol->colIndex < size);
-//            tfree(pNode);
-//          }
         }
       }
     } else {
@@ -1260,7 +1239,6 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
   }
 
   if (isSTable) {
-    pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_QUERY;
     /*
      * transfer sql functions that need secondary merge into another format
      * in dealing with metric queries such as: count/first/last
@@ -1311,10 +1289,8 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c
     index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta);
   
     tscColumnListInsert(pTableMetaInfo->tagColList, &index);
-    pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY;
   } else {
     index.columnIndex = colIndex;
-    pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
   }
   
   return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes,
@@ -1399,7 +1375,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
 
 int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pItem) {
   const char* msg0 = "invalid column name";
-  const char* msg1 = "tag for table query is not allowed";
+  const char* msg1 = "tag for normal table query is not allowed";
   
   int32_t startPos = tscSqlExprNumOfExprs(pQueryInfo);
 
@@ -1493,7 +1469,8 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
 
 int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) {
   STableMetaInfo* pTableMetaInfo = NULL;
-  int32_t         optr = pItem->pNode->nSQLOptr;
+  
+  int32_t optr = pItem->pNode->nSQLOptr;
 
   const char* msg1 = "not support column types";
   const char* msg2 = "invalid parameters";
@@ -1501,7 +1478,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
   const char* msg4 = "invalid table name";
   const char* msg5 = "parameter is out of range [0, 100]";
   const char* msg6 = "function applied to tags not allowed";
-
+  const char* msg7 = "normal table can not apply this function";
+  
   switch (optr) {
     case TK_COUNT: {
       if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 1) {
@@ -1883,13 +1861,68 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
         }
       }
 
+      return TSDB_CODE_SUCCESS;
+    };
+    
+    case TK_TBID: {
+      pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+      if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg7);
+      }
+    
+      // no parameters or more than one parameter for function
+      if (pItem->pNode->pParam == NULL || pItem->pNode->pParam->nExpr != 1) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg2);
+      }
+      
+      tSQLExpr* pParam = pItem->pNode->pParam->a[0].pNode;
+    
+      SColumnIndex index = COLUMN_INDEX_INITIALIZER;
+      if (getColumnIndexByName(&pParam->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg3);
+      }
+    
+      pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
+      SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
+  
+      // functions can not be applied to normal columns
+      int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
+      if (index.columnIndex < numOfCols) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg6);
+      }
+    
+      index.columnIndex -= numOfCols;
+      
+      // 2. valid the column type
+      int16_t colType = pSchema[index.columnIndex].type;
+      if (colType == TSDB_DATA_TYPE_BOOL || colType >= TSDB_DATA_TYPE_BINARY) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg1);
+      }
+
+      tscColumnListInsert(pTableMetaInfo->tagColList, &index);
+      SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
+      SSchema s = pTagSchema[index.columnIndex];
+
+      int16_t bytes = 0;
+      int16_t type  = 0;
+      int16_t inter = 0;
+
+      int32_t ret = getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
+      assert(ret == TSDB_CODE_SUCCESS);
+      
+      s.type  = type;
+      s.bytes = bytes;
+
+      TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
+      tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG);
+      
       return TSDB_CODE_SUCCESS;
     }
+    
     default:
       return TSDB_CODE_INVALID_SQL;
   }
   
-  
 }
 
 // todo refactor
@@ -2222,14 +2255,14 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
   char* portStr = strtok(NULL, &delim);
 
   if (!validateIpAddress(ipStr, strlen(ipStr))) {
-    memset(pCmd->payload, 0, tListLen(pCmd->payload));
+    memset(pCmd->payload, 0, strlen(pCmd->payload));
 
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
 
   uint16_t port = (uint16_t)strtol(portStr, NULL, 10);
   if (port <= 0 || port > 65535) {
-    memset(pCmd->payload, 0, tListLen(pCmd->payload));
+    memset(pCmd->payload, 0, strlen(pCmd->payload));
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
   }
 
@@ -2278,7 +2311,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
 
       tscSqlExprUpdate(pQueryInfo, k, functionId, pExpr->colInfo.colIndex, TSDB_DATA_TYPE_BINARY, bytes);
       // todo refactor
-      pExpr->interResBytes = intermediateBytes;
+      pExpr->interBytes = intermediateBytes;
     }
   }
 
@@ -4765,7 +4798,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo*
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
+static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
   const char* msg = "invalid time precision";
 
   pMsg->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond by default
@@ -4790,15 +4823,15 @@ static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCrea
 }
 
 static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
-  pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode);
+  pMsg->maxTables = htonl(pCreateDb->maxTablesPerVnode);
-  pMsg->cacheBlockSize = htonl(-1);
+  pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
-  pMsg->totalBlocks = htonl(-1);
+  pMsg->numOfBlocks = htonl(pCreateDb->numOfBlocks);
   pMsg->daysPerFile = htonl(pCreateDb->daysPerFile);
   pMsg->commitTime = htonl(pCreateDb->commitTime);
-  pMsg->minRowsPerFileBlock = htonl(-1);
+  pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock);
-  pMsg->maxRowsPerFileBlock = htonl(-1);
+  pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock);
   pMsg->compression = pCreateDb->compressionLevel;
-  pMsg->commitLog = (char)pCreateDb->commitLog;
+  pMsg->walLevel = (char)pCreateDb->walLevel;
   pMsg->replications = pCreateDb->replica;
   pMsg->ignoreExist = pCreateDb->ignoreExists;
 }
@@ -4811,7 +4844,7 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
     return TSDB_CODE_INVALID_SQL;
   }
 
-  if (setTimePrecisionOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) {
+  if (setTimePrecision(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_INVALID_SQL;
   }
 
@@ -4937,7 +4970,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) {
     if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) {
       SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex];
       getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, pExpr->param[0].i64Key, &pExpr->resType,
-                        &pExpr->resBytes, &pExpr->interResBytes, tagLength, true);
+                        &pExpr->resBytes, &pExpr->interBytes, tagLength, true);
     }
   }
 }
@@ -5331,8 +5364,8 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
 int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
   char msg[512] = {0};
 
-  if (pCreate->commitLog != -1 && (pCreate->commitLog < TSDB_MIN_CLOG_LEVEL || pCreate->commitLog > TSDB_MAX_CLOG_LEVEL)) {
+  if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) {
-    snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog);
+    snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 0-2 allowed", pCreate->walLevel);
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
 
@@ -5357,7 +5390,7 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
 
-  val = htonl(pCreate->maxSessions);
+  val = htonl(pCreate->maxTables);
   if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) {
     snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val,
              TSDB_MIN_TABLES, TSDB_MAX_TABLES);
@@ -5738,12 +5771,16 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
   }
 
   assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr);
+  bool isSTable = false;
   
   if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
-   code = tscGetSTableVgroupInfo(pSql, index);
+    isSTable = true;
-   if (code != TSDB_CODE_SUCCESS) {
+    code = tscGetSTableVgroupInfo(pSql, index);
-     return code;
+    if (code != TSDB_CODE_SUCCESS) {
-   }
+      return code;
+    }
+    
+    TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_QUERY);
   } else {
     TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY);
   }
@@ -5753,7 +5790,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     return TSDB_CODE_INVALID_SQL;
   }
 
-  bool isSTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo);
   if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_INVALID_SQL;
   }

src/client/src/tscSecondaryMerge.c

@@ -101,7 +101,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu
     }
 
     SResultInfo *pResInfo = &pReducer->pResInfo[i];
-    pResInfo->bufLen = pExpr->interResBytes;
+    pResInfo->bufLen = pExpr->interBytes;
     pResInfo->interResultBuf = calloc(1, (size_t)pResInfo->bufLen);
 
     pCtx->resultInfo = &pReducer->pResInfo[i];

src/client/src/tscServer.c

@@ -201,30 +201,22 @@ int tscSendMsgToServer(SSqlObj *pSql) {
   }
 
   if (pSql->cmd.command < TSDB_SQL_MGMT) {
-    tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port[0]);
     memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen);
+  } else {
+    pSql->ipList = tscMgmtIpSet;
+    memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
+  }
 
-    SRpcMsg rpcMsg = {
+  tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
+
+  SRpcMsg rpcMsg = {
       .msgType = pSql->cmd.msgType,
       .pCont   = pMsg,
       .contLen = pSql->cmd.payloadLen,
       .handle  = pSql,
       .code    = 0
-    };
+  };
-    rpcSendRequest(pVnodeConn, &pSql->ipList, &rpcMsg);
+  rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
-  } else {
-    pSql->ipList = tscMgmtIpSet;
-    memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
-    SRpcMsg rpcMsg = {
-        .msgType = pSql->cmd.msgType,
-        .pCont   = pMsg,
-        .contLen = pSql->cmd.payloadLen,
-        .handle  = pSql,
-        .code   = 0
-    };
-    tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
-    rpcSendRequest(pObj->pMgmtConn, &pSql->ipList, &rpcMsg);
-  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -439,25 +431,6 @@ int tscProcessSql(SSqlObj *pSql) {
   } else {  // local handler
     return (*tscProcessMsgRsp[pCmd->command])(pSql);
   }
-
-//  if (QUERY_IS_JOIN_QUERY(type)) {
-//    if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0) {
-//      return tscHandleMasterJoinQuery(pSql);
-//    } else {
-//      // for first stage sub query, iterate all vnodes to get all timestamp
-//      if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) {
-//        return doProcessSql(pSql);
-//      }
-//    }
-//  }
-//
-//  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query
-//    tscHandleMasterSTableQuery(pSql);
-//    return pRes->code;
-//  } else if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) {  // multi-vnodes insertion
-//    tscHandleMultivnodeInsert(pSql);
-//    return pRes->code;
-//  }
   
   return doProcessSql(pSql);
 }
@@ -504,7 +477,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
   tscTrace("%p super table query cancelled", pSql);
 }
 
-int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
+int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   char *pMsg, *pStart;
 
   pStart = pSql->cmd.payload + tsRpcHeadSize;
@@ -533,7 +506,7 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   
   pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen);
   
-  pSql->cmd.msgType = TSDB_MSG_TYPE_RETRIEVE;
+  pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -599,14 +572,14 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
   if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
     
     SCMVgroupInfo* pVgroupInfo = NULL;
-    if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
-      pVgroupInfo = &pTableMeta->vgroupInfo;
-    } else {
       int32_t index = pTableMetaInfo->vgroupIndex;
       assert(index >= 0);
-      
+  
       pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index];
       tscTrace("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups);
+    } else {
+      pVgroupInfo = &pTableMeta->vgroupInfo;
     }
     
     tscSetDnodeIpList(pSql, pVgroupInfo);
@@ -1032,13 +1005,7 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SCMCfgDnodeMsg);
-
+  pCmd->msgType = TSDB_MSG_TYPE_CM_CONFIG_DNODE;
-  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
-    tscError("%p failed to malloc for query msg", pSql);
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
-  }
-
-  pCmd->msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1216,7 +1183,7 @@ int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &(pSql->cmd);
 
-  int32_t size = minMsgSize() + sizeof(SMgmtHead) + sizeof(SCMCreateTableMsg);
+  int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg);
 
   SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo;
   if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) {
@@ -1304,7 +1271,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
 int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) {
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  return minMsgSize() + sizeof(SMgmtHead) + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) +
+  return minMsgSize() + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) +
          TSDB_EXTRA_PAYLOAD_SIZE;
 }
 
@@ -1372,7 +1339,7 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
 int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE;
+  pCmd->msgType = TSDB_MSG_TYPE_CM_RETRIEVE;
   pCmd->payloadLen = sizeof(SRetrieveTableMsg);
 
   if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@@ -1561,6 +1528,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  *      no used         4B
  **/
 int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
+#if 0
   SSqlCmd *pCmd = &pSql->cmd;
 
   // copy payload content to temp buff
@@ -1593,6 +1561,8 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
            pCmd->payloadLen);
 
   return pCmd->payloadLen;
+#endif
+  return 0;  
 }
 
 //static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) {
@@ -1792,7 +1762,7 @@ int tscEstimateHeartBeatMsgLength(SSqlObj *pSql) {
   int      size = 0;
   STscObj *pObj = pSql->pTscObj;
 
-  size += tsRpcHeadSize + sizeof(SMgmtHead);
+  size += tsRpcHeadSize;
   size += sizeof(SQqueryList);
 
   SSqlObj *tpSql = pObj->sqlList;
@@ -1828,13 +1798,9 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     return -1;
   }
 
-  pMsg = pCmd->payload + tsRpcHeadSize;
+  pMsg = pCmd->payload;
   pStart = pMsg;
 
-  SMgmtHead *pMgmt = (SMgmtHead *)pMsg;
-  strcpy(pMgmt->db, pObj->db);
-  pMsg += sizeof(SMgmtHead);
-
   pMsg = tscBuildQueryStreamDesc(pMsg, pObj);
   pthread_mutex_unlock(&pObj->mutex);
 
@@ -1907,7 +1873,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   assert(pTableMetaInfo->pTableMeta == NULL);
 
   pTableMetaInfo->pTableMeta =
-      (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, pTableMeta, size, tsMeterMetaKeepTimer);
+      (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, pTableMeta, size, tsTableMetaKeepTimer);
   
   // todo handle out of memory case
   if (pTableMetaInfo->pTableMeta == NULL) {
@@ -2016,7 +1982,7 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
     //    int32_t size = (int32_t)(rsp - ((char *)pMeta));  // Consistent with STableMeta in cache
     //
     //    pMeta->index = 0;
-    //    (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsMeterMetaKeepTimer);
+    //    (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer);
     //  }
   }
   
@@ -2215,7 +2181,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
   size_t size = 0;
   STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
   
-  pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer);
+  pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsTableMetaKeepTimer);
   SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
 
   if (pQueryInfo->colList == NULL) {
@@ -2363,8 +2329,10 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) {
   pRes->data      = pRetrieve->data;
   
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  tscSetResultPointer(pQueryInfo, pRes);
+  if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) {
-
+    return pRes->code;
+  }
+  
   if (pSql->pSubscription != NULL) {
     int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput;
     
@@ -2436,7 +2404,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
 
   strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name));
   memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE);  // tag information if table does not exists.
-  tscTrace("%p new pSqlObj:%p to get tableMeta", pSql, pNew);
+  tscTrace("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
 
   pNew->fp = tscTableMetaCallBack;
   pNew->param = pSql;
@@ -2586,7 +2554,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
 void tscInitMsgsFp() {
   tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg;
   tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg;
-  tscBuildMsg[TSDB_SQL_FETCH] = tscBuildRetrieveMsg;
+  tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg;
 
   tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg;
   tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg;

src/client/src/tscSql.c

@@ -65,9 +65,9 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     terrno = TSDB_CODE_INVALID_PASS;
     return NULL;
   }
-
+  
-  void* pMgmtConn = NULL;
+  void *pDnodeConn = NULL;
-  if (tscInitRpc(user, pass, &pMgmtConn) != 0) {
+  if (tscInitRpc(user, pass, &pDnodeConn) != 0) {
     terrno = TSDB_CODE_NETWORK_UNAVAIL;
     return NULL;
   }
@@ -78,7 +78,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     tscMgmtIpSet.inUse = 0;
     tscMgmtIpSet.numOfIps = 1;
     strcpy(tscMgmtIpSet.fqdn[0], ip);
-    tscMgmtIpSet.port[0] = port? port: tsMnodeShellPort;
+    tscMgmtIpSet.port[0] = port? port: tsDnodeShellPort;
   } else {
     if (tsFirst[0] != 0) {
       taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
@@ -94,6 +94,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
   STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
   if (NULL == pObj) {
     terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    rpcClose(pDnodeConn);
     return NULL;
   }
 
@@ -101,14 +102,15 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   strncpy(pObj->user, user, TSDB_USER_LEN);
   taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass);
-  pObj->mgmtPort = port ? port : tsMnodeShellPort;
+  pObj->mgmtPort = port ? port : tsDnodeShellPort;
 
   if (db) {
     int32_t len = strlen(db);
     /* db name is too long */
     if (len > TSDB_DB_NAME_LEN) {
-      free(pObj);
       terrno = TSDB_CODE_INVALID_DB;
+      rpcClose(pDnodeConn);
+      free(pObj);
       return NULL;
     }
 
@@ -119,12 +121,12 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     strtolower(pObj->db, tmp);
   }
 
-  pObj->pMgmtConn = pMgmtConn;
   pthread_mutex_init(&pObj->mutex, NULL);
 
   SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
   if (NULL == pSql) {
     terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    rpcClose(pDnodeConn);
     free(pObj);
     return NULL;
   }
@@ -136,6 +138,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
   tsem_init(&pSql->rspSem, 0, 0);
   
   pObj->pSql = pSql;
+  pObj->pDnodeConn = pDnodeConn;
+  
   pSql->fp = fp;
   pSql->param = param;
   if (taos != NULL) {
@@ -145,6 +149,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
   pSql->cmd.command = TSDB_SQL_CONNECT;
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    rpcClose(pDnodeConn);
     free(pSql);
     free(pObj);
     return NULL;
@@ -167,14 +172,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) {
 }
 
 TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
-  tscTrace("try to create a connection to %s", ip);
+  tscTrace("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db);
-  if (port != 0) {
-    tsServerPort = port;
-    tsMnodeShellPort = tsServerPort + TSDB_PORT_MNODESHELL;
-    tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL;
-    tsMnodeDnodePort = tsServerPort + TSDB_PORT_MNODEDNODE;
-    tsDnodeMnodePort = tsServerPort + TSDB_PORT_DNODEMNODE;
-  }
 
   STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL);
   if (pObj != NULL) {
@@ -193,7 +191,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
       return NULL;
     }
     
-    tscTrace("%p DB connection is opening", pObj);
+    tscTrace("%p DB connection is opening, dnodeConn:%p", pObj, pObj->pDnodeConn);
     
     // version compare only requires the first 3 segments of the version string
     int code = taosCheckVersion(version, taos_get_server_info(pObj), 3);
@@ -440,23 +438,6 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
   return (pQueryInfo->order.order == TSDB_ORDER_DESC) ? pRes->numOfRows : -pRes->numOfRows;
 }
 
-static UNUSED_FUNC char *getArithemicInputSrc(void *param, const char *name, int32_t colId) {
-//  SArithmeticSupport *pSupport = (SArithmeticSupport *)param;
-//  SExprInfo *  pExpr = pSupport->pArithExpr;
-
-//  int32_t index = -1;
-//  for (int32_t i = 0; i < pExpr->numOfCols; ++i) {
-//    if (strcmp(name, pExpr->colList[i].name) == 0) {
-//      index = i;
-//      break;
-//    }
-//  }
-//
-//  assert(index >= 0 && index < pExpr->numOfCols);
-//  return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index];
-return 0;
-}
-
 static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
   SSqlObj* pSql = (SSqlObj*) tres;
   
@@ -618,42 +599,18 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) {
 
     tscTrace("%p code:%d, numOfRows:%d, command:%d", pSql, pRes->code, pRes->numOfRows, pCmd->command);
 
-    void *fp = pSql->fp;
+    pSql->freed = 1;
-    if (fp != NULL) {
-      pSql->freed = 1;
-    }
-
     tscProcessSql(pSql);
 
     /*
      *  If release connection msg is sent to vnode, the corresponding SqlObj for async query can not be freed instantly,
      *  since its free operation is delegated to callback function, which is tscProcessMsgFromServer.
      */
-    if (fp == NULL) {
+    STscObj* pObj = pSql->pTscObj;
-      /*
+    if (pObj->pSql == pSql) {
-       * fp may be released here, so we cannot use the pSql->fp
+      pObj->pSql = NULL;
-       *
-       * In case of handle sync model query, the main SqlObj cannot be freed.
-       * So, we only free part attributes, including allocated resources and references on metermeta/metricmeta
-       * data in cache.
-       *
-       * Then this object will be reused and no free operation is required.
-       */
-      if (keepCmd) {
-        tscFreeSqlResult(pSql);
-        tscTrace("%p sql result is freed by app while sql command is kept", pSql);
-      } else {
-        tscPartiallyFreeSqlObj(pSql);
-        tscTrace("%p sql result is freed by app", pSql);
-      }
-    } else {  // for async release, remove its link
-      STscObj* pObj = pSql->pTscObj;
-      if (pObj->pSql == pSql) {
-        pObj->pSql = NULL;
-      }
     }
-  } else {
+  } else { // if no free resource msg is sent to vnode, we free this object immediately.
-    // if no free resource msg is sent to vnode, we free this object immediately.
     STscObj* pTscObj = pSql->pTscObj;
     
     if (pTscObj->pSql != pSql) {
@@ -885,7 +842,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
 
 static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) {
   // must before clean the sqlcmd object
-  tscCleanSqlCmd(&pSql->cmd);
+  tscResetSqlCmdObj(&pSql->cmd);
 
   SSqlCmd *pCmd = &pSql->cmd;
 

src/client/src/tscStream.c

@@ -508,7 +508,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
   SSqlInfo SQLInfo = {0};
   tSQLParse(&SQLInfo, pSql->sqlstr);
 
-  tscCleanSqlCmd(&pSql->cmd);
+  tscResetSqlCmdObj(&pSql->cmd);
   ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE);
   if (TSDB_CODE_SUCCESS != ret) {
     setErrorInfo(pObj, ret, NULL);

src/client/src/tscSub.c

@@ -374,7 +374,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
     } else {
       SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
       
-      uint16_t type = pQueryInfo->type;
+      uint32_t type = pQueryInfo->type;
       taos_free_result_imp(pSql, 1);
       pRes->numOfRows = 1;
       pRes->numOfTotal = 0;

src/client/src/tscSubquery.c

@@ -14,6 +14,7 @@
  */
 
 #include "tscSubquery.h"
+#include <qast.h>
 #include <tcompare.h>
 #include <tschemautil.h>
 #include "os.h"
@@ -411,10 +412,6 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et)
 
 static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj* pSql) {
   SSqlObj* pParentSql = pSupporter->pObj;
-//  SSqlCmd* pCmd = &pSql->cmd;
-//  SSqlRes* pRes = &pSql->res;
-  
-//  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
   
 //  if (tscNonOrderedProjectionQueryOnSTable(pParentQueryInfo, 0)) {
@@ -601,21 +598,6 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
   
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   
-//  if (pSupporter->pState->code != TSDB_CODE_SUCCESS) {
-//    tscError("%p abort query due to other subquery failure. code:%d, global code:%s", pSql, numOfRows,
-//             tstrerror(pSupporter->pState->code));
-//
-//    quitAllSubquery(pParentSql, pSupporter);
-//    return;
-//  }
-//
-//  if (numOfRows < 0) {
-//    tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
-//    pSupporter->pState->code = numOfRows;
-//    quitAllSubquery(pParentSql, pSupporter);
-//    return;
-//  }
-  
   // response of tag retrieve
   if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
     if (numOfRows == 0 || pSql->res.completed) {
@@ -1454,7 +1436,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
       return;
     } else {  // reach the maximum retry count, abort
       atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows);
-      tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql,
+      tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql,
                numOfRows, subqueryIndex, tstrerror(pState->code));
     }
   }
@@ -1924,7 +1906,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
 static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) {
   SSqlRes *pRes = &pSql->res;
   
-  if (isNull(pRes->tsrow[columnIndex], pField->type)) {
+  if (pRes->tsrow[columnIndex] != NULL && isNull(pRes->tsrow[columnIndex], pField->type)) {
     pRes->tsrow[columnIndex] = NULL;
   } else if (pField->type == TSDB_DATA_TYPE_NCHAR) {
     // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
@@ -1944,6 +1926,24 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
   }
 }
 
+static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) {
+  SArithmeticSupport *pSupport = (SArithmeticSupport *) param;
+
+  int32_t index = -1;
+  SSqlExpr* pExpr = NULL;
+  
+  for (int32_t i = 0; i < pSupport->numOfCols; ++i) {
+    pExpr = taosArrayGetP(pSupport->exprList, i);
+    if (strncmp(name, pExpr->aliasName, TSDB_COL_NAME_LEN) == 0) {
+      index = i;
+      break;
+    }
+  }
+
+  assert(index >= 0 && index < pSupport->numOfCols);
+  return pSupport->data[index] + pSupport->offset * pExpr->resBytes;
+}
+
 void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
@@ -1981,28 +1981,30 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
     
     // calculate the result from several other columns
     if (pSup->pArithExprInfo != NULL) {
-//      SArithmeticSupport *sas = (SArithmeticSupport *)calloc(1, sizeof(SArithmeticSupport));
+      if (pRes->pArithSup == NULL) {
-//      sas->offset = 0;
+        SArithmeticSupport *sas = (SArithmeticSupport *) calloc(1, sizeof(SArithmeticSupport));
-//      sas-> = pQueryInfo->fieldsInfo.pExpr[i];
+        sas->offset     = 0;
-//
+        sas->pArithExpr = pSup->pArithExprInfo;
-//      sas->numOfCols = sas->pExpr->binExprInfo.numOfCols;
+        sas->numOfCols  = tscSqlExprNumOfExprs(pQueryInfo);
-//
+        sas->exprList   = pQueryInfo->exprList;
-//      if (pRes->buffer[i] == NULL) {
+        sas->data       = calloc(sas->numOfCols, POINTER_BYTES);
-//        pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes);
+        
-//      }
+        pRes->pArithSup = sas;
-//
+      }
-//      for(int32_t k = 0; k < sas->numOfCols; ++k) {
+      
-//        int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIdxInBuf;
+      if (pRes->buffer[i] == NULL) {
-//        SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex);
+        TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
-//
+        pRes->buffer[i] = malloc(field->bytes);
-//        sas->elemSize[k] = pExpr->resBytes;
+      }
-//        sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes;
+
-//      }
+      for(int32_t k = 0; k < pRes->pArithSup->numOfCols; ++k) {
-//
+        SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k);
-//      tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc);
+        pRes->pArithSup->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes;
-//      pRes->tsrow[i] = pRes->buffer[i];
+      }
-//
+  
-//      free(sas); //todo optimization
+      tExprTreeCalcTraverse(pRes->pArithSup->pArithExpr->pExpr, 1, pRes->buffer[i], pRes->pArithSup,
+          TSDB_ORDER_ASC, getArithemicInputSrc);
+      pRes->tsrow[i] = pRes->buffer[i];
     }
   }
   
@@ -2010,7 +2012,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
   return pRes->tsrow;
 }
 
-static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) {
+static UNUSED_FUNC bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) {
   bool     hasData = true;
   SSqlCmd *pCmd = &pSql->cmd;
   

src/client/src/tscSystem.c

@@ -30,7 +30,6 @@
 #include "tlocale.h"
 
 // global, not configurable
-void *  pVnodeConn;
 void *  tscCacheHandle;
 void *  tscTmr;
 void *  tscQhandle;
@@ -48,15 +47,15 @@ void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
   taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
 }
 
-int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) {
+int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) {
   SRpcInit rpcInit;
   char secretEncrypt[32] = {0};
   taosEncryptPass((uint8_t *)secret, strlen(secret), secretEncrypt);
 
-  if (pVnodeConn == NULL) {
+  if (*pDnodeConn == NULL) {
     memset(&rpcInit, 0, sizeof(rpcInit));
     rpcInit.localPort = 0;
-    rpcInit.label = "TSC-vnode";
+    rpcInit.label = "TSC";
     rpcInit.numOfThreads = tscNumOfThreads;
     rpcInit.cfp = tscProcessMsgFromServer;
     rpcInit.sessions = tsMaxVnodeConnections;
@@ -66,32 +65,12 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) {
     rpcInit.ckey = "key";
     rpcInit.secret = secretEncrypt;
 
-    pVnodeConn = rpcOpen(&rpcInit);
+    *pDnodeConn = rpcOpen(&rpcInit);
-    if (pVnodeConn == NULL) {
+    if (*pDnodeConn == NULL) {
-      tscError("failed to init connection to vnode");
+      tscError("failed to init connection to TDengine");
-      return -1;
-    }
-  }
-
-  if (*pMgmtConn == NULL) {
-    memset(&rpcInit, 0, sizeof(rpcInit));
-    rpcInit.localPort = 0;
-    rpcInit.label = "TSC-mgmt";
-    rpcInit.numOfThreads = 1;
-    rpcInit.cfp = tscProcessMsgFromServer;
-    rpcInit.ufp = tscUpdateIpSet;
-    rpcInit.sessions = tsMaxMgmtConnections;
-    rpcInit.connType = TAOS_CONN_CLIENT;
-    rpcInit.idleTime = 2000;
-    rpcInit.user = (char*)user;
-    rpcInit.ckey = "key";
-    rpcInit.spi = 1;
-    rpcInit.secret = secretEncrypt;
-
-    *pMgmtConn = rpcOpen(&rpcInit);
-    if (*pMgmtConn == NULL) {
-      tscError("failed to init connection to mgmt");
       return -1;
+    } else {
+      tscTrace("dnodeConn:%p is created, user:%s", *pDnodeConn, user);
     }
   }
 
@@ -165,7 +144,7 @@ void taos_init_imp() {
     taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr);      
   }
   
-  int64_t refreshTime = tsMetricMetaKeepTimer < tsMeterMetaKeepTimer ? tsMetricMetaKeepTimer : tsMeterMetaKeepTimer;
+  int64_t refreshTime = tsTableMetaKeepTimer;
   refreshTime = refreshTime > 2 ? 2 : refreshTime;
   refreshTime = refreshTime < 1 ? 1 : refreshTime;
 
@@ -190,11 +169,6 @@ void taos_cleanup() {
   
   taosCloseLog();
   
-  if (pVnodeConn != NULL) {
-    rpcClose(pVnodeConn);
-    pVnodeConn = NULL;
-  }
-  
   taosTmrCleanUp(tscTmr);
 }
 

src/client/src/tscUtil.c

@@ -30,7 +30,10 @@
 #include "ttokendef.h"
 #include "tscLog.h"
 
-SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
+static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
+static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache);
+  
+  SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
   if (pTagCond->pCond == NULL) {
     return NULL;
   }
@@ -309,6 +312,7 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) {
     for (int i = 0; i < pRes->numOfCols; i++) {
       tfree(pRes->buffer[i]);
     }
+    
     pRes->numOfCols = 0;
   }
   
@@ -320,9 +324,32 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) {
   tfree(pRes->pColumnIndex);
   tfree(pRes->buffer);
   
+  if (pRes->pArithSup != NULL) {
+    tfree(pRes->pArithSup->data);
+    tfree(pRes->pArithSup);
+  }
+  
   pRes->data = NULL;  // pRes->data points to the buffer of pRsp, no need to free
 }
 
+static void tscFreeQueryInfo(SSqlCmd* pCmd) {
+  if (pCmd == NULL || pCmd->numOfClause == 0) {
+    return;
+  }
+  
+  for (int32_t i = 0; i < pCmd->numOfClause; ++i) {
+    char* addr = (char*)pCmd - offsetof(SSqlObj, cmd);
+    SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i);
+    
+    freeQueryInfoImpl(pQueryInfo);
+    clearAllTableMetaInfo(pQueryInfo, (const char*)addr, false);
+    tfree(pQueryInfo);
+  }
+  
+  pCmd->numOfClause = 0;
+  tfree(pCmd->pQueryInfo);
+}
+
 void tscResetSqlCmdObj(SSqlCmd* pCmd) {
   pCmd->command   = 0;
   pCmd->numOfCols = 0;
@@ -330,11 +357,13 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) {
   pCmd->curSql    = NULL;
   pCmd->msgType   = 0;
   pCmd->parseFinished = 0;
+  pCmd->autoCreated = 0;
   
   taosHashCleanup(pCmd->pTableList);
-  pCmd->pTableList= NULL;
+  pCmd->pTableList = NULL;
   
   pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
+  
   tscFreeQueryInfo(pCmd);
 }
 
@@ -343,6 +372,7 @@ void tscFreeSqlResult(SSqlObj* pSql) {
   
   SSqlRes* pRes = &pSql->res;
   tscDestroyResPointerInfo(pRes);
+  
   memset(&pSql->res, 0, sizeof(SSqlRes));
 }
 
@@ -366,8 +396,8 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   pthread_mutex_unlock(&pObj->mutex);
   
   tscFreeSqlResult(pSql);
-  tfree(pSql->pSubs);
   
+  tfree(pSql->pSubs);
   pSql->freed = 0;
   pSql->numOfSubs = 0;
   
@@ -726,10 +756,17 @@ void tscCloseTscObj(STscObj* pObj) {
   taosTmrStopA(&(pObj->pTimer));
   tscFreeSqlObj(pSql);
 
-  rpcClose(pObj->pMgmtConn);
+  if (pSql) {
+    sem_destroy(&pSql->rspSem);
+  }
+  
   pthread_mutex_destroy(&pObj->mutex);
   
-  tscTrace("%p DB connection is closed", pObj);
+  if (pObj->pDnodeConn != NULL) {
+    rpcClose(pObj->pDnodeConn);
+  }
+  
+  tscTrace("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn);
   tfree(pObj);
 }
 
@@ -913,6 +950,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
     
     if (pInfo->pArithExprInfo != NULL) {
       tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL);
+      tfree(pInfo->pArithExprInfo);
     }
   }
   
@@ -947,7 +985,7 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
   pExpr->colInfo.colIndex = pColIndex->columnIndex;
   pExpr->resType       = type;
   pExpr->resBytes      = size;
-  pExpr->interResBytes = interSize;
+  pExpr->interBytes = interSize;
   pExpr->uid           = pTableMetaInfo->pTableMeta->uid;
   
   return pExpr;
@@ -1422,20 +1460,6 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) {
   return pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE;
 }
 
-void tscCleanSqlCmd(SSqlCmd* pCmd) {
-  pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
-  tscFreeQueryInfo(pCmd);
-
-  uint32_t allocSize = pCmd->allocSize;
-  char*    allocPtr = pCmd->payload;
-
-  memset(pCmd, 0, sizeof(SSqlCmd));
-
-  // restore values
-  pCmd->allocSize = allocSize;
-  pCmd->payload = allocPtr;
-}
-
 /*
  * the following three kinds of SqlObj should not be freed
  * 1. SqlObj for stream computing
@@ -1630,24 +1654,6 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem
   tfree(pQueryInfo->pTableMetaInfo);
 }
 
-void tscFreeQueryInfo(SSqlCmd* pCmd) {
-  if (pCmd == NULL || pCmd->numOfClause == 0) {
-    return;
-  }
-
-  for (int32_t i = 0; i < pCmd->numOfClause; ++i) {
-    char* addr = (char*)pCmd - offsetof(SSqlObj, cmd);
-    SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i);
-
-    freeQueryInfoImpl(pQueryInfo);
-    clearAllTableMetaInfo(pQueryInfo, (const char*)addr, false);
-    tfree(pQueryInfo);
-  }
-
-  pCmd->numOfClause = 0;
-  tfree(pCmd->pQueryInfo);
-}
-
 STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
                                     SVgroupsInfo* vgroupList, SArray* pTagCols) {
   void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES);

src/common/inc/tdataformat.h

@@ -28,16 +28,16 @@ extern "C" {
 
 #define STR_TO_VARSTR(x, str) do {VarDataLenT __len = strlen(str); \
   *(VarDataLenT*)(x) = __len; \
-  strncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), __len);} while(0);
+  strncpy(varDataVal(x), (str), __len);} while(0);
 
 #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) do {\
-  char* _e = stpncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_maxs));\
+  char* _e = stpncpy(varDataVal(x), (str), (_maxs));\
-  *(VarDataLenT*)(x) = (_e - (x) - VARSTR_HEADER_SIZE);\
+  varDataSetLen(x, (_e - (x) - VARSTR_HEADER_SIZE));\
 } while(0)
 
 #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) do {\
   *(VarDataLenT*)(x) = (_size); \
-  strncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_size));\
+  strncpy(varDataVal(x), (str), (_size));\
 } while(0);
 
 // ----------------- TSDB COLUMN DEFINITION
@@ -97,7 +97,7 @@ typedef void *SDataRow;
 #define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
 
 #define dataRowLen(r) (*(int32_t *)(r))
-#define dataRowTuple(r) POINTER_DRIFT(r, TD_DATA_ROW_HEAD_SIZE)
+#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
 #define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
@@ -114,10 +114,10 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o
   switch (type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-      return POINTER_DRIFT(row, *(VarDataOffsetT *)POINTER_DRIFT(row, offset));
+      return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
       break;
     default:
-      return POINTER_DRIFT(row, offset);
+      return POINTER_SHIFT(row, offset);
       break;
   }
 }
@@ -149,11 +149,11 @@ static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
   switch (pCol->type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-      return POINTER_DRIFT(pCol->pData, pCol->dataOff[row]);
+      return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
       break;
 
     default:
-      return POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
+      return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
       break;
   }
 }

src/common/inc/tglobal.h

@@ -55,18 +55,15 @@ extern char  tsFirst[];
 extern char  tsSecond[];
 extern char  tsLocalEp[];
 extern uint16_t tsServerPort;
-extern uint16_t tsMnodeDnodePort;
-extern uint16_t tsMnodeShellPort;
 extern uint16_t tsDnodeShellPort;
-extern uint16_t tsDnodeMnodePort;
+extern uint16_t tsDnodeDnodePort;
 extern uint16_t tsSyncPort;
 
 extern int32_t tsStatusInterval;
 extern int32_t tsShellActivityTimer;
 extern int32_t tsVnodePeerHBTimer;
 extern int32_t tsMgmtPeerHBTimer;
-extern int32_t tsMeterMetaKeepTimer;
+extern int32_t tsTableMetaKeepTimer;
-extern int32_t tsMetricMetaKeepTimer;
 
 extern float    tsNumOfThreadsPerCore;
 extern float    tsRatioOfQueryThreads;
@@ -77,8 +74,8 @@ extern int16_t  tsNumOfTotalVnodes;
 extern uint32_t tsPublicIpInt;
 
 extern int32_t tsCacheBlockSize;
-extern int32_t tsTotalBlocks;
+extern int32_t tsBlocksPerVnode;
-extern int32_t tsTablesPerVnode;
+extern int32_t tsMaxTablePerVnode;
 extern int16_t tsDaysPerFile;
 extern int32_t tsDaysToKeep;
 extern int32_t tsMinRowsInFileBlock;
@@ -86,7 +83,7 @@ extern int32_t tsMaxRowsInFileBlock;
 extern int16_t tsCommitTime;  // seconds
 extern int32_t tsTimePrecision;
 extern int16_t tsCompression;
-extern int16_t tsCommitLog;
+extern int16_t tsWAL;
 extern int32_t tsReplications;
 
 extern int16_t tsAffectedRowsMod;
@@ -94,7 +91,6 @@ extern int32_t tsNumOfMPeers;
 extern int32_t tsMaxShellConns;
 extern int32_t tsMaxTables;
 
-extern char tsLocalIp[];
 extern char tsDefaultDB[];
 extern char tsDefaultUser[];
 extern char tsDefaultPass[];
@@ -136,7 +132,6 @@ extern int32_t  tsHttpEnableRecordSql;
 extern int32_t  tsTelegrafUseFieldNum;
 
 extern int32_t  tsTscEnableRecordSql;
-extern int32_t  tsAnyIp;
 
 extern char     tsMonitorDbName[];
 extern char     tsInternalPass[];
@@ -144,10 +139,11 @@ extern int32_t  tsMonitorInterval;
 
 extern int32_t tsAsyncLog;
 extern int32_t tsNumOfLogLines;
-extern int32_t ddebugFlag;
+extern int32_t dDebugFlag;
-extern int32_t mdebugFlag;
+extern int32_t vDebugFlag;
-extern int32_t cdebugFlag;
+extern int32_t mDebugFlag;
-extern int32_t jnidebugFlag;
+extern int32_t cDebugFlag;
+extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
@@ -155,8 +151,8 @@ extern int32_t monitorDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t debugFlag;
-extern int32_t odbcdebugFlag;
+extern int32_t odbcDebugFlag;
-extern int32_t qdebugFlag;
+extern int32_t qDebugFlag;
 
 extern uint32_t taosMaxTmrCtrl;
 

src/common/src/tdataformat.c

@@ -167,17 +167,17 @@ void tdFreeDataRow(SDataRow row) {
 int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
   ASSERT(value != NULL);
   int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
-  char *  ptr = POINTER_DRIFT(row, dataRowLen(row));
+  char *  ptr = POINTER_SHIFT(row, dataRowLen(row));
 
   switch (type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-      *(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row);
+      *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
       memcpy(ptr, value, varDataTLen(value));
       dataRowLen(row) += varDataTLen(value);
       break;
     default:
-      memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]);
+      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
       break;
   }
 
@@ -202,13 +202,13 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
   if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
     pDataCol->spaceSize = (sizeof(VarDataLenT) + pDataCol->bytes) * maxPoints;
     pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
-    pDataCol->pData = POINTER_DRIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints);
+    pDataCol->pData = POINTER_SHIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints);
-    *pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints);
+    *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints);
   } else {
     pDataCol->spaceSize = pDataCol->bytes * maxPoints;
     pDataCol->dataOff = NULL;
     pDataCol->pData = *pBuf;
-    *pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize);
+    *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize);
   }
 
 }
@@ -222,13 +222,13 @@ void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoint
       // set offset
       pCol->dataOff[numOfPoints] = pCol->len;
       // Copy data
-      memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
       // Update the length
       pCol->len += varDataTLen(value);
       break;
     default:
       ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
-      memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, pCol->bytes);
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
       pCol->len += pCol->bytes;
       break;
   }
@@ -244,12 +244,12 @@ void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) {
     VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
     pCol->len = pCol->len - toffset;
     ASSERT(pCol->len > 0);
-    memmove(pCol->pData, POINTER_DRIFT(pCol->pData, toffset), pCol->len);
+    memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
     dataColSetOffset(pCol, pointsLeft);
   } else {
     ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
     pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
-    memmove(pCol->pData, POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
+    memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
   }
 }
 
@@ -301,7 +301,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) {
   for (int i = 0; i < nEle; i++) {
     pCol->dataOff[i] = offset;
     offset += varDataTLen(tptr);
-    tptr = POINTER_DRIFT(tptr, varDataTLen(tptr));
+    tptr = POINTER_SHIFT(tptr, varDataTLen(tptr));
   }
 }
 

src/common/src/tglobal.c

@@ -66,16 +66,13 @@ char  tsSecond[TSDB_FQDN_LEN] = {0};
 char  tsArbitrator[TSDB_FQDN_LEN] = {0};
 char  tsLocalEp[TSDB_FQDN_LEN] = {0};  // Local End Point, hostname:port
 uint16_t tsServerPort = 6030;
-uint16_t tsMnodeShellPort = 6030;   // udp[6030-6034] tcp[6030]
+uint16_t tsDnodeShellPort = 6030;  // udp[6035-6039] tcp[6035]
-uint16_t tsDnodeShellPort = 6035;  // udp[6035-6039] tcp[6035]
+uint16_t tsDnodeDnodePort = 6035;   // udp/tcp
-uint16_t tsMnodeDnodePort = 6040;   // udp/tcp
+uint16_t tsSyncPort = 6040;
-uint16_t tsDnodeMnodePort = 6045;   // udp/tcp
-uint16_t tsSyncPort = 6050;
 
 int32_t tsStatusInterval = 1;         // second
 int32_t tsShellActivityTimer = 3;     // second
-int32_t tsMeterMetaKeepTimer = 7200;  // second
+int32_t tsTableMetaKeepTimer = 7200;  // second
-int32_t tsMetricMetaKeepTimer = 600;  // second
 int32_t tsRpcTimer = 300;
 int32_t tsRpcMaxTime = 600;      // seconds;
 
@@ -85,22 +82,22 @@ int16_t tsNumOfVnodesPerCore = 8;
 int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
 
 #ifdef _TD_ARM_32_
-int32_t tsTablesPerVnode = 100;
+int32_t tsMaxTablePerVnode = 100;
 #else
-int32_t tsTablesPerVnode = TSDB_DEFAULT_TABLES;
+int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
 #endif
 
 int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
-int32_t tsTotalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
+int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
-int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
+int16_t tsDaysPerFile    = TSDB_DEFAULT_DAYS_PER_FILE;
-int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP;
+int32_t tsDaysToKeep     = TSDB_DEFAULT_KEEP;
 int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
 int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
-int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME;  // seconds
+int16_t tsCommitTime    = TSDB_DEFAULT_COMMIT_TIME;  // seconds
 int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
-int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
+int16_t tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
-int16_t tsCommitLog = TSDB_DEFAULT_CLOG_LEVEL;
+int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
-int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM;
+int32_t tsReplications  = TSDB_DEFAULT_REPLICA_NUM;
 
 /**
  * Change the meaning of affected rows:
@@ -130,19 +127,20 @@ int32_t tsRestRowLimit = 10240;
 int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
 
 int32_t tsNumOfLogLines = 10000000;
-int32_t mdebugFlag = 135;
+int32_t mDebugFlag = 135;
 int32_t sdbDebugFlag = 135;
-int32_t ddebugFlag = 131;
+int32_t dDebugFlag = 135;
-int32_t cdebugFlag = 131;
+int32_t vDebugFlag = 135;
-int32_t jnidebugFlag = 131;
+int32_t cDebugFlag = 135;
-int32_t odbcdebugFlag = 131;
+int32_t jniDebugFlag = 131;
+int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
 int32_t monitorDebugFlag = 131;
-int32_t qdebugFlag = 131;
+int32_t qDebugFlag = 131;
-int32_t rpcDebugFlag = 131;
+int32_t rpcDebugFlag = 135;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
-int32_t sDebugFlag = 131;
+int32_t sDebugFlag = 135;
 
 // the maximum number of results for projection query on super table that are returned from
 // one virtual node, to order according to timestamp
@@ -206,10 +204,19 @@ static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
 
 void taosSetAllDebugFlag() {
   for (int32_t i = 0; i < tsGlobalConfigNum; ++i) {
-    SGlobalCfg *cfg = &tsGlobalConfig[i];
+    mDebugFlag = debugFlag;
-    if ((cfg->cfgType & TSDB_CFG_CTYPE_B_LOG) && cfg->cfgType == TAOS_CFG_VTYPE_INT32) {
+    sdbDebugFlag = debugFlag;
-      *((int32_t*)cfg->ptr) = debugFlag;
+    dDebugFlag = debugFlag;
-    }
+    vDebugFlag = debugFlag;
+    cDebugFlag = debugFlag;
+    jniDebugFlag = debugFlag;
+    odbcDebugFlag = debugFlag;
+    httpDebugFlag = debugFlag;
+    monitorDebugFlag = debugFlag;
+    rpcDebugFlag = debugFlag;
+    uDebugFlag = debugFlag;
+    sDebugFlag = debugFlag;
+    //qDebugFlag = debugFlag;    
   }
   uPrint("all debug flag are set to %d", debugFlag);
 }
@@ -506,18 +513,8 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_SECOND;
   taosInitConfigOption(cfg);
 
-  cfg.option = "meterMetaKeepTimer";
+  cfg.option = "tableMetaKeepTimer";
-  cfg.ptr = &tsMeterMetaKeepTimer;
+  cfg.ptr = &tsTableMetaKeepTimer;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 1;
-  cfg.maxValue = 8640000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_SECOND;
-  taosInitConfigOption(cfg);
-
-  cfg.option = "metricMetaKeepTimer";
-  cfg.ptr = &tsMetricMetaKeepTimer;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 1;
@@ -587,8 +584,8 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   // database configs
-  cfg.option = "tables";
+  cfg.option = "maxtablesPerVnode";
-  cfg.ptr = &tsTablesPerVnode;
+  cfg.ptr = &tsMaxTablePerVnode;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = TSDB_MIN_TABLES;
@@ -608,7 +605,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "blocks";
-  cfg.ptr = &tsTotalBlocks;
+  cfg.ptr = &tsBlocksPerVnode;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = TSDB_MIN_TOTAL_BLOCKS;
@@ -677,12 +674,12 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "clog";
+  cfg.option = "wallevel";
-  cfg.ptr = &tsCommitLog;
+  cfg.ptr = &tsWAL;
   cfg.valType = TAOS_CFG_VTYPE_INT16;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_CLOG_LEVEL;
+  cfg.minValue = TSDB_MIN_WAL_LEVEL;
-  cfg.maxValue = TSDB_MAX_CLOG_LEVEL;
+  cfg.maxValue = TSDB_MAX_WAL_LEVEL;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
@@ -1005,7 +1002,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "mDebugFlag";
-  cfg.ptr = &mdebugFlag;
+  cfg.ptr = &mDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
   cfg.minValue = 0;
@@ -1015,7 +1012,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "dDebugFlag";
-  cfg.ptr = &ddebugFlag;
+  cfg.ptr = &dDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
   cfg.minValue = 0;
@@ -1065,7 +1062,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "cDebugFlag";
-  cfg.ptr = &cdebugFlag;
+  cfg.ptr = &cDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1075,7 +1072,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "jniDebugFlag";
-  cfg.ptr = &jnidebugFlag;
+  cfg.ptr = &jniDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1085,7 +1082,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "odbcDebugFlag";
-  cfg.ptr = &odbcdebugFlag;
+  cfg.ptr = &odbcDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1125,7 +1122,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "qDebugFlag";
-  cfg.ptr = &qdebugFlag;
+  cfg.ptr = &qDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1201,6 +1198,10 @@ void taosInitGlobalCfg() {
 }
 
 bool taosCheckGlobalCfg() {
+  if (debugFlag == 135 || debugFlag == 199) {
+    taosSetAllDebugFlag();
+  }
+  
   taosGetFqdn(tsLocalEp);
   sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort);
   uPrint("localEp is %s", tsLocalEp);
@@ -1244,10 +1245,8 @@ bool taosCheckGlobalCfg() {
   
   tsVersion = 10 * tsVersion;
 
-  tsMnodeShellPort = tsServerPort + TSDB_PORT_MNODESHELL;   // udp[6030-6034] tcp[6030]
   tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL;  // udp[6035-6039] tcp[6035]
-  tsMnodeDnodePort = tsServerPort + TSDB_PORT_MNODEDNODE;   // udp/tcp
+  tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE;   // udp/tcp
-  tsDnodeMnodePort = tsServerPort + TSDB_PORT_DNODEMNODE;   // udp/tcp
   tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
 
   return true;

src/common/src/ttypes.c

@@ -92,9 +92,9 @@ bool isNull(const char *val, int32_t type) {
     case TSDB_DATA_TYPE_DOUBLE:
       return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
     case TSDB_DATA_TYPE_NCHAR:
-      return *(uint32_t *)val == TSDB_DATA_NCHAR_NULL;
+      return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
     case TSDB_DATA_TYPE_BINARY:
-      return *(uint8_t *)val == TSDB_DATA_BINARY_NULL;
+      return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
     default:
       return false;
   };

src/connector/python/linux/python2/taos/cinterface.py

@@ -96,7 +96,8 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
-                res.append( (ctypes.cast(data+nbytes*(abs(num_of_rows - i -1)),  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+                tmpstr = ctypes.c_char_p(data)
+                res.append( tmpstr.value.decode() )
             else:
                 res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
         except ValueError:
@@ -146,6 +147,7 @@ class CTaosInterface(object):
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
+    libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
 
     def __init__(self, config=None):
         '''
@@ -314,6 +316,8 @@ class CTaosInterface(object):
 
         isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
         blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
         for i in range(len(fields)):
             data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
             if data == None:
@@ -323,7 +327,7 @@ class CTaosInterface(object):
             if fields[i]['type'] not in _CONVERT_FUNC:
                 raise DatabaseError("Invalid data type returned from database")
             
-            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fields[i]['bytes'], isMicro)
+            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
 
         return blocks, abs(num_of_rows)
 

src/connector/python/linux/python3/taos/cinterface.py

@@ -96,7 +96,8 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
-                res.append( (ctypes.cast(data+nbytes*(abs(num_of_rows - i -1)),  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+                tmpstr = ctypes.c_char_p(data)
+                res.append( tmpstr.value.decode() )
             else:
                 res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
         except ValueError:
@@ -146,6 +147,7 @@ class CTaosInterface(object):
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
+    libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
 
     def __init__(self, config=None):
         '''
@@ -314,6 +316,8 @@ class CTaosInterface(object):
 
         isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
         blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
         for i in range(len(fields)):
             data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
             if data == None:
@@ -323,7 +327,7 @@ class CTaosInterface(object):
             if fields[i]['type'] not in _CONVERT_FUNC:
                 raise DatabaseError("Invalid data type returned from database")
             
-            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fields[i]['bytes'], isMicro)
+            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
 
         return blocks, abs(num_of_rows)
 

src/connector/python/windows/python2/taos/cinterface.py

@@ -96,7 +96,8 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
-                res.append( (ctypes.cast(data+nbytes*(abs(num_of_rows - i -1)),  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+                tmpstr = ctypes.c_char_p(data)
+                res.append( tmpstr.value.decode() )
             else:
                 res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
         except ValueError:
@@ -146,6 +147,7 @@ class CTaosInterface(object):
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
+    libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
 
     def __init__(self, config=None):
         '''
@@ -314,6 +316,8 @@ class CTaosInterface(object):
 
         isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
         blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
         for i in range(len(fields)):
             data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
             if data == None:
@@ -323,7 +327,7 @@ class CTaosInterface(object):
             if fields[i]['type'] not in _CONVERT_FUNC:
                 raise DatabaseError("Invalid data type returned from database")
             
-            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fields[i]['bytes'], isMicro)
+            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
 
         return blocks, abs(num_of_rows)
 

src/connector/python/windows/python3/taos/cinterface.py

@@ -96,7 +96,8 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
-                res.append( (ctypes.cast(data+nbytes*(abs(num_of_rows - i -1)),  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+                tmpstr = ctypes.c_char_p(data)
+                res.append( tmpstr.value.decode() )
             else:
                 res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
         except ValueError:
@@ -146,6 +147,7 @@ class CTaosInterface(object):
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
+    libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
 
     def __init__(self, config=None):
         '''
@@ -314,6 +316,8 @@ class CTaosInterface(object):
 
         isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
         blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
         for i in range(len(fields)):
             data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
             if data == None:
@@ -323,7 +327,7 @@ class CTaosInterface(object):
             if fields[i]['type'] not in _CONVERT_FUNC:
                 raise DatabaseError("Invalid data type returned from database")
             
-            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fields[i]['bytes'], isMicro)
+            blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
 
         return blocks, abs(num_of_rows)
 

src/cq/src/cqMain.c

@@ -26,10 +26,10 @@
 #include "tcq.h"
 #include "taos.h"
 
-#define cError(...) if (cqDebugFlag & DEBUG_ERROR) {taosPrintLog("ERROR CQ ", cqDebugFlag, __VA_ARGS__);}
+#define cError(...) if (cqDebugFlag & DEBUG_ERROR) {taosPrintLog("ERROR CQ  ", cqDebugFlag, __VA_ARGS__);}
-#define cWarn(...) if (cqDebugFlag & DEBUG_WARN) {taosPrintLog("WARN CQ ", cqDebugFlag, __VA_ARGS__);}
+#define cWarn(...) if (cqDebugFlag & DEBUG_WARN) {taosPrintLog("WARN CQ  ", cqDebugFlag, __VA_ARGS__);}
-#define cTrace(...) if (cqDebugFlag & DEBUG_TRACE) {taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__);}
+#define cTrace(...) if (cqDebugFlag & DEBUG_TRACE) {taosPrintLog("CQ  ", cqDebugFlag, __VA_ARGS__);}
-#define cPrint(...) {taosPrintLog("WAL ", 255, __VA_ARGS__);}
+#define cPrint(...) {taosPrintLog("CQ  ", 255, __VA_ARGS__);}
 
 typedef struct {
   int      vgId;

src/cq/test/cqtest.c

@@ -33,13 +33,13 @@ int main(int argc, char *argv[]) {
 
   for (int i=1; i<argc; ++i) {
     if (strcmp(argv[i], "-d")==0 && i < argc-1) {
-      ddebugFlag = atoi(argv[++i]);
+      dDebugFlag = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-n") == 0 && i <argc-1) {
       num = atoi(argv[++i]);
     } else {
       printf("\nusage: %s [options] \n", argv[0]);
       printf("  [-n num]: number of streams, default:%d\n", num);
-      printf("  [-d debugFlag]: debug flag, default:%d\n", ddebugFlag);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", dDebugFlag);
       printf("  [-h help]: print out this help\n\n");
       exit(0);
     }

src/dnode/inc/dnodeInt.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_LOG_H
+#define TDENGINE_DNODE_LOG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tlog.h"
+
+extern int32_t dDebugFlag;
+
+#define dError(...) if (dDebugFlag & DEBUG_ERROR) {taosPrintLog("ERROR DND ", 255, __VA_ARGS__); }
+#define dWarn(...) if (dDebugFlag & DEBUG_WARN) {taosPrintLog("WARN  DND ", dDebugFlag, __VA_ARGS__); }
+#define dTrace(...) if (dDebugFlag & DEBUG_TRACE) {taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }
+#define dPrint(...) {taosPrintLog("DND ", 255, __VA_ARGS__); }
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/dnode/inc/dnodeLog.h

@@ -1,46 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_DNODE_LOG_H
-#define TDENGINE_DNODE_LOG_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "tlog.h"
-
-extern int32_t ddebugFlag;
-
-#define dError(...)                          \
-  if (ddebugFlag & DEBUG_ERROR) {            \
-    taosPrintLog("ERROR DND ", 255, __VA_ARGS__); \
-  }
-#define dWarn(...)                                  \
-  if (ddebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  DND ", ddebugFlag, __VA_ARGS__); \
-  }
-#define dTrace(...)                           \
-  if (ddebugFlag & DEBUG_TRACE) {             \
-    taosPrintLog("DND ", ddebugFlag, __VA_ARGS__); \
-  }
-#define dPrint(...) \
-  { taosPrintLog("DND ", 255, __VA_ARGS__); }
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/dnode/inc/dnodeMgmt.h

@@ -22,7 +22,7 @@ extern "C" {
 
 int32_t dnodeInitMgmt();
 void    dnodeCleanupMgmt();
-void    dnodeMgmt(SRpcMsg *rpcMsg);
+void    dnodeDispatchToDnodeMgmt(SRpcMsg *rpcMsg);
 
 void*   dnodeGetVnode(int32_t vgId);
 int32_t dnodeGetVnodeStatus(void *pVnode);

src/dnode/inc/dnodePeer.h

@@ -13,16 +13,17 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_MCLIENT_H
+#ifndef TDENGINE_DNODE_DNODE_H
-#define TDENGINE_DNODE_MCLIENT_H
+#define TDENGINE_DNODE_DNODE_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int32_t  dnodeInitMClient();
+int32_t dnodeInitServer();
-void     dnodeCleanupMClient();
+void    dnodeCleanupServer();
-void     dnodeSendMsgToMnode(SRpcMsg *rpcMsg);
+int32_t dnodeInitClient();
+void    dnodeCleanupClient();
 
 #ifdef __cplusplus
 }

src/dnode/inc/dnodeVRead.h

@@ -22,7 +22,7 @@ extern "C" {
 
 int32_t dnodeInitRead();
 void    dnodeCleanupRead();
-void    dnodeRead(SRpcMsg *pMsg);
+void    dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }

src/dnode/inc/dnodeVWrite.h

@@ -22,7 +22,7 @@ extern "C" {
 
 int32_t dnodeInitWrite();
 void    dnodeCleanupWrite();
-void    dnodeWrite(SRpcMsg *pMsg);
+void    dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg);
 void    dnodeSendWriteResponse(void *pVnode, void *param, int32_t code);
 
 #ifdef __cplusplus

src/dnode/src/dnodeMClient.c

@@ -1,468 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "cJSON.h"
-#include "taosmsg.h"
-#include "trpc.h"
-#include "tutil.h"
-#include "tsync.h"
-#include "ttime.h"
-#include "ttimer.h"
-#include "tbalance.h"
-#include "tglobal.h"
-#include "vnode.h"
-#include "mnode.h"
-#include "dnode.h"
-#include "dnodeLog.h"
-#include "dnodeMClient.h"
-#include "dnodeModule.h"
-#include "dnodeMgmt.h"
-
-#define MPEER_CONTENT_LEN 2000
-
-static void   dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes);
-static bool   dnodeReadMnodeInfos();
-static void   dnodeSaveMnodeInfos();
-static void   dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg);
-static bool   dnodeReadDnodeCfg();
-static void   dnodeSaveDnodeCfg();
-static void   dnodeProcessRspFromMnode(SRpcMsg *pMsg);
-static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
-static void   dnodeSendStatusMsg(void *handle, void *tmrId);
-static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
-
-static void    *tsDnodeMClientRpc = NULL;
-static void    *tsDnodeTmr = NULL;
-static void    *tsStatusTimer = NULL;
-static uint32_t tsRebootTime;
-
-static SRpcIpSet     tsMnodeIpSet  = {0};
-static SDMMnodeInfos tsMnodeInfos = {0};
-static SDMDnodeCfg   tsDnodeCfg = {0};
-
-void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
-  dTrace("mgmt IP list is changed for ufp is called");
-  tsMnodeIpSet = *pIpSet;
-}
-
-void dnodeGetMnodeDnodeIpSet(void *ipSetRaw) {
-  SRpcIpSet *ipSet = ipSetRaw;
-  ipSet->numOfIps = tsMnodeInfos.nodeNum;
-  ipSet->inUse = tsMnodeInfos.inUse;
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) {
-    taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, ipSet->fqdn[i], &ipSet->port[i]);
-    ipSet->port[i] += TSDB_PORT_MNODEDNODE;
-  }
-}
-
-int32_t dnodeInitMClient() {
-  dnodeReadDnodeCfg();
-  tsRebootTime = taosGetTimestampSec();
-
-  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
-  if (tsDnodeTmr == NULL) {
-    dError("failed to init dnode timer");
-    return -1;
-  }
-  
-  if (!dnodeReadMnodeInfos()) {
-    memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet));
-    memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
-    tsMnodeIpSet.numOfIps = 1;
-    taosGetFqdnPortFromEp(tsFirst, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
-    tsMnodeIpSet.port[0] += TSDB_PORT_MNODEDNODE;
-    if (strcmp(tsSecond, tsFirst) != 0) {
-      tsMnodeIpSet.numOfIps = 2;
-      taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]);
-      tsMnodeIpSet.port[1] += TSDB_PORT_MNODEDNODE;
-    }
-  } else {
-    tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
-    tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
-    for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-      taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
-      tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE;
-    }
-  }
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.label        = "DND-MC";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp          = dnodeProcessRspFromMnode;
-  rpcInit.ufp          = dnodeUpdateIpSet;
-  rpcInit.sessions     = 100;
-  rpcInit.connType     = TAOS_CONN_CLIENT;
-  rpcInit.idleTime     = tsShellActivityTimer * 2000;
-  rpcInit.user         = "t";
-  rpcInit.ckey         = "key";
-  rpcInit.secret       = "secret";
-
-  tsDnodeMClientRpc = rpcOpen(&rpcInit);
-  if (tsDnodeMClientRpc == NULL) {
-    dError("failed to init mnode rpc client");
-    return -1;
-  }
-
-  tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp;
-  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
-  
-  dPrint("mnode rpc client is opened");
-  return 0;
-}
-
-void dnodeCleanupMClient() {
-  if (tsStatusTimer != NULL) {
-    taosTmrStopA(&tsStatusTimer);
-    tsStatusTimer = NULL;
-  }
-
-  if (tsDnodeTmr != NULL) {
-    taosTmrCleanUp(tsDnodeTmr);
-    tsDnodeTmr = NULL;
-  }
-
-  if (tsDnodeMClientRpc) {
-    rpcClose(tsDnodeMClientRpc);
-    tsDnodeMClientRpc = NULL;
-    dPrint("mnode rpc client is closed");
-  }
-}
-
-static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) {
-  if (tsDnodeProcessMgmtRspFp[pMsg->msgType]) {
-    (*tsDnodeProcessMgmtRspFp[pMsg->msgType])(pMsg);
-  } else {
-    dError("%s is not processed in dnode mclient", taosMsg[pMsg->msgType]);
-    SRpcMsg rpcRsp = {.pCont = 0, .contLen = 0, .code = TSDB_CODE_OPS_NOT_SUPPORT, .handle = pMsg->handle};
-    rpcSendResponse(&rpcRsp);
-  }
-
-  rpcFreeCont(pMsg->pCont);
-}
-
-static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
-  if (pMsg->code != TSDB_CODE_SUCCESS) {
-    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    return;
-  }
-
-  SDMStatusRsp *pStatusRsp = pMsg->pCont;
-  SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes;
-  if (pMnodes->nodeNum <= 0) {
-    dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum);
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    return;
-  }
-
-  SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
-  pCfg->numOfVnodes  = htonl(pCfg->numOfVnodes);
-  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
-  pCfg->dnodeId      = htonl(pCfg->dnodeId);
-
-  for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
-    SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
-    pMnodeInfo->nodeId   = htonl(pMnodeInfo->nodeId);
-  }
-
-  SDMVgroupAccess *pVgAcccess = pStatusRsp->vgAccess;
-  for (int32_t i = 0; i < pCfg->numOfVnodes; ++i) {
-    pVgAcccess[i].vgId = htonl(pVgAcccess[i].vgId);
-  }
-  
-  dnodeProcessModuleStatus(pCfg->moduleStatus);
-  dnodeUpdateDnodeCfg(pCfg);
-  dnodeUpdateMnodeInfos(pMnodes);
-  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-}
-
-static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
-  bool mnodesChanged = (memcmp(&tsMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
-  bool mnodesNotInit = (tsMnodeInfos.nodeNum == 0);
-  if (!(mnodesChanged || mnodesNotInit)) return;
-
-  memcpy(&tsMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
-
-  tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
-  tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
-    tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE;
-  }
-
-  dPrint("mnodes is changed, nodeNum:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
-  }
-
-  dnodeSaveMnodeInfos();
-  sdbUpdateSync();
-}
-
-void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) {
-  if (tsDnodeMClientRpc) {
-    rpcSendRequest(tsDnodeMClientRpc, &tsMnodeIpSet, rpcMsg);
-  }
-}
-
-static bool dnodeReadMnodeInfos() {
-  char ipFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
-  FILE *fp = fopen(ipFile, "r");
-  if (!fp) {
-    dTrace("failed to read mnode mgmtIpList.json, file not exist");
-    return false;
-  }
-
-  bool  ret = false;
-  int   maxLen = 2000;
-  char *content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    free(content);
-    fclose(fp);
-    dError("failed to read mnode mgmtIpList.json, content is null");
-    return false;
-  }
-
-  cJSON* root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read mnode mgmtIpList.json, invalid json format");
-    goto PARSE_OVER;
-  }
-
-  cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
-  if (!inUse || inUse->type != cJSON_Number) {
-    dError("failed to read mnode mgmtIpList.json, inUse not found");
-    goto PARSE_OVER;
-  }
-  tsMnodeInfos.inUse = inUse->valueint;
-
-  cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
-  if (!nodeNum || nodeNum->type != cJSON_Number) {
-    dError("failed to read mnode mgmtIpList.json, nodeNum not found");
-    goto PARSE_OVER;
-  }
-  tsMnodeInfos.nodeNum = nodeNum->valueint;
-
-  cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
-  if (!nodeInfos || nodeInfos->type != cJSON_Array) {
-    dError("failed to read mnode mgmtIpList.json, nodeInfos not found");
-    goto PARSE_OVER;
-  }
-
-  int size = cJSON_GetArraySize(nodeInfos);
-  if (size != tsMnodeInfos.nodeNum) {
-    dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched");
-    goto PARSE_OVER;
-  }
-
-  for (int i = 0; i < size; ++i) {
-    cJSON* nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
-    if (nodeInfo == NULL) continue;
-
-    cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
-    if (!nodeId || nodeId->type != cJSON_Number) {
-      dError("failed to read mnode mgmtIpList.json, nodeId not found");
-      goto PARSE_OVER;
-    }
-    tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
-
-    cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
-    if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
-      dError("failed to read mnode mgmtIpList.json, nodeName not found");
-      goto PARSE_OVER;
-    }
-    strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_FQDN_LEN);
- }
-
-  ret = true;
-
-  dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
-  }
-
-PARSE_OVER:
-  free(content);
-  cJSON_Delete(root);
-  fclose(fp);
-  return ret;
-}
-
-static void dnodeSaveMnodeInfos() {
-  char ipFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
-  FILE *fp = fopen(ipFile, "w");
-  if (!fp) return;
-
-  int32_t len = 0;
-  int32_t maxLen = 2000;
-  char *  content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsMnodeInfos.inUse);
-  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsMnodeInfos.nodeNum);
-  len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId);
-    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeEp);
-    if (i < tsMnodeInfos.nodeNum -1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");  
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");  
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n"); 
-
-  fwrite(content, 1, len, fp);
-  fclose(fp);
-  free(content);
-  
-  dPrint("save mnode iplist successed");
-}
-
-char *dnodeGetMnodeMasterEp() {
-  return tsMnodeInfos.nodeInfos[tsMnodeIpSet.inUse].nodeEp;
-}
-
-void* dnodeGetMnodeInfos() {
-  return &tsMnodeInfos;
-}
-
-static void dnodeSendStatusMsg(void *handle, void *tmrId) {
-  if (tsDnodeTmr == NULL) {
-    dError("dnode timer is already released");
-    return;
-  }
-
-  if (tsStatusTimer == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to start status timer");
-    return;
-  }
-
-  int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-  SDMStatusMsg *pStatus = rpcMallocCont(contLen);
-  if (pStatus == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to malloc status message");
-    return;
-  }
-
-  //strcpy(pStatus->dnodeName, tsDnodeName);
-  pStatus->version          = htonl(tsVersion);
-  pStatus->dnodeId          = htonl(tsDnodeCfg.dnodeId);
-  strcpy(pStatus->dnodeEp, tsLocalEp);
-  pStatus->lastReboot       = htonl(tsRebootTime);
-  pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
-  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
-  pStatus->diskAvailable    = tsAvailDataDirGB;
-  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
-  
-  vnodeBuildStatusMsg(pStatus);
-  contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
-  pStatus->openVnodes = htons(pStatus->openVnodes);
-  
-  SRpcMsg rpcMsg = {
-    .pCont   = pStatus,
-    .contLen = contLen,
-    .msgType = TSDB_MSG_TYPE_DM_STATUS
-  };
-
-  dnodeSendMsgToMnode(&rpcMsg);
-}
-
-static bool dnodeReadDnodeCfg() {
-  char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
-  FILE *fp = fopen(dnodeCfgFile, "r");
-  if (!fp) {
-    dTrace("failed to read dnodeCfg.json, file not exist");
-    return false;
-  }
-
-  bool  ret = false;
-  int   maxLen = 100;
-  char *content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    free(content);
-    fclose(fp);
-    dError("failed to read dnodeCfg.json, content is null");
-    return false;
-  }
-
-  cJSON* root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read dnodeCfg.json, invalid json format");
-    goto PARSE_CFG_OVER;
-  }
-
-  cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId");
-  if (!dnodeId || dnodeId->type != cJSON_Number) {
-    dError("failed to read dnodeCfg.json, dnodeId not found");
-    goto PARSE_CFG_OVER;
-  }
-  tsDnodeCfg.dnodeId = dnodeId->valueint;
-
-  ret = true;
-
-  dPrint("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId);
-
-PARSE_CFG_OVER:
-  free(content);
-  cJSON_Delete(root);
-  fclose(fp);
-  return ret;
-}
-
-static void dnodeSaveDnodeCfg() {
-  char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
-  FILE *fp = fopen(dnodeCfgFile, "w");
-  if (!fp) return;
-
-  int32_t len = 0;
-  int32_t maxLen = 100;
-  char *  content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d\n", tsDnodeCfg.dnodeId);
-  len += snprintf(content + len, maxLen - len, "}\n"); 
-
-  fwrite(content, 1, len, fp);
-  fclose(fp);
-  free(content);
-  
-  dPrint("save dnodeId successed");
-}
-
-void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
-  if (tsDnodeCfg.dnodeId == 0) {
-    dPrint("dnodeId is set to %d", pCfg->dnodeId);  
-    tsDnodeCfg.dnodeId = pCfg->dnodeId;
-    dnodeSaveDnodeCfg();
-  }
-}
-
-int32_t dnodeGetDnodeId() {
-  return tsDnodeCfg.dnodeId;
-}

src/dnode/src/dnodeMain.c

@@ -22,14 +22,13 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "dnode.h"
-#include "dnodeLog.h"
+#include "dnodeInt.h"
-#include "dnodeMClient.h"
 #include "dnodeMgmt.h"
-#include "dnodeMnode.h"
+#include "dnodePeer.h"
 #include "dnodeModule.h"
-#include "dnodeRead.h"
+#include "dnodeVRead.h"
 #include "dnodeShell.h"
-#include "dnodeWrite.h"
+#include "dnodeVWrite.h"
 #include "tgrant.h"
 
 static int32_t dnodeInitSystem();
@@ -167,10 +166,10 @@ static int32_t dnodeInitSystem() {
   if (dnodeInitStorage() != 0) return -1;
   if (dnodeInitRead() != 0) return -1;
   if (dnodeInitWrite() != 0) return -1;
-  if (dnodeInitMClient() != 0) return -1;
+  if (dnodeInitClient() != 0) return -1;
-  if (dnodeInitModules() != 0) return -1;
+  if (dnodeInitServer() != 0) return -1;
-  if (dnodeInitMnode() != 0) return -1;
   if (dnodeInitMgmt() != 0) return -1;
+  if (dnodeInitModules() != 0) return -1;
   if (dnodeInitShell() != 0) return -1;
 
   dnodeStartModules();
@@ -185,14 +184,14 @@ static void dnodeCleanUpSystem() {
   if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
     dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
     dnodeCleanupShell();
-    dnodeCleanupMnode();
+    dnodeCleanUpModules();
     dnodeCleanupMgmt();
-    dnodeCleanupMClient();
+    dnodeCleanupServer();
+    dnodeCleanupClient();
     dnodeCleanupWrite();
     dnodeCleanupRead();
-    dnodeCleanUpModules();
-    taos_cleanup();
     dnodeCleanupStorage();
+    taos_cleanup();
     taosCloseLog();
   }
 }

src/dnode/src/dnodeMgmt.c

@@ -15,19 +15,47 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "cJSON.h"
 #include "ihash.h"
 #include "taoserror.h"
 #include "taosmsg.h"
+#include "ttime.h"
+#include "ttimer.h"
 #include "trpc.h"
 #include "tsdb.h"
 #include "twal.h"
-#include "vnode.h"
+#include "tsync.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tbalance.h"
 #include "tglobal.h"
-#include "dnodeLog.h"
+#include "dnode.h"
-#include "dnodeMClient.h"
+#include "vnode.h"
+#include "mnode.h"
+#include "dnodeInt.h"
 #include "dnodeMgmt.h"
-#include "dnodeRead.h"
+#include "dnodeVRead.h"
-#include "dnodeWrite.h"
+#include "dnodeVWrite.h"
+#include "dnodeModule.h"
+
+#define MPEER_CONTENT_LEN 2000
+
+static void   dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes);
+static bool   dnodeReadMnodeInfos();
+static void   dnodeSaveMnodeInfos();
+static void   dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg);
+static bool   dnodeReadDnodeCfg();
+static void   dnodeSaveDnodeCfg();
+static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
+static void   dnodeSendStatusMsg(void *handle, void *tmrId);
+
+static void    *tsDnodeTmr = NULL;
+static void    *tsStatusTimer = NULL;
+static uint32_t tsRebootTime;
+
+static SRpcIpSet     tsMnodeIpSet  = {0};
+static SDMMnodeInfos tsMnodeInfos = {0};
+static SDMDnodeCfg   tsDnodeCfg = {0};
 
 static int32_t  dnodeOpenVnodes();
 static void     dnodeCloseVnodes();
@@ -43,19 +71,63 @@ int32_t dnodeInitMgmt() {
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
 
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP,  dnodeProcessStatusRsp);
+  dnodeReadDnodeCfg();
+  tsRebootTime = taosGetTimestampSec();
+
+  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
+  if (tsDnodeTmr == NULL) {
+    dError("failed to init dnode timer");
+    return -1;
+  }
+  
+  if (!dnodeReadMnodeInfos()) {
+    memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet));
+    memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
+    tsMnodeIpSet.numOfIps = 1;
+    taosGetFqdnPortFromEp(tsFirst, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
+    tsMnodeIpSet.port[0] += TSDB_PORT_DNODEDNODE;
+    if (strcmp(tsSecond, tsFirst) != 0) {
+      tsMnodeIpSet.numOfIps = 2;
+      taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]);
+      tsMnodeIpSet.port[1] += TSDB_PORT_DNODEDNODE;
+    }
+  } else {
+    tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
+    tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
+    for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
+      taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
+      tsMnodeIpSet.port[i] += TSDB_PORT_DNODEDNODE;
+    }
+  }
+
   int32_t code = dnodeOpenVnodes();
   if (code != TSDB_CODE_SUCCESS) {
     return -1;
   }
 
+  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
+  
+  dPrint("dnode mgmt is initialized");
+ 
   return TSDB_CODE_SUCCESS;
 }
 
 void dnodeCleanupMgmt() {
+  if (tsStatusTimer != NULL) {
+    taosTmrStopA(&tsStatusTimer);
+    tsStatusTimer = NULL;
+  }
+
+  if (tsDnodeTmr != NULL) {
+    taosTmrCleanUp(tsDnodeTmr);
+    tsDnodeTmr = NULL;
+  }
+
   dnodeCloseVnodes();
 }
 
-void dnodeMgmt(SRpcMsg *pMsg) {
+void dnodeDispatchToDnodeMgmt(SRpcMsg *pMsg) {
   SRpcMsg rsp;
 
   if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
@@ -116,7 +188,7 @@ static int32_t dnodeOpenVnodes() {
 
   free(vnodeList);
 
-  dPrint("there are total vnodes:%d, failed to open:%d", numOfVnodes, failed);
+  dPrint("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, numOfVnodes-failed, failed);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -193,3 +265,326 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
   SMDCfgDnodeMsg *pCfg = (SMDCfgDnodeMsg *)pMsg->pCont;
   return taosCfgDynamicOptions(pCfg->config);
 }
+
+
+void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
+  dTrace("mgmt IP list is changed for ufp is called");
+  tsMnodeIpSet = *pIpSet;
+}
+
+void dnodeGetMnodeDnodeIpSet(void *ipSetRaw) {
+  SRpcIpSet *ipSet = ipSetRaw;
+  ipSet->numOfIps = tsMnodeInfos.nodeNum;
+  ipSet->inUse = tsMnodeInfos.inUse;
+  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) {
+    taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, ipSet->fqdn[i], &ipSet->port[i]);
+    ipSet->port[i] += TSDB_PORT_DNODEDNODE;
+  }
+}
+
+static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
+  if (pMsg->code != TSDB_CODE_SUCCESS) {
+    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    return;
+  }
+
+  SDMStatusRsp *pStatusRsp = pMsg->pCont;
+  SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes;
+  if (pMnodes->nodeNum <= 0) {
+    dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum);
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    return;
+  }
+
+  SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+  pCfg->numOfVnodes  = htonl(pCfg->numOfVnodes);
+  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
+  pCfg->dnodeId      = htonl(pCfg->dnodeId);
+
+  for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
+    SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
+    pMnodeInfo->nodeId   = htonl(pMnodeInfo->nodeId);
+  }
+
+  SDMVgroupAccess *pVgAcccess = pStatusRsp->vgAccess;
+  for (int32_t i = 0; i < pCfg->numOfVnodes; ++i) {
+    pVgAcccess[i].vgId = htonl(pVgAcccess[i].vgId);
+  }
+  
+  dnodeProcessModuleStatus(pCfg->moduleStatus);
+  dnodeUpdateDnodeCfg(pCfg);
+  dnodeUpdateMnodeInfos(pMnodes);
+  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+}
+
+static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
+  bool mnodesChanged = (memcmp(&tsMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
+  bool mnodesNotInit = (tsMnodeInfos.nodeNum == 0);
+  if (!(mnodesChanged || mnodesNotInit)) return;
+
+  memcpy(&tsMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
+
+  tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
+  tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
+  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
+    taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
+    tsMnodeIpSet.port[i] += TSDB_PORT_DNODEDNODE;
+  }
+
+  dPrint("mnodes is changed, nodeNum:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
+  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
+    dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
+  }
+
+  dnodeSaveMnodeInfos();
+  sdbUpdateSync();
+}
+
+static bool dnodeReadMnodeInfos() {
+  char ipFile[TSDB_FILENAME_LEN] = {0};
+  sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
+  FILE *fp = fopen(ipFile, "r");
+  if (!fp) {
+    dTrace("failed to read mnode mgmtIpList.json, file not exist");
+    return false;
+  }
+
+  bool  ret = false;
+  int   maxLen = 2000;
+  char *content = calloc(1, maxLen + 1);
+  int   len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    free(content);
+    fclose(fp);
+    dError("failed to read mnode mgmtIpList.json, content is null");
+    return false;
+  }
+
+  cJSON* root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read mnode mgmtIpList.json, invalid json format");
+    goto PARSE_OVER;
+  }
+
+  cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
+  if (!inUse || inUse->type != cJSON_Number) {
+    dError("failed to read mnode mgmtIpList.json, inUse not found");
+    goto PARSE_OVER;
+  }
+  tsMnodeInfos.inUse = inUse->valueint;
+
+  cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
+  if (!nodeNum || nodeNum->type != cJSON_Number) {
+    dError("failed to read mnode mgmtIpList.json, nodeNum not found");
+    goto PARSE_OVER;
+  }
+  tsMnodeInfos.nodeNum = nodeNum->valueint;
+
+  cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
+  if (!nodeInfos || nodeInfos->type != cJSON_Array) {
+    dError("failed to read mnode mgmtIpList.json, nodeInfos not found");
+    goto PARSE_OVER;
+  }
+
+  int size = cJSON_GetArraySize(nodeInfos);
+  if (size != tsMnodeInfos.nodeNum) {
+    dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched");
+    goto PARSE_OVER;
+  }
+
+  for (int i = 0; i < size; ++i) {
+    cJSON* nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
+    if (nodeInfo == NULL) continue;
+
+    cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
+    if (!nodeId || nodeId->type != cJSON_Number) {
+      dError("failed to read mnode mgmtIpList.json, nodeId not found");
+      goto PARSE_OVER;
+    }
+    tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
+
+    cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
+    if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
+      dError("failed to read mnode mgmtIpList.json, nodeName not found");
+      goto PARSE_OVER;
+    }
+    strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_FQDN_LEN);
+ }
+
+  ret = true;
+
+  dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
+  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
+    dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
+  }
+
+PARSE_OVER:
+  free(content);
+  cJSON_Delete(root);
+  fclose(fp);
+  return ret;
+}
+
+static void dnodeSaveMnodeInfos() {
+  char ipFile[TSDB_FILENAME_LEN] = {0};
+  sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
+  FILE *fp = fopen(ipFile, "w");
+  if (!fp) return;
+
+  int32_t len = 0;
+  int32_t maxLen = 2000;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsMnodeInfos.inUse);
+  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsMnodeInfos.nodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
+  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
+    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId);
+    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeEp);
+    if (i < tsMnodeInfos.nodeNum -1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");  
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");  
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n"); 
+
+  fwrite(content, 1, len, fp);
+  fclose(fp);
+  free(content);
+  
+  dPrint("save mnode iplist successed");
+}
+
+char *dnodeGetMnodeMasterEp() {
+  return tsMnodeInfos.nodeInfos[tsMnodeIpSet.inUse].nodeEp;
+}
+
+void* dnodeGetMnodeInfos() {
+  return &tsMnodeInfos;
+}
+
+static void dnodeSendStatusMsg(void *handle, void *tmrId) {
+  if (tsDnodeTmr == NULL) {
+    dError("dnode timer is already released");
+    return;
+  }
+
+  if (tsStatusTimer == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    dError("failed to start status timer");
+    return;
+  }
+
+  int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+  SDMStatusMsg *pStatus = rpcMallocCont(contLen);
+  if (pStatus == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    dError("failed to malloc status message");
+    return;
+  }
+
+  //strcpy(pStatus->dnodeName, tsDnodeName);
+  pStatus->version          = htonl(tsVersion);
+  pStatus->dnodeId          = htonl(tsDnodeCfg.dnodeId);
+  strcpy(pStatus->dnodeEp, tsLocalEp);
+  pStatus->lastReboot       = htonl(tsRebootTime);
+  pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
+  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
+  pStatus->diskAvailable    = tsAvailDataDirGB;
+  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
+  
+  vnodeBuildStatusMsg(pStatus);
+  contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
+  pStatus->openVnodes = htons(pStatus->openVnodes);
+  
+  SRpcMsg rpcMsg = {
+    .pCont   = pStatus,
+    .contLen = contLen,
+    .msgType = TSDB_MSG_TYPE_DM_STATUS
+  };
+
+  dnodeSendMsgToDnode(&tsMnodeIpSet, &rpcMsg);
+}
+
+static bool dnodeReadDnodeCfg() {
+  char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
+  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
+
+  FILE *fp = fopen(dnodeCfgFile, "r");
+  if (!fp) {
+    dTrace("failed to read dnodeCfg.json, file not exist");
+    return false;
+  }
+
+  bool  ret = false;
+  int   maxLen = 100;
+  char *content = calloc(1, maxLen + 1);
+  int   len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    free(content);
+    fclose(fp);
+    dError("failed to read dnodeCfg.json, content is null");
+    return false;
+  }
+
+  cJSON* root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read dnodeCfg.json, invalid json format");
+    goto PARSE_CFG_OVER;
+  }
+
+  cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+  if (!dnodeId || dnodeId->type != cJSON_Number) {
+    dError("failed to read dnodeCfg.json, dnodeId not found");
+    goto PARSE_CFG_OVER;
+  }
+  tsDnodeCfg.dnodeId = dnodeId->valueint;
+
+  ret = true;
+
+  dPrint("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId);
+
+PARSE_CFG_OVER:
+  free(content);
+  cJSON_Delete(root);
+  fclose(fp);
+  return ret;
+}
+
+static void dnodeSaveDnodeCfg() {
+  char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
+  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
+
+  FILE *fp = fopen(dnodeCfgFile, "w");
+  if (!fp) return;
+
+  int32_t len = 0;
+  int32_t maxLen = 100;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d\n", tsDnodeCfg.dnodeId);
+  len += snprintf(content + len, maxLen - len, "}\n"); 
+
+  fwrite(content, 1, len, fp);
+  fclose(fp);
+  free(content);
+  
+  dPrint("save dnodeId successed");
+}
+
+void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
+  if (tsDnodeCfg.dnodeId == 0) {
+    dPrint("dnodeId is set to %d", pCfg->dnodeId);  
+    tsDnodeCfg.dnodeId = pCfg->dnodeId;
+    dnodeSaveDnodeCfg();
+  }
+}
+
+int32_t dnodeGetDnodeId() {
+  return tsDnodeCfg.dnodeId;
+}
+

src/dnode/src/dnodeMnode.c

@@ -1,97 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "os.h"
-#include "taosmsg.h"
-#include "tglobal.h"
-#include "trpc.h"
-#include "dnode.h"
-#include "dnodeLog.h"
-#include "dnodeMgmt.h"
-#include "dnodeWrite.h"
-
-static void (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
-static void   dnodeProcessMsgFromMnode(SRpcMsg *pMsg);
-static void  *tsDnodeMnodeRpc = NULL;
-
-int32_t dnodeInitMnode() {
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeWrite;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = dnodeWrite;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeWrite;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeWrite;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeMgmt;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeMgmt;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeMgmt;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeMgmt;
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.localPort    = tsDnodeMnodePort;
-  rpcInit.label        = "DND-MS";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp          = dnodeProcessMsgFromMnode;
-  rpcInit.sessions     = 100;
-  rpcInit.connType     = TAOS_CONN_SERVER;
-  rpcInit.idleTime     = tsShellActivityTimer * 2000;
-
-  tsDnodeMnodeRpc = rpcOpen(&rpcInit);
-  if (tsDnodeMnodeRpc == NULL) {
-    dError("failed to init mnode rpc server");
-    return -1;
-  }
-
-  dPrint("mnode rpc server is opened");
-  return 0;
-}
-
-void dnodeCleanupMnode() {
-  if (tsDnodeMnodeRpc) {
-    rpcClose(tsDnodeMnodeRpc);
-    tsDnodeMnodeRpc = NULL;
-    dPrint("mnode rpc server is closed");
-  }
-}
-
-static void dnodeProcessMsgFromMnode(SRpcMsg *pMsg) {
-  SRpcMsg rspMsg;
-  rspMsg.handle  = pMsg->handle;
-  rspMsg.pCont   = NULL;
-  rspMsg.contLen = 0;
-
-  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
-    rspMsg.code = TSDB_CODE_NOT_READY;
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pMsg->pCont);
-    dTrace("thandle:%p, query msg is ignored since dnode not running", pMsg->handle);
-    return;
-  }
-
-  if (pMsg->pCont == NULL) {
-    rspMsg.code = TSDB_CODE_INVALID_MSG_LEN;
-    rpcSendResponse(&rspMsg);
-    return;
-  }
-
-  if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
-    (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
-  } else {
-    dError("%s is not processed in dnode mserver", taosMsg[pMsg->msgType]);
-    rspMsg.code = TSDB_CODE_MSG_NOT_PROCESSED;
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-

src/dnode/src/dnodeModule.c

@@ -17,10 +17,11 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tglobal.h"
+#include "trpc.h"
 #include "mnode.h"
 #include "http.h"
 #include "monitor.h"
-#include "dnodeLog.h"
+#include "dnodeInt.h"
 #include "dnodeModule.h"
 
 typedef struct {

src/dnode/src/dnodePeer.c

@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* this file is mainly responsible for the communication between DNODEs. Each 
+ * dnode works as both server and client. Dnode may send status, grant, config
+ * messages to mnode, mnode may send create/alter/drop table/vnode messages 
+ * to dnode. All theses messages are handled from here
+ */
+
+#include "os.h"
+#include "taosmsg.h"
+#include "tglobal.h"
+#include "trpc.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMgmt.h"
+#include "dnodeVWrite.h"
+#include "mnode.h"
+
+extern void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet);
+static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
+static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg);
+static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
+static void dnodeProcessRspFromDnode(SRpcMsg *pMsg);
+static void *tsDnodeServerRpc = NULL;
+static void *tsDnodeClientRpc = NULL;
+
+int32_t dnodeInitServer() {
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVnodeWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = dnodeDispatchToVnodeWriteQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVnodeWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVnodeWriteQueue;
+
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToDnodeMgmt; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToDnodeMgmt;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToDnodeMgmt;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToDnodeMgmt;
+
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = mgmtProcessReqMsgFromDnode;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mgmtProcessReqMsgFromDnode;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = mgmtProcessReqMsgFromDnode;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = mgmtProcessReqMsgFromDnode;
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = tsDnodeDnodePort;
+  rpcInit.label        = "DND-S";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = dnodeProcessReqMsgFromDnode;
+  rpcInit.sessions     = 100;
+  rpcInit.connType     = TAOS_CONN_SERVER;
+  rpcInit.idleTime     = tsShellActivityTimer * 1000;
+
+  tsDnodeServerRpc = rpcOpen(&rpcInit);
+  if (tsDnodeServerRpc == NULL) {
+    dError("failed to init inter-dnodes RPC server");
+    return -1;
+  }
+
+  dPrint("inter-dnodes RPC server is opened");
+  return 0;
+}
+
+void dnodeCleanupServer() {
+  if (tsDnodeServerRpc) {
+    rpcClose(tsDnodeServerRpc);
+    tsDnodeServerRpc = NULL;
+    dPrint("inter-dnodes RPC server is closed");
+  }
+}
+
+static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg) {
+  SRpcMsg rspMsg;
+  rspMsg.handle  = pMsg->handle;
+  rspMsg.pCont   = NULL;
+  rspMsg.contLen = 0;
+
+  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+    rspMsg.code = TSDB_CODE_NOT_READY;
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pMsg->pCont);
+    dTrace("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
+    return;
+  }
+
+  if (pMsg->pCont == NULL) {
+    rspMsg.code = TSDB_CODE_INVALID_MSG_LEN;
+    rpcSendResponse(&rspMsg);
+    return;
+  }
+ 
+  if (dnodeProcessReqMsgFp[pMsg->msgType]) {
+    (*dnodeProcessReqMsgFp[pMsg->msgType])(pMsg);
+  } else {
+    rspMsg.code = TSDB_CODE_MSG_NOT_PROCESSED;
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pMsg->pCont);
+    dTrace("RPC %p, message:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); 
+    return;
+
+  }
+}
+
+int32_t dnodeInitClient() {
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.label        = "DND-C";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = dnodeProcessRspFromDnode;
+  rpcInit.ufp          = dnodeUpdateIpSet;
+  rpcInit.sessions     = 100;
+  rpcInit.connType     = TAOS_CONN_CLIENT;
+  rpcInit.idleTime     = tsShellActivityTimer * 1000;
+  rpcInit.user         = "t";
+  rpcInit.ckey         = "key";
+  rpcInit.secret       = "secret";
+
+  tsDnodeClientRpc = rpcOpen(&rpcInit);
+  if (tsDnodeClientRpc == NULL) {
+    dError("failed to init mnode rpc client");
+    return -1;
+  }
+
+  dPrint("inter-dnodes rpc client is opened");
+  return 0;
+}
+
+void dnodeCleanupClient() {
+  if (tsDnodeClientRpc) {
+    rpcClose(tsDnodeClientRpc);
+    tsDnodeClientRpc = NULL;
+    dPrint("inter-dnodes rpc client is closed");
+  }
+}
+
+static void dnodeProcessRspFromDnode(SRpcMsg *pMsg) {
+
+  if (dnodeProcessRspMsgFp[pMsg->msgType]) {
+    (*dnodeProcessRspMsgFp[pMsg->msgType])(pMsg);
+  } else {
+    dError("RPC %p, msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]);
+  }
+
+  rpcFreeCont(pMsg->pCont);
+}
+
+void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
+  dnodeProcessRspMsgFp[msgType] = fp;
+}
+
+void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) {
+  rpcSendRequest(tsDnodeClientRpc, ipSet, rpcMsg);
+}

src/dnode/src/dnodeShell.c

@@ -22,9 +22,9 @@
 #include "tglobal.h"
 #include "http.h"
 #include "dnode.h"
-#include "dnodeLog.h"
+#include "dnodeInt.h"
-#include "dnodeRead.h"
+#include "dnodeVRead.h"
-#include "dnodeWrite.h"
+#include "dnodeVWrite.h"
 #include "dnodeShell.h"
 
 static void  (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
@@ -34,10 +34,43 @@ static void  * tsDnodeShellRpc = NULL;
 static int32_t tsDnodeQueryReqNum  = 0;
 static int32_t tsDnodeSubmitReqNum = 0;
 
+void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg);
+
 int32_t dnodeInitShell() {
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT]   = dnodeWrite;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY]    = dnodeRead;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY]  = dnodeDispatchToVnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_RETRIEVE] = dnodeRead;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH]  = dnodeDispatchToVnodeReadQueue;
+
+  // the following message shall be treated as mnode write
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]     = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]  = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]   = mgmtProcessMsgFromShell; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = mgmtProcessMsgFromShell; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]  = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]   = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]  = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]   = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]   = mgmtProcessMsgFromShell;
+
+  // the following message shall be treated as mnode query 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]      = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]  = mgmtProcessMsgFromShell; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= mgmtProcessMsgFromShell;   
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= mgmtProcessMsgFromShell;
 
   int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
   numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
@@ -48,12 +81,12 @@ int32_t dnodeInitShell() {
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort    = tsDnodeShellPort;
-  rpcInit.label        = "DND-shell";
+  rpcInit.label        = "SHELL";
   rpcInit.numOfThreads = numOfThreads;
   rpcInit.cfp          = dnodeProcessMsgFromShell;
   rpcInit.sessions     = TSDB_SESSIONS_PER_DNODE;
   rpcInit.connType     = TAOS_CONN_SERVER;
-  rpcInit.idleTime     = tsShellActivityTimer * 1500;
+  rpcInit.idleTime     = tsShellActivityTimer * 1000;
   rpcInit.afp          = dnodeRetrieveUserAuthInfo;
 
   tsDnodeShellRpc = rpcOpen(&rpcInit);
@@ -80,7 +113,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg) {
   rpcMsg.contLen = 0;
 
   if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
-    dError("RPC %p, shell msg is ignored since dnode not running", pMsg->handle);
+    dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
     rpcMsg.code = TSDB_CODE_NOT_READY;
     rpcSendResponse(&rpcMsg);
     rpcFreeCont(pMsg->pCont);
@@ -96,10 +129,11 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg) {
   if ( dnodeProcessShellMsgFp[pMsg->msgType] ) {
     (*dnodeProcessShellMsgFp[pMsg->msgType])(pMsg);
   } else {
-    dError("RPC %p, msg:%s from shell is not handled", pMsg->handle, taosMsg[pMsg->msgType]);
+    dError("RPC %p, shell msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]);
     rpcMsg.code = TSDB_CODE_MSG_NOT_PROCESSED;
     rpcSendResponse(&rpcMsg);
     rpcFreeCont(pMsg->pCont);
+    return;
   }
 }
 

src/dnode/src/dnodeVRead.c

@@ -21,9 +21,9 @@
 #include "trpc.h"
 #include "twal.h"
 #include "tglobal.h"
-#include "dnodeLog.h"
+#include "dnodeInt.h"
 #include "dnodeMgmt.h"
-#include "dnodeRead.h"
+#include "dnodeVRead.h"
 #include "vnode.h"
 
 typedef struct {
@@ -81,10 +81,12 @@ void dnodeCleanupRead() {
   }
 
   taosCloseQset(readQset);
+  free(readPool.readWorker);
+
   dPrint("dnode read is closed");
 }
 
-void dnodeRead(SRpcMsg *pMsg) {
+void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
   int32_t     queuedMsgNum = 0;
   int32_t     leftLen      = pMsg->contLen;
   char        *pCont       = (char *) pMsg->pCont;
@@ -97,7 +99,7 @@ void dnodeRead(SRpcMsg *pMsg) {
     pHead->vgId    = htonl(pHead->vgId);
     pHead->contLen = htonl(pHead->contLen);
 
-    if (pMsg->msgType == TSDB_MSG_TYPE_RETRIEVE) {
+    if (pMsg->msgType == TSDB_MSG_TYPE_FETCH) {
       pVnode = vnodeGetVnode(pHead->vgId);
     } else {
       pVnode = vnodeAccquireVnode(pHead->vgId);

src/dnode/src/dnodeVWrite.c

@@ -24,8 +24,8 @@
 #include "tglobal.h"
 #include "vnode.h"
 #include "tdataformat.h"
-#include "dnodeLog.h"
+#include "dnodeInt.h"
-#include "dnodeWrite.h"
+#include "dnodeVWrite.h"
 #include "dnodeMgmt.h"
 
 typedef struct {
@@ -82,10 +82,10 @@ void dnodeCleanupWrite() {
   dPrint("dnode write is closed");
 }
 
-void dnodeWrite(SRpcMsg *pMsg) {
+void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
   char *pCont = (char *)pMsg->pCont;
 
-  if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT || pMsg->msgType == TSDB_MSG_TYPE_MD_DROP_STABLE) {
+  if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
     SMsgDesc *pDesc = (SMsgDesc *)pCont;
     pDesc->numOfVnodes = htonl(pDesc->numOfVnodes);
     pCont += sizeof(SMsgDesc);

src/inc/dnode.h

@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+#include "trpc.h"
+
 typedef struct {
   int32_t queryReqNum;
   int32_t submitReqNum;
@@ -47,6 +49,10 @@ void     dnodeGetMnodeDnodeIpSet(void *ipSet);
 void *   dnodeGetMnodeInfos();
 int32_t  dnodeGetDnodeId();
 
+void     dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
+void     dnodeAddServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
+void     dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg);
+
 #ifdef __cplusplus
 }
 #endif

src/inc/mnode.h

@@ -26,6 +26,9 @@ void    mgmtCleanUpSystem();
 void    mgmtStopSystem();
 void    sdbUpdateSync();
 
+void   mgmtProcessMsgFromShell(SRpcMsg *rpcMsg);
+void   mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg);
+
 #ifdef __cplusplus
 }
 #endif

src/inc/taosdef.h

@@ -40,9 +40,10 @@ typedef int16_t VarDataLenT;
 
 #define varDataLen(v)       ((VarDataLenT *)(v))[0]
 #define varDataTLen(v)      (sizeof(VarDataLenT) + varDataLen(v))
-#define varDataVal(v)       ((void *)((char *)v + sizeof(VarDataLenT)))
+#define varDataVal(v)       ((void *)((char *)v + VARSTR_HEADER_SIZE))
 #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
 #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
+#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))
 
 // this data type is internally used only in 'in' query to hold the values
 #define TSDB_DATA_TYPE_ARRAY      (TSDB_DATA_TYPE_NCHAR + 1)
@@ -240,7 +241,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MULTI_METERMETA_MAX_NUM    100000  // maximum batch size allowed to load metermeta
 
 #define TSDB_MIN_CACHE_BLOCK_SIZE       1
-#define TSDB_MAX_CACHE_BLOCK_SIZE       1000000
+#define TSDB_MAX_CACHE_BLOCK_SIZE       10240   // 10GB for each vnode
 #define TSDB_DEFAULT_CACHE_BLOCK_SIZE   16
 
 #define TSDB_MIN_TOTAL_BLOCKS           2
@@ -279,9 +280,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MAX_COMP_LEVEL             2
 #define TSDB_DEFAULT_COMP_LEVEL         2
 
-#define TSDB_MIN_CLOG_LEVEL             0
+#define TSDB_MIN_WAL_LEVEL             0
-#define TSDB_MAX_CLOG_LEVEL             2
+#define TSDB_MAX_WAL_LEVEL             2
-#define TSDB_DEFAULT_CLOG_LEVEL         2
+#define TSDB_DEFAULT_WAL_LEVEL         2
 
 #define TSDB_MIN_REPLICA_NUM            1
 #define TSDB_MAX_REPLICA_NUM            3
@@ -338,11 +339,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MAX_NORMAL_TABLES 1000
 #define TSDB_MAX_CHILD_TABLES  100000
 
-#define TSDB_PORT_MNODESHELL 0
+#define TSDB_PORT_DNODESHELL 0 
-#define TSDB_PORT_DNODESHELL 5
+#define TSDB_PORT_DNODEDNODE 5 
-#define TSDB_PORT_DNODEMNODE 10
+#define TSDB_PORT_SYNC       10 
-#define TSDB_PORT_MNODEDNODE 15 
-#define TSDB_PORT_SYNC 20
 
 #define TAOS_QTYPE_RPC      0
 #define TAOS_QTYPE_FWD      1

src/inc/taosmsg.h

@@ -39,56 +39,70 @@ enum {
   TSDB_MESSAGE_NULL = 0,
 #endif
 
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_REG, "registration" )             	// 1
+// message from client to dnode
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SUBMIT, "submit" )	                // 3
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SUBMIT, "submit" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_QUERY, "query" )	                  // 5
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_QUERY, "query" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE, "retrieve" )	            // 7
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_FETCH, "fetch" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY0, "dummy0" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY1, "dummy1" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY2, "dummy2" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY3, "dummy3" )
 
 // message from mnode to dnode
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CREATE_TABLE, "create-table" )	  // 9
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CREATE_TABLE, "create-table" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_TABLE, "drop-table" )	      // 11
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_TABLE, "drop-table" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_TABLE, "alter-table" )	    // 13
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_TABLE, "alter-table" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CREATE_VNODE, "create-vnode" )	  // 15
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CREATE_VNODE, "create-vnode" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_VNODE, "drop-vnode" )	      // 17
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_VNODE, "drop-vnode" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_STABLE, "drop-stable" )	    // 19
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_STABLE, "drop-stable" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_STREAM, "alter-stream" )	  // 21
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_STREAM, "alter-stream" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CONFIG_DNODE, "config-dnode" )	  // 23
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CONFIG_DNODE, "config-dnode" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY4, "dummy4" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY5, "dummy5" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY6, "dummy6" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY7, "dummy7" )
 
 // message from client to mnode
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CONNECT, "connect" )	            // 31
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CONNECT, "connect" )	 
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_ACCT, "create-acct" )	    // 33
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_ACCT, "create-acct" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_ACCT, "alter-acct" )	      // 35
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_ACCT, "alter-acct" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_ACCT, "drop-acct" )	        // 37
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_ACCT, "drop-acct" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_USER, "create-user" )	    // 39
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_USER, "create-user" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_USER, "alter-user" )	      // 41
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_USER, "alter-user" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_USER, "drop-user" )         // 43
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_USER, "drop-user" ) 
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DNODE, "create-dnode" )	  // 45
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DNODE, "create-dnode" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DNODE, "drop-dnode" )     	// 47
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DNODE, "drop-dnode" )   
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DB, "create-db" )	        // 49
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DB, "create-db" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )	            // 51
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )	  
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )	              // 53
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )	 
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_DB, "alter-db" )	          // 55
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_DB, "alter-db" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TABLE, "create-table" )	  // 57
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TABLE, "create-table" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_TABLE, "drop-table" )	      // 59
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_TABLE, "drop-table" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_TABLE, "alter-table" )	    // 61
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_TABLE, "alter-table" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_TABLE_META, "table-meta" )	      // 63
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_TABLE_META, "table-meta" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_STABLE_VGROUP, "stable-vgroup" )	// 65
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_STABLE_VGROUP, "stable-vgroup" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_TABLES_META, "tables-meta" )	    // 67
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_TABLES_META, "tables-meta" )	  
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_STREAM, "alter-stream" )	  // 69
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_STREAM, "alter-stream" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_SHOW, "show" )	                  // 71
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_SHOW, "show" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_QUERY, "kill-query" )	      // 73
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_RETRIEVE, "retrieve" )     
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_STREAM, "kill-stream" )	    // 75
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_QUERY, "kill-query" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_CONN, "kill-conn" )	        // 77
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_STREAM, "kill-stream" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_HEARTBEAT, "heartbeat" )	        // 79
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_CONN, "kill-conn" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CONFIG_DNODE, "cm-config-dnode" ) 
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_HEARTBEAT, "heartbeat" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY8, "dummy8" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY10, "dummy10" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY11, "dummy11" )
 
 // message from dnode to mnode
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_TABLE, "config-table" )	  // 91
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_TABLE, "config-table" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_VNODE, "config-vnode" )	  // 93
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_VNODE, "config-vnode" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_STATUS, "status" )	              // 95
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_STATUS, "status" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_GRANT, "grant" )	                // 97
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_GRANT, "grant" )
-
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SDB_SYNC, "sdb-sync" )	            // 101
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SDB_FORWARD, "sdb-forward" )	      // 103
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
 
 #ifndef TAOS_MESSAGE_C
   TSDB_MSG_TYPE_MAX  // 105
@@ -96,9 +110,6 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SDB_FORWARD, "sdb-forward" )	      // 10
 
 };
 
-#define TSDB_MSG_TYPE_CM_CONFIG_DNODE     TSDB_MSG_TYPE_MD_CONFIG_DNODE
-#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP
-
 // IE type
 #define TSDB_IE_TYPE_SEC 1
 #define TSDB_IE_TYPE_META 2
@@ -306,10 +317,6 @@ typedef struct {
   int8_t flag;
 } SCMCreateUserMsg, SCMAlterUserMsg;
 
-typedef struct {
-  char db[TSDB_TABLE_ID_LEN + 1];
-} SMgmtHead;
-
 typedef struct {
   int32_t  contLen;
   int32_t  vgId;
@@ -319,6 +326,7 @@ typedef struct {
 } SMDDropTableMsg;
 
 typedef struct {
+  int32_t contLen;
   int32_t vgId;
   int64_t uid;
   char    tableId[TSDB_TABLE_ID_LEN + 1];
@@ -358,7 +366,7 @@ typedef struct SExprInfo {
   struct tExprNode* pExpr;
   int16_t     bytes;
   int16_t     type;
-  int16_t     interResBytes;
+  int16_t     interBytes;
 } SExprInfo;
 
 typedef struct SColumnFilterInfo {
@@ -475,9 +483,9 @@ typedef struct {
 typedef struct {
   char     acct[TSDB_USER_LEN + 1];
   char     db[TSDB_DB_NAME_LEN + 1];
-  int32_t  maxSessions;
+  int32_t  maxTables;
   int32_t  cacheBlockSize; //MB
-  int32_t  totalBlocks;
+  int32_t  numOfBlocks;
   int32_t  daysPerFile;
   int32_t  daysToKeep1;
   int32_t  daysToKeep2;
@@ -486,7 +494,7 @@ typedef struct {
   int32_t  minRowsPerFileBlock;
   int32_t  maxRowsPerFileBlock;
   int8_t   compression;
-  int8_t   commitLog;
+  int8_t   walLevel;
   int8_t   replications;
   uint8_t  precision;   // time resolution
   int8_t   ignoreExist;
@@ -565,7 +573,7 @@ typedef struct {
   int32_t  commitTime;
   int8_t   precision;
   int8_t   compression;
-  int8_t   commitLog;
+  int8_t   walLevel;
   int8_t   replications;
   int8_t   wals;
   int8_t   quorum;

src/inc/tsync.h

@@ -54,8 +54,8 @@ typedef struct {
   int       role[TAOS_SYNC_MAX_REPLICA];  
 } SNodesRole;
  
-// if name is null, get the file from index or after, used by master
+// if name is empty(name[0] is zero), get the file from index or after, used by master
-// if name is provided, get the named file at the specified index, used by unsynced node
+// if name is provided(name[0] is not zero), get the named file at the specified index, used by unsynced node
 // it returns the file magic number and size, if file not there, magic shall be 0.
 typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); 
 
@@ -72,6 +72,9 @@ typedef void     (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
 // when role is changed, call this to notify app
 typedef void     (*FNotifyRole)(void *ahandle, int8_t role);
 
+// when data file is synced successfully, notity app
+typedef void     (*FNotifyFileSynced)(void *ahandle);
+
 typedef struct {
   int32_t    vgId;      // vgroup ID
   uint64_t   version;   // initial version
@@ -84,7 +87,7 @@ typedef struct {
   FWriteToCache   writeToCache;
   FConfirmForward confirmForward;
   FNotifyRole     notifyRole;
-
+  FNotifyFileSynced notifyFileSynced;
 } SSyncInfo;
 
 typedef void* tsync_h;

src/inc/ttokendef.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSQLDEF_H
+#ifndef TDENGINE_TTOKENDEF_H
-#define TDENGINE_TSQLDEF_H
+#define TDENGINE_TTOKENDEF_H
 
 #define TK_ID                               1
 #define TK_BOOL                             2
@@ -101,123 +101,125 @@
 #define TK_CONNS                           83
 #define TK_STATE                           84
 #define TK_KEEP                            85
-#define TK_CACHE                           86
+#define TK_MAXTABLES                       86
-#define TK_REPLICA                         87
+#define TK_CACHE                           87
-#define TK_DAYS                            88
+#define TK_REPLICA                         88
-#define TK_ROWS                            89
+#define TK_DAYS                            89
-#define TK_ABLOCKS                         90
+#define TK_MINROWS                         90
-#define TK_TBLOCKS                         91
+#define TK_MAXROWS                         91
-#define TK_CTIME                           92
+#define TK_BLOCKS                          92
-#define TK_CLOG                            93
+#define TK_CTIME                           93
-#define TK_COMP                            94
+#define TK_WAL                             94
-#define TK_PRECISION                       95
+#define TK_COMP                            95
-#define TK_LP                              96
+#define TK_PRECISION                       96
-#define TK_RP                              97
+#define TK_LP                              97
-#define TK_TAGS                            98
+#define TK_RP                              98
-#define TK_USING                           99
+#define TK_TAGS                            99
-#define TK_AS                             100
+#define TK_USING                          100
-#define TK_COMMA                          101
+#define TK_AS                             101
-#define TK_NULL                           102
+#define TK_COMMA                          102
-#define TK_SELECT                         103
+#define TK_NULL                           103
-#define TK_UNION                          104
+#define TK_SELECT                         104
-#define TK_ALL                            105
+#define TK_UNION                          105
-#define TK_FROM                           106
+#define TK_ALL                            106
-#define TK_VARIABLE                       107
+#define TK_FROM                           107
-#define TK_INTERVAL                       108
+#define TK_VARIABLE                       108
-#define TK_FILL                           109
+#define TK_INTERVAL                       109
-#define TK_SLIDING                        110
+#define TK_FILL                           110
-#define TK_ORDER                          111
+#define TK_SLIDING                        111
-#define TK_BY                             112
+#define TK_ORDER                          112
-#define TK_ASC                            113
+#define TK_BY                             113
-#define TK_DESC                           114
+#define TK_ASC                            114
-#define TK_GROUP                          115
+#define TK_DESC                           115
-#define TK_HAVING                         116
+#define TK_GROUP                          116
-#define TK_LIMIT                          117
+#define TK_HAVING                         117
-#define TK_OFFSET                         118
+#define TK_LIMIT                          118
-#define TK_SLIMIT                         119
+#define TK_OFFSET                         119
-#define TK_SOFFSET                        120
+#define TK_SLIMIT                         120
-#define TK_WHERE                          121
+#define TK_SOFFSET                        121
-#define TK_NOW                            122
+#define TK_WHERE                          122
-#define TK_RESET                          123
+#define TK_NOW                            123
-#define TK_QUERY                          124
+#define TK_RESET                          124
-#define TK_ADD                            125
+#define TK_QUERY                          125
-#define TK_COLUMN                         126
+#define TK_ADD                            126
-#define TK_TAG                            127
+#define TK_COLUMN                         127
-#define TK_CHANGE                         128
+#define TK_TAG                            128
-#define TK_SET                            129
+#define TK_CHANGE                         129
-#define TK_KILL                           130
+#define TK_SET                            130
-#define TK_CONNECTION                     131
+#define TK_KILL                           131
-#define TK_COLON                          132
+#define TK_CONNECTION                     132
-#define TK_STREAM                         133
+#define TK_COLON                          133
-#define TK_ABORT                          134
+#define TK_STREAM                         134
-#define TK_AFTER                          135
+#define TK_ABORT                          135
-#define TK_ATTACH                         136
+#define TK_AFTER                          136
-#define TK_BEFORE                         137
+#define TK_ATTACH                         137
-#define TK_BEGIN                          138
+#define TK_BEFORE                         138
-#define TK_CASCADE                        139
+#define TK_BEGIN                          139
-#define TK_CLUSTER                        140
+#define TK_CASCADE                        140
-#define TK_CONFLICT                       141
+#define TK_CLUSTER                        141
-#define TK_COPY                           142
+#define TK_CONFLICT                       142
-#define TK_DEFERRED                       143
+#define TK_COPY                           143
-#define TK_DELIMITERS                     144
+#define TK_DEFERRED                       144
-#define TK_DETACH                         145
+#define TK_DELIMITERS                     145
-#define TK_EACH                           146
+#define TK_DETACH                         146
-#define TK_END                            147
+#define TK_EACH                           147
-#define TK_EXPLAIN                        148
+#define TK_END                            148
-#define TK_FAIL                           149
+#define TK_EXPLAIN                        149
-#define TK_FOR                            150
+#define TK_FAIL                           150
-#define TK_IGNORE                         151
+#define TK_FOR                            151
-#define TK_IMMEDIATE                      152
+#define TK_IGNORE                         152
-#define TK_INITIALLY                      153
+#define TK_IMMEDIATE                      153
-#define TK_INSTEAD                        154
+#define TK_INITIALLY                      154
-#define TK_MATCH                          155
+#define TK_INSTEAD                        155
-#define TK_KEY                            156
+#define TK_MATCH                          156
-#define TK_OF                             157
+#define TK_KEY                            157
-#define TK_RAISE                          158
+#define TK_OF                             158
-#define TK_REPLACE                        159
+#define TK_RAISE                          159
-#define TK_RESTRICT                       160
+#define TK_REPLACE                        160
-#define TK_ROW                            161
+#define TK_RESTRICT                       161
-#define TK_STATEMENT                      162
+#define TK_ROW                            162
-#define TK_TRIGGER                        163
+#define TK_STATEMENT                      163
-#define TK_VIEW                           164
+#define TK_TRIGGER                        164
-#define TK_COUNT                          165
+#define TK_VIEW                           165
-#define TK_SUM                            166
+#define TK_COUNT                          166
-#define TK_AVG                            167
+#define TK_SUM                            167
-#define TK_MIN                            168
+#define TK_AVG                            168
-#define TK_MAX                            169
+#define TK_MIN                            169
-#define TK_FIRST                          170
+#define TK_MAX                            170
-#define TK_LAST                           171
+#define TK_FIRST                          171
-#define TK_TOP                            172
+#define TK_LAST                           172
-#define TK_BOTTOM                         173
+#define TK_TOP                            173
-#define TK_STDDEV                         174
+#define TK_BOTTOM                         174
-#define TK_PERCENTILE                     175
+#define TK_STDDEV                         175
-#define TK_APERCENTILE                    176
+#define TK_PERCENTILE                     176
-#define TK_LEASTSQUARES                   177
+#define TK_APERCENTILE                    177
-#define TK_HISTOGRAM                      178
+#define TK_LEASTSQUARES                   178
-#define TK_DIFF                           179
+#define TK_HISTOGRAM                      179
-#define TK_SPREAD                         180
+#define TK_DIFF                           180
-#define TK_TWA                            181
+#define TK_SPREAD                         181
-#define TK_INTERP                         182
+#define TK_TWA                            182
-#define TK_LAST_ROW                       183
+#define TK_INTERP                         183
-#define TK_RATE                           184
+#define TK_LAST_ROW                       184
-#define TK_IRATE                          185
+#define TK_RATE                           185
-#define TK_SUM_RATE                       186
+#define TK_IRATE                          186
-#define TK_SUM_IRATE                      187
+#define TK_SUM_RATE                       187
-#define TK_AVG_RATE                       188
+#define TK_SUM_IRATE                      188
-#define TK_AVG_IRATE                      189
+#define TK_AVG_RATE                       189
-#define TK_SEMI                           190
+#define TK_AVG_IRATE                      190
-#define TK_NONE                           191
+#define TK_TBID                           191
-#define TK_PREV                           192
+#define TK_SEMI                           192
-#define TK_LINEAR                         193
+#define TK_NONE                           193
-#define TK_IMPORT                         194
+#define TK_PREV                           194
-#define TK_METRIC                         195
+#define TK_LINEAR                         195
-#define TK_TBNAME                         196
+#define TK_IMPORT                         196
-#define TK_JOIN                           197
+#define TK_METRIC                         197
-#define TK_METRICS                        198
+#define TK_TBNAME                         198
-#define TK_STABLE                         199
+#define TK_JOIN                           199
-#define TK_INSERT                         200
+#define TK_METRICS                        200
-#define TK_INTO                           201
+#define TK_STABLE                         201
-#define TK_VALUES                         202
+#define TK_INSERT                         202
+#define TK_INTO                           203
+#define TK_VALUES                         204
 
 #endif
 

src/inc/twal.h

@@ -34,7 +34,7 @@ typedef struct {
 } SWalHead;
 
 typedef struct {
-  int8_t    commitLog; // commitLog
+  int8_t    walLevel;  // wal level
   int8_t    wals;      // number of WAL files;
   int8_t    keep;      // keep the wal file when closed
 } SWalCfg;

src/kit/shell/src/shellEngine.c

@@ -60,12 +60,10 @@ TAOS *shellInit(struct arguments *args) {
 
   taos_init();
   /*
-   * set tsMetricMetaKeepTimer = 3000ms
+   * set tsTableMetaKeepTimer = 3000ms
-   * set tsMeterMetaKeepTimer = 3000ms
    * means not save cache in shell
    */
-  tsMetricMetaKeepTimer = 3;
+  tsTableMetaKeepTimer = 3000;
-  tsMeterMetaKeepTimer = 3000;
 
   // Connect to the database.
   TAOS *con = taos_connect(args->host, args->user, args->password, args->database, args->port);

src/kit/shell/src/shellImport.c

@@ -229,7 +229,7 @@ static void shellRunImportThreads(struct arguments* args)
     ShellThreadObj *pThread = threadObj + t;
     pThread->threadIndex = t;
     pThread->totalThreads = args->threadNum;
-    pThread->taos = taos_connect(args->host, args->user, args->password, args->database, tsMnodeShellPort);
+    pThread->taos = taos_connect(args->host, args->user, args->password, args->database, tsDnodeShellPort);
     if (pThread->taos == NULL) {
       fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taos));
       exit(0);

src/kit/shell/src/shellLinux.c

@@ -63,7 +63,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
       break;
     case 'P':
       if (arg) {
-        arguments->port = atoi(arg);
+        tsDnodeShellPort = atoi(arg);
       } else {
         fprintf(stderr, "Invalid port\n");
         return -1;

src/kit/taosdemo/taosdemo.c

@@ -634,7 +634,7 @@ void *readMetric(void *sarg) {
   fprintf(fp, "Querying On %d records:\n", totalData);
 
   for (int j = 0; j < n; j++) {
-    char condition[BUFFER_SIZE] = "\0";
+    char condition[BUFFER_SIZE - 30] = "\0";
     char tempS[BUFFER_SIZE] = "\0";
 
     int m = 10 < num_of_tables ? 10 : num_of_tables;

src/mnode/inc/mgmtDef.h

@@ -151,7 +151,7 @@ typedef struct {
   int32_t commitTime;
   int8_t  precision;
   int8_t  compression;
-  int8_t  commitLog;
+  int8_t  walLevel;
   int8_t  replications;
   int8_t  reserved[16];
 } SDbCfg;

src/mnode/inc/mgmtInt.h

@@ -23,43 +23,23 @@ extern "C" {
 #include "tlog.h"
 #include "monitor.h"
 
-extern int32_t mdebugFlag;
+extern int32_t mDebugFlag;
 extern int32_t sdbDebugFlag;
 
 // mnode log function
-#define mError(...)                          \
+#define mError(...) if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND ", 255, __VA_ARGS__); }
-  if (mdebugFlag & DEBUG_ERROR) {            \
+#define mWarn(...)  if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  MND ", mDebugFlag, __VA_ARGS__); }
-    taosPrintLog("ERROR MND ", 255, __VA_ARGS__); \
+#define mTrace(...) if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }
-  }
+#define mPrint(...)                               { taosPrintLog("MND ", 255, __VA_ARGS__); }
-#define mWarn(...)                                  \
-  if (mdebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  MND ", mdebugFlag, __VA_ARGS__); \
-  }
-#define mTrace(...)                           \
-  if (mdebugFlag & DEBUG_TRACE) {             \
-    taosPrintLog("MND ", mdebugFlag, __VA_ARGS__); \
-  }
-#define mPrint(...) \
-  { taosPrintLog("MND ", 255, __VA_ARGS__); }
 
 #define mLError(...) monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__)
 #define mLWarn(...)  monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__)
 #define mLPrint(...) monitorSaveLog(0, __VA_ARGS__); mPrint(__VA_ARGS__)
 
-#define sdbError(...)                            \
+#define sdbError(...) if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND-SDB ", 255, __VA_ARGS__); }
-  if (sdbDebugFlag & DEBUG_ERROR) {              \
+#define sdbWarn(...)  if (sdbDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  MND-SDB ", sdbDebugFlag, __VA_ARGS__); }
-    taosPrintLog("ERROR MND-SDB ", 255, __VA_ARGS__); \
+#define sdbTrace(...) if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("MND-SDB ", sdbDebugFlag, __VA_ARGS__);}
-  }
+#define sdbPrint(...)                                 { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); }
-#define sdbWarn(...)                                      \
-  if (sdbDebugFlag & DEBUG_WARN) {                        \
-    taosPrintLog("WARN  MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
-  }
-#define sdbTrace(...)                               \
-  if (sdbDebugFlag & DEBUG_TRACE) {                 \
-    taosPrintLog("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
-  }
-#define sdbPrint(...) \
-  { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); }
 
 #define sdbLError(...) monitorSaveLog(2, __VA_ARGS__); sdbError(__VA_ARGS__)
 #define sdbLWarn(...)  monitorSaveLog(1, __VA_ARGS__); sdbWarn(__VA_ARGS__)

src/mnode/inc/mgmtServer.h

@@ -13,15 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_MNODE_H
+#ifndef TDENGINE_MGMT_DSERVER_H
-#define TDENGINE_DNODE_MNODE_H
+#define TDENGINE_MGMT_DSERVER_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int32_t dnodeInitMnode();
+int32_t mgmtInitServer();
-void    dnodeCleanupMnode();
+void    mgmtCleanupServer();
 
 #ifdef __cplusplus
 }

src/mnode/src/mgmtAcct.c

@@ -20,7 +20,7 @@
 #include "tutil.h"
 #include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtAcct.h"
 #include "mgmtDb.h"
 #include "mgmtSdb.h"

src/mnode/src/mgmtBalance.c

@@ -19,7 +19,7 @@
 #include "tbalance.h"
 #include "tglobal.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtMnode.h"
 #include "mgmtDnode.h"
 #include "mgmtVgroup.h"

src/mnode/src/mgmtDClient.c

@@ -1,87 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taoserror.h"
-#include "tsched.h"
-#include "tsystem.h"
-#include "tutil.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "tgrant.h"
-#include "mgmtDef.h"
-#include "mgmtLog.h"
-#include "mgmtMnode.h"
-#include "mgmtDb.h"
-#include "mgmtDnode.h"
-#include "mgmtProfile.h"
-#include "mgmtShell.h"
-#include "mgmtTable.h"
-#include "mgmtVgroup.h"
-
-static void   mgmtProcessRspFromDnode(SRpcMsg *rpcMsg);
-static void (*mgmtProcessDnodeRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
-static void  *tsMgmtDClientRpc = NULL;
-
-int32_t mgmtInitDClient() {
-  SRpcInit rpcInit = {0};
-  rpcInit.localPort    = 0;
-  rpcInit.label        = "MND-DC";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp          = mgmtProcessRspFromDnode;
-  rpcInit.sessions     = 100;
-  rpcInit.connType     = TAOS_CONN_CLIENT;
-  rpcInit.idleTime     = tsShellActivityTimer * 1000;
-  rpcInit.user         = "mgmtDClient";
-  rpcInit.ckey         = "key";
-  rpcInit.secret       = "secret";
-
-  tsMgmtDClientRpc = rpcOpen(&rpcInit);
-  if (tsMgmtDClientRpc == NULL) {
-    mError("failed to init client connection to dnode");
-    return -1;
-  }
-
-  mPrint("client connection to dnode is opened");
-  return 0;
-}
-
-void mgmtCleanupDClient() {
-  if (tsMgmtDClientRpc) {
-    rpcClose(tsMgmtDClientRpc);
-    tsMgmtDClientRpc = NULL;
-  }
-}
-
-void mgmtAddDClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
-  mgmtProcessDnodeRspFp[msgType] = fp;
-}
-
-void mgmtSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) {
-  rpcSendRequest(tsMgmtDClientRpc, ipSet, rpcMsg);
-}
-
-static void mgmtProcessRspFromDnode(SRpcMsg *rpcMsg) {
-  if (mgmtProcessDnodeRspFp[rpcMsg->msgType]) {
-    (*mgmtProcessDnodeRspFp[rpcMsg->msgType])(rpcMsg);
-  } else {
-    mError("%s is not processed in mgmt dclient", taosMsg[rpcMsg->msgType]);
-    SRpcMsg rpcRsp = {.pCont = 0, .contLen = 0, .code = TSDB_CODE_OPS_NOT_SUPPORT, .handle = rpcMsg->handle};
-    rpcSendResponse(&rpcRsp);
-  }
-
-  rpcFreeCont(rpcMsg->pCont);
-}

src/mnode/src/mgmtDb.c

@@ -25,7 +25,7 @@
 #include "tbalance.h"
 #include "tdataformat.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtAcct.h"
 #include "mgmtDb.h"
 #include "mgmtDnode.h"
@@ -251,8 +251,8 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_INVALID_OPTION;
   }
 
-  if (pCfg->commitLog < TSDB_MIN_CLOG_LEVEL || pCfg->commitLog > TSDB_MAX_CLOG_LEVEL) {
+  if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) {
-    mError("invalid db option commitLog:%d, only 0-2 allowed", pCfg->commitLog);
+    mError("invalid db option walLevel:%d, only 0-2 allowed", pCfg->walLevel);
     return TSDB_CODE_INVALID_OPTION;
   }
 
@@ -274,8 +274,8 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
 
 static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = tsCacheBlockSize;
-  if (pCfg->totalBlocks < 0) pCfg->totalBlocks = tsTotalBlocks;
+  if (pCfg->totalBlocks < 0) pCfg->totalBlocks = tsBlocksPerVnode;
-  if (pCfg->maxTables < 0) pCfg->maxTables = tsTablesPerVnode;
+  if (pCfg->maxTables < 0) pCfg->maxTables = tsMaxTablePerVnode;
   if (pCfg->daysPerFile < 0) pCfg->daysPerFile = tsDaysPerFile;
   if (pCfg->daysToKeep < 0) pCfg->daysToKeep = tsDaysToKeep;
   if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep;
@@ -285,7 +285,7 @@ static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime;
   if (pCfg->precision < 0) pCfg->precision = tsTimePrecision;
   if (pCfg->compression < 0) pCfg->compression = tsCompression;
-  if (pCfg->commitLog < 0) pCfg->commitLog = tsCommitLog;
+  if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
   if (pCfg->replications < 0) pCfg->replications = tsReplications;
 }
 
@@ -312,8 +312,8 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
   pDb->createdTime = taosGetTimestampMs(); 
   pDb->cfg = (SDbCfg) {
     .cacheBlockSize      = pCreate->cacheBlockSize,
-    .totalBlocks         = pCreate->totalBlocks,
+    .totalBlocks         = pCreate->numOfBlocks,
-    .maxTables           = pCreate->maxSessions,
+    .maxTables           = pCreate->maxTables,
     .daysPerFile         = pCreate->daysPerFile,
     .daysToKeep          = pCreate->daysToKeep,
     .daysToKeep1         = pCreate->daysToKeep1,
@@ -323,7 +323,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
     .commitTime          = pCreate->commitTime,
     .precision           = pCreate->precision,
     .compression         = pCreate->compression,
-    .commitLog           = pCreate->commitLog,
+    .walLevel            = pCreate->walLevel,
     .replications        = pCreate->replications
   };
 
@@ -491,7 +491,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
 #endif
     pShow->bytes[cols] = 4;
     pSchema[cols].type = TSDB_DATA_TYPE_INT;
-    strcpy(pSchema[cols].name, "tables");
+    strcpy(pSchema[cols].name, "maxtables");
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
@@ -521,13 +521,13 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
 
     pShow->bytes[cols] = 4;
     pSchema[cols].type = TSDB_DATA_TYPE_INT;
-    strcpy(pSchema[cols].name, "ctime(s)");
+    strcpy(pSchema[cols].name, "ctime(Sec.)");
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
     pShow->bytes[cols] = 1;
     pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
-    strcpy(pSchema[cols].name, "clog");
+    strcpy(pSchema[cols].name, "wallevel");
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
@@ -659,7 +659,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      *(int8_t *)pWrite = pDb->cfg.commitLog;
+      *(int8_t *)pWrite = pDb->cfg.walLevel;
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -677,9 +677,11 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     if (pDb->status == TSDB_DB_STATUS_READY) {
-      STR_WITH_SIZE_TO_VARSTR(pWrite, "ready", 5);
+      const char *src = "ready";
+      STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src));
     } else {
-      STR_WITH_SIZE_TO_VARSTR(pWrite, "dropping", 8);
+      const char *src = "dropping";
+      STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src));
     }
     cols++;
 
@@ -728,9 +730,10 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
 
 static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
   SCMCreateDbMsg *pCreate = pMsg->pCont;
-  pCreate->maxSessions     = htonl(pCreate->maxSessions);
+  
+  pCreate->maxTables       = htonl(pCreate->maxTables);
   pCreate->cacheBlockSize  = htonl(pCreate->cacheBlockSize);
-  pCreate->totalBlocks     = htonl(pCreate->totalBlocks);
+  pCreate->numOfBlocks     = htonl(pCreate->numOfBlocks);
   pCreate->daysPerFile     = htonl(pCreate->daysPerFile);
   pCreate->daysToKeep      = htonl(pCreate->daysToKeep);
   pCreate->daysToKeep1     = htonl(pCreate->daysToKeep1);
@@ -757,14 +760,15 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
 static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
   SDbCfg  newCfg = pDb->cfg;
   int32_t cacheBlockSize = htonl(pAlter->daysToKeep);
-  int32_t totalBlocks = htonl(pAlter->totalBlocks);
+  int32_t totalBlocks = htonl(pAlter->numOfBlocks);
-  int32_t maxTables = htonl(pAlter->maxSessions);
+  int32_t maxTables   = htonl(pAlter->maxTables);
-  int32_t daysToKeep = htonl(pAlter->daysToKeep);
+  int32_t daysToKeep  = htonl(pAlter->daysToKeep);
   int32_t daysToKeep1 = htonl(pAlter->daysToKeep1);
   int32_t daysToKeep2 = htonl(pAlter->daysToKeep2);
   int8_t  compression = pAlter->compression;
   int8_t  replications = pAlter->replications;
-
+  int8_t  walLevel   = pAlter->walLevel;
+  
   terrno = TSDB_CODE_SUCCESS;
 
   if (cacheBlockSize > 0 && cacheBlockSize != pDb->cfg.cacheBlockSize) {
@@ -801,7 +805,7 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     newCfg.daysToKeep2 = daysToKeep2;
   }
 
-  if (compression > 0 && compression != pDb->cfg.compression) {
+  if (compression >= 0 && compression != pDb->cfg.compression) {
     mTrace("db:%s, compression:%d change to %d", pDb->name, pDb->cfg.compression, compression);
     newCfg.compression = compression;
   }
@@ -809,15 +813,20 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
   if (replications > 0 && replications != pDb->cfg.replications) {
     mTrace("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications);
     newCfg.replications = replications;
-  } 
 
-  if (replications > mgmtGetDnodesNum()) {
+    if (replications > mgmtGetDnodesNum()) {
-    mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications);
+      mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications);
-    terrno = TSDB_CODE_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_NO_ENOUGH_DNODES;
+    }
+
+    if (pDb->cfg.replications - replications >= 2) {
+      mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications);
+      terrno = TSDB_CODE_INVALID_OPTION;
+    }
   }
 
-  if (pDb->cfg.replications - replications >= 2) {
+  if (walLevel >= 0 && (walLevel < TSDB_MIN_WAL_LEVEL || walLevel > TSDB_MAX_WAL_LEVEL)) {
-    mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications);
+    mError("db:%s, wal level %d should be between 0-2, origin:%d", pDb->name, walLevel, pDb->cfg.walLevel);
     terrno = TSDB_CODE_INVALID_OPTION;
   }
 

src/mnode/src/mgmtDnode.c

@@ -27,9 +27,7 @@
 #include "tdataformat.h"
 #include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
-#include "mgmtDClient.h"
-#include "mgmtDServer.h"
 #include "mgmtDnode.h"
 #include "mgmtMnode.h"
 #include "mgmtSdb.h"
@@ -153,8 +151,8 @@ int32_t mgmtInitDnodes() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DNODE, mgmtProcessCreateDnodeMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DNODE, mgmtProcessDropDnodeMsg); 
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg);
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp);
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp);
-  mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg);
+  dnodeAddServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg);
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta);
   mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules);
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta);
@@ -225,7 +223,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
   if (pCmCfgDnode->ep[0] == 0) {
     strcpy(pCmCfgDnode->ep, tsLocalEp);
   } else {
-    strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep);
+    // TODO temporary disabled for compiling: strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep); 
   }
 
   if (strcmp(pMsg->pUser->user, "root") != 0) {
@@ -242,7 +240,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
         .pCont = pMdCfgDnode,
         .contLen = sizeof(SMDCfgDnodeMsg)
     };
-    mgmtSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg);
+    dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg);
     rpcRsp.code = TSDB_CODE_SUCCESS;
   }
 
@@ -254,7 +252,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
 }
 
 static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
-  mPrint("cfg vnode rsp is received, result:%s", tstrerror(rpcMsg->code));
+  mPrint("cfg dnode rsp is received");
 }
 
 void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {

src/mnode/src/mgmtGrant.c

@@ -18,7 +18,7 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tgrant.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 
 int32_t grantInit() { return TSDB_CODE_SUCCESS; }
 void    grantCleanUp() {}

src/mnode/src/mgmtMain.c

@@ -23,13 +23,12 @@
 #include "tglobal.h"
 #include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
+#include "mgmtServer.h"
 #include "mgmtAcct.h"
 #include "mgmtDnode.h"
 #include "mgmtMnode.h"
 #include "mgmtDb.h"
-#include "mgmtDClient.h"
-#include "mgmtDServer.h"
 #include "mgmtSdb.h"
 #include "mgmtVgroup.h"
 #include "mgmtUser.h"
@@ -56,11 +55,6 @@ int32_t mgmtStartSystem() {
     return -1;
   }
 
-  if (grantInit() < 0) {
-    mError("failed to init grant");
-    return -1;
-  }
-
   if (mgmtInitUsers() < 0) {
     mError("failed to init users");
     return -1;
@@ -100,11 +94,12 @@ int32_t mgmtStartSystem() {
     mError("failed to init balance")
   }
 
-  if (mgmtInitDClient() < 0) {
+  if (grantInit() < 0) {
+    mError("failed to init grant");
     return -1;
   }
 
-  if (mgmtInitDServer() < 0) {
+  if (mgmtInitServer() < 0) {
     return -1;
   }
 
@@ -137,21 +132,19 @@ int32_t mgmtInitSystem() {
 
 void mgmtCleanUpSystem() {
   mPrint("starting to clean up mgmt");
-  grantCleanUp();
+  tsMgmtIsRunning = false;
-  mgmtCleanupMnodes();
-  balanceCleanUp();
   mgmtCleanUpShell();
-  mgmtCleanupDClient();
+  mgmtCleanupServer();
-  mgmtCleanupDServer();
+  grantCleanUp();
-  mgmtCleanUpAccts();
+  balanceCleanUp();
+  sdbCleanUp();
+  mgmtCleanupMnodes();
   mgmtCleanUpTables();
   mgmtCleanUpVgroups();
   mgmtCleanUpDbs();
   mgmtCleanupDnodes();
   mgmtCleanUpUsers();
-  sdbCleanUp();
+  mgmtCleanUpAccts();
-  taosTmrCleanUp(tsMgmtTmr);
-  tsMgmtIsRunning = false;
   mPrint("mgmt is cleaned up");
 }
 

src/mnode/src/mgmtMnode.c

@@ -24,7 +24,7 @@
 #include "tsocket.h"
 #include "tdataformat.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtMnode.h"
 #include "mgmtDnode.h"
 #include "mgmtSdb.h"

src/mnode/src/mgmtProfile.c

@@ -19,7 +19,7 @@
 #include "taoserror.h"
 #include "tutil.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtAcct.h"
 #include "mgmtDnode.h"
 #include "mgmtDb.h"

src/mnode/src/mgmtSdb.c

@@ -27,7 +27,7 @@
 #include "hashstr.h"
 #include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtMnode.h"
 #include "mgmtSdb.h"
 
@@ -142,7 +142,7 @@ static void *sdbGetTableFromId(int32_t tableId) {
 }
 
 static int32_t sdbInitWal() {
-  SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1};
+  SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1};
   char temp[TSDB_FILENAME_LEN];
   sprintf(temp, "%s/wal", tsMnodeDir);
   tsSdbObj.wal = walOpen(temp, &walCfg);
@@ -278,7 +278,7 @@ void sdbUpdateSync() {
     sdbPrint("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort);
   }
 
-  SSyncInfo syncInfo;
+  SSyncInfo syncInfo = {0};
   syncInfo.vgId = 1;
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;
@@ -323,12 +323,19 @@ void sdbCleanUp() {
   if (tsSdbObj.status != SDB_STATUS_SERVING) return;
 
   tsSdbObj.status = SDB_STATUS_CLOSING;
-  syncStop(tsSdbObj.sync);
+  
-  free(tsSdbObj.sync);
+  if (tsSdbObj.sync) {
-  walClose(tsSdbObj.wal);
+    syncStop(tsSdbObj.sync);
+    tsSdbObj.sync = NULL;
+  }
+
+  if (tsSdbObj.wal) {
+    walClose(tsSdbObj.wal);
+    tsSdbObj.wal = NULL;
+  }
+  
   sem_destroy(&tsSdbObj.sem);
   pthread_mutex_destroy(&tsSdbObj.mutex);
-  memset(&tsSdbObj, 0, sizeof(tsSdbObj));
 }
 
 void sdbIncRef(void *handle, void *pRow) {

src/mnode/src/mgmtServer.c

@@ -25,9 +25,8 @@
 #include "tglobal.h"
 #include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtDb.h"
-#include "mgmtDServer.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
 #include "mgmtShell.h"
@@ -35,67 +34,49 @@
 #include "mgmtTable.h"
 #include "mgmtVgroup.h"
 
-static void   mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg);
-static int    mgmtDServerRetrieveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
 static void (*mgmtProcessDnodeMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
-static void  *tsMgmtDServerRpc;
+static void  *tsMgmtServerQhandle = NULL;
-static void  *tsMgmtDServerQhandle = NULL;
 
-int32_t mgmtInitDServer() {
+int32_t mgmtInitServer() {
-  SRpcInit rpcInit = {0};
-  rpcInit.localPort    = tsMnodeDnodePort;
-  rpcInit.label        = "MND-DS";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp          = mgmtProcessMsgFromDnode;
-  rpcInit.sessions     = 100;
-  rpcInit.connType     = TAOS_CONN_SERVER;
-  rpcInit.idleTime     = tsShellActivityTimer * 1000;
-  rpcInit.afp          = mgmtDServerRetrieveAuth;
 
-  tsMgmtDServerRpc = rpcOpen(&rpcInit);
+  tsMgmtServerQhandle = taosInitScheduler(tsMaxShellConns, 1, "MS");
-  if (tsMgmtDServerRpc == NULL) {
-    mError("failed to init server connection to dnode");
-    return -1;
-  }
-
-  tsMgmtDServerQhandle = taosInitScheduler(tsMaxShellConns, 1, "MS");
 
   mPrint("server connection to dnode is opened");
   return 0;
 }
 
-void mgmtCleanupDServer() {
+void mgmtCleanupServer() {
-  if (tsMgmtDServerQhandle) {
+  if (tsMgmtServerQhandle) {
-    taosCleanUpScheduler(tsMgmtDServerQhandle);
+    taosCleanUpScheduler(tsMgmtServerQhandle);
-    tsMgmtDServerQhandle = NULL;
+    tsMgmtServerQhandle = NULL;
-  }
-
-  if (tsMgmtDServerRpc) {
-    rpcClose(tsMgmtDServerRpc);
-    tsMgmtDServerRpc = NULL;
-    mPrint("server connection to dnode is closed");
   }
 }
 
-void mgmtAddDServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
+void dnodeAddServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
   mgmtProcessDnodeMsgFp[msgType] = fp;
 }
 
-static void mgmtProcessDServerRequest(SSchedMsg *sched) {
+static void mgmtProcessRequestFromDnode(SSchedMsg *sched) {
   SRpcMsg *pMsg = sched->msg;
   (*mgmtProcessDnodeMsgFp[pMsg->msgType])(pMsg);
   rpcFreeCont(pMsg->pCont);
   free(pMsg);
 }
 
-static void mgmtAddToDServerQueue(SRpcMsg *pMsg) {
+static void mgmtAddToServerQueue(SRpcMsg *pMsg) {
   SSchedMsg schedMsg;
   schedMsg.msg = pMsg;
-  schedMsg.fp  = mgmtProcessDServerRequest;
+  schedMsg.fp  = mgmtProcessRequestFromDnode;
-  taosScheduleTask(tsMgmtDServerQhandle, &schedMsg);
+  taosScheduleTask(tsMgmtServerQhandle, &schedMsg);
 }
 
-static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) {
+void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg) {
+  if (mgmtProcessDnodeMsgFp[rpcMsg->msgType] == NULL) {
+    mError("%s is not processed in mnode", taosMsg[rpcMsg->msgType]);
+    mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_MSG_NOT_PROCESSED);
+    rpcFreeCont(rpcMsg->pCont);
+  }
+
   if (rpcMsg->pCont == NULL) {
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_MSG_LEN);
     return;
@@ -107,26 +88,19 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) {
     
     SRpcIpSet ipSet = {0};
     dnodeGetMnodeDnodeIpSet(&ipSet);
+    for (int i = 0; i < ipSet.numOfIps; ++i) 
+      ipSet.port[i] = htons(ipSet.port[i]);
     
     mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
     for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
-      mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
+      mTrace("mnode index:%d %s:%d", i, ipSet.fqdn[i], htons(ipSet.port[i]));
     }
     rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
     return;
   }
   
-  if (mgmtProcessDnodeMsgFp[rpcMsg->msgType]) {
+  SRpcMsg *pMsg = malloc(sizeof(SRpcMsg));
-    SRpcMsg *pMsg = malloc(sizeof(SRpcMsg));
+  memcpy(pMsg, rpcMsg, sizeof(SRpcMsg));
-    memcpy(pMsg, rpcMsg, sizeof(SRpcMsg));
+  mgmtAddToServerQueue(pMsg);
-    mgmtAddToDServerQueue(pMsg);
-  } else {
-    mError("%s is not processed in mgmt dserver", taosMsg[rpcMsg->msgType]);
-    mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_MSG_NOT_PROCESSED);
-    rpcFreeCont(rpcMsg->pCont);
-  }
 }
 
-static int mgmtDServerRetrieveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  return TSDB_CODE_SUCCESS;
-}

src/mnode/src/mgmtShell.c

@@ -26,7 +26,7 @@
 #include "tcache.h"
 #include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtAcct.h"
 #include "mgmtDb.h"
 #include "mgmtDnode.h"
@@ -41,9 +41,8 @@
 typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-static int  mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
+//static int  mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
 static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg);
-static void mgmtProcessMsgFromShell(SRpcMsg *pMsg);
 static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg);
 static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg);
@@ -52,7 +51,6 @@ static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg);
 
 void *tsMgmtTmr;
-static void *tsMgmtShellRpc = NULL;
 static void *tsMgmtTranQhandle = NULL;
 static void (*tsMgmtProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SQueuedMsg *) = {0};
 static void *tsQhandleCache = NULL;
@@ -61,57 +59,33 @@ static SShowRetrieveFp tsMgmtShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
 
 int32_t mgmtInitShell() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mgmtProcessShowMsg);
-  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_RETRIEVE, mgmtProcessRetrieveMsg);
+  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_RETRIEVE, mgmtProcessRetrieveMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mgmtProcessUseMsg);
   
   tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
   tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT");
-  tsQhandleCache = taosCacheInit(tsMgmtTmr, 2);
+  tsQhandleCache = taosCacheInit(tsMgmtTmr, 10);
 
-  int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0;
-  if (numOfThreads < 1) {
-    numOfThreads = 1;
-  }
-
-  SRpcInit rpcInit = {0};
-  rpcInit.localPort    = tsMnodeShellPort;
-  rpcInit.label        = "MND-shell";
-  rpcInit.numOfThreads = numOfThreads;
-  rpcInit.cfp          = mgmtProcessMsgFromShell;
-  rpcInit.sessions     = tsMaxShellConns;
-  rpcInit.connType     = TAOS_CONN_SERVER;
-  rpcInit.idleTime     = tsShellActivityTimer * 1000;
-  rpcInit.afp          = mgmtShellRetriveAuth;
-
-  tsMgmtShellRpc = rpcOpen(&rpcInit);
-  if (tsMgmtShellRpc == NULL) {
-    mError("failed to init server connection to shell");
-    return -1;
-  }
-
-  mPrint("server connection to shell is opened");
   return 0;
 }
 
 void mgmtCleanUpShell() {
-  if (tsMgmtTranQhandle) {
+  if (tsMgmtTmr != NULL){
-    taosCleanUpScheduler(tsMgmtTranQhandle);
+    taosTmrCleanUp(tsMgmtTmr);
-    tsMgmtTranQhandle = NULL;
+    tsMgmtTmr = NULL;
   }
 
-  if (tsMgmtShellRpc) {
+  if (tsQhandleCache != NULL) {
-    rpcClose(tsMgmtShellRpc);
-    tsMgmtShellRpc = NULL;
-    mPrint("server connection to shell is closed");
-  }
-
-  if (tsQhandleCache) {
-    taosCacheEmpty(tsQhandleCache);
     taosCacheCleanup(tsQhandleCache);
     tsQhandleCache = NULL;
   }
+  
+  if (tsMgmtTranQhandle != NULL) {
+    taosCleanUpScheduler(tsMgmtTranQhandle);
+    tsMgmtTranQhandle = NULL;
+  }
 }
 
 void mgmtAddShellMsgHandle(uint8_t showType, void (*fp)(SQueuedMsg *queuedMsg)) {
@@ -148,8 +122,7 @@ void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) {
   taosTmrReset(mgmtDoDealyedAddToShellQueue, 1000, queuedMsg, tsMgmtTmr, &unUsed);
 }
 
-static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
+void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
-  assert(rpcMsg);
 
   if (rpcMsg->pCont == NULL) {
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_MSG_LEN);
@@ -164,7 +137,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
     mgmtGetMnodeIpSet(&ipSet);
     mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
     for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
-      mTrace("index:%d ip:%s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
+      mTrace("mnode index:%d ip:%s:%d", i, ipSet.fqdn[i], htons(ipSet.port[i]));
     }
 
     rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
@@ -370,6 +343,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
+/*
 static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
   *spi = 1;
   *encrypt = 0;
@@ -390,6 +364,7 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secr
     return TSDB_CODE_SUCCESS;
   }
 }
+*/
 
 static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
@@ -490,7 +465,7 @@ static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg) {
     return mgmtCheckTableMetaMsgReadOnly(pMsg);
   }
 
-  if (pMsg->msgType == TSDB_MSG_TYPE_CM_STABLE_VGROUP || pMsg->msgType == TSDB_MSG_TYPE_RETRIEVE       ||
+  if (pMsg->msgType == TSDB_MSG_TYPE_CM_STABLE_VGROUP || pMsg->msgType == TSDB_MSG_TYPE_CM_RETRIEVE    ||
       pMsg->msgType == TSDB_MSG_TYPE_CM_SHOW          || pMsg->msgType == TSDB_MSG_TYPE_CM_TABLES_META ||
       pMsg->msgType == TSDB_MSG_TYPE_CM_CONNECT) {
     return true;

src/mnode/src/mgmtTable.c

@@ -24,13 +24,12 @@
 #include "tname.h"
 #include "tidpool.h"
 #include "tglobal.h"
+#include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtAcct.h"
-#include "mgmtDClient.h"
 #include "mgmtDb.h"
 #include "mgmtDnode.h"
-#include "mgmtDServer.h"
 #include "tgrant.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
@@ -539,12 +538,12 @@ int32_t mgmtInitTables() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLE_META, mgmtProcessTableMetaMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_VGROUP, mgmtProcessSuperTableVgroupMsg);
   
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateChildTableRsp);
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateChildTableRsp);
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropChildTableRsp);
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropChildTableRsp);
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropSuperTableRsp);
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropSuperTableRsp);
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp);
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp);
 
-  mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg);
+  dnodeAddServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg);
 
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mgmtGetShowTableMeta);
   mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mgmtRetrieveShowTables);
@@ -702,10 +701,10 @@ static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
   }
 
   if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
-    mTrace("table:%s, start to drop stable", pDrop->tableId);
+    mPrint("table:%s, start to drop stable", pDrop->tableId);
     mgmtProcessDropSuperTableMsg(pMsg);
   } else {
-    mTrace("table:%s, start to drop ctable", pDrop->tableId);
+    mPrint("table:%s, start to drop ctable", pDrop->tableId);
     mgmtProcessDropChildTableMsg(pMsg);
   }
 }
@@ -803,36 +802,39 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
       int32_t vgId = pStable->vgList[vg];
       if (vgId == 0) break;
 
+      SVgObj *pVgroup = mgmtGetVgroup(vgId);
+      if (pVgroup == NULL) break;
+      
       SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg));
+      pDrop->contLen = htonl(sizeof(SMDDropSTableMsg));
       pDrop->vgId = htonl(vgId);
       pDrop->uid = htobe64(pStable->uid);
       mgmtExtractTableName(pStable->info.tableId, pDrop->tableId);
-
+        
-      SVgObj *pVgroup = mgmtGetVgroup(vgId);
+      mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, vgId);
-      if (pVgroup != NULL) {
+      SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
-        SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
+      SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
-        SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
+      dnodeSendMsgToDnode(&ipSet, &rpcMsg);
-        mgmtSendMsgToDnode(&ipSet, &rpcMsg);
+      mgmtDecVgroupRef(pVgroup);
-        mgmtDecVgroupRef(pVgroup);
-      }
     }
-  } else {
+  } 
-    SSdbOper oper = {
+  
-      .type = SDB_OPER_GLOBAL,
+  SSdbOper oper = {
-      .table = tsSuperTableSdb,
+    .type = SDB_OPER_GLOBAL,
-      .pObj = pStable
+    .table = tsSuperTableSdb,
-    };
+    .pObj = pStable
-    int32_t code = sdbDeleteRow(&oper);
+  };
-    mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code));
+  
-    mgmtSendSimpleResp(pMsg->thandle, code);
+  int32_t code = sdbDeleteRow(&oper);
-  }
+  mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code));
+  mgmtSendSimpleResp(pMsg->thandle, code);
 }
 
 static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {
-  for (int32_t i = 0; i < pStable->numOfTags; i++) {
+  SSchema *schema = (SSchema *) pStable->schema;
-    SSchema *schema = (SSchema *)(pStable->schema + (pStable->numOfColumns + i) * sizeof(SSchema));
+  for (int32_t tag = 0; tag < pStable->numOfTags; tag++) {
-    if (strcasecmp(tagName, schema->name) == 0) {
+    if (strcasecmp(schema[pStable->numOfColumns + tag].name, tagName) == 0) {
-      return i;
+      return tag;
     }
   }
 
@@ -1304,7 +1306,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
 }
 
 static void mgmtProcessDropSuperTableRsp(SRpcMsg *rpcMsg) {
- mTrace("drop stable rsp received, handle:%p code:%s", rpcMsg->handle, tstrerror(rpcMsg->code));
+  mPrint("drop stable rsp received, result:%s", tstrerror(rpcMsg->code));
 }
 
 static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) {
@@ -1514,7 +1516,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
       .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
   };
 
-  mgmtSendMsgToDnode(&ipSet, &rpcMsg);
+  dnodeSendMsgToDnode(&ipSet, &rpcMsg);
 }
 
 static void mgmtProcessDropChildTableMsg(SQueuedMsg *pMsg) {
@@ -1541,7 +1543,7 @@ static void mgmtProcessDropChildTableMsg(SQueuedMsg *pMsg) {
 
   SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pMsg->pVgroup);
 
-  mTrace("table:%s, send drop ctable msg", pDrop->tableId);
+  mPrint("table:%s, send drop ctable msg", pDrop->tableId);
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
   newMsg->ahandle = pMsg->pTable;
   SRpcMsg rpcMsg = {
@@ -1552,7 +1554,7 @@ static void mgmtProcessDropChildTableMsg(SQueuedMsg *pMsg) {
     .msgType = TSDB_MSG_TYPE_MD_DROP_TABLE
   };
 
-  mgmtSendMsgToDnode(&ipSet, &rpcMsg);
+  dnodeSendMsgToDnode(&ipSet, &rpcMsg);
 }
 
 static int32_t mgmtModifyChildTableTagValue(SChildTableObj *pTable, char *tagName, char *nContent) {
@@ -1854,7 +1856,7 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
       .code    = 0,
       .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
   };
-  mgmtSendMsgToDnode(&ipSet, &rpcRsp);
+  dnodeSendMsgToDnode(&ipSet, &rpcRsp);
 
   mgmtDecTableRef(pTable);
   mgmtDecDnodeRef(pDnode);
@@ -1868,7 +1870,7 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
   queueMsg->received++;
 
   SChildTableObj *pTable = queueMsg->ahandle;
-  mTrace("table:%s, drop table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, tstrerror(rpcMsg->code));
+  mPrint("table:%s, drop table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, tstrerror(rpcMsg->code));
 
   if (rpcMsg->code != TSDB_CODE_SUCCESS) {
     mError("table:%s, failed to drop in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code));

src/mnode/src/mgmtUser.c

@@ -23,7 +23,7 @@
 #include "tdataformat.h"
 #include "dnode.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtAcct.h"
 #include "mgmtMnode.h"
 #include "mgmtSdb.h"
@@ -309,11 +309,14 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     if (pUser->superAuth) {
-      STR_WITH_SIZE_TO_VARSTR(pWrite, "super", 5);
+      const char *src = "super";
+      STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src));
     } else if (pUser->writeAuth) {
-      STR_WITH_SIZE_TO_VARSTR(pWrite, "writable", 8);
+      const char *src = "writable";
+      STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src));
     } else {
-      STR_WITH_SIZE_TO_VARSTR(pWrite, "readable", 8);
+      const char *src = "readable";
+      STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src));
     }
     cols++;
 
@@ -396,7 +399,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
       code = mgmtUpdateUser(pUser);
       mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
     } else {
-      mError("user:%s, no rights to ater user", pOperUser->user);
+      mError("user:%s, no rights to alter user", pOperUser->user);
       code = TSDB_CODE_NO_RIGHTS;
     }
 
@@ -439,13 +442,13 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
       code = mgmtUpdateUser(pUser);
       mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
     } else {
-      mError("user:%s, no rights to ater user", pOperUser->user);
+      mError("user:%s, no rights to alter user", pOperUser->user);
       code = TSDB_CODE_NO_RIGHTS;
     }
 
     mgmtSendSimpleResp(pMsg->thandle, code);
   } else {
-    mError("user:%s, no rights to ater user", pOperUser->user);
+    mError("user:%s, no rights to alter user", pOperUser->user);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
   }
 
@@ -525,4 +528,4 @@ void  mgmtDropAllUsers(SAcctObj *pAcct)  {
   }
 
   mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);
-}
+}

src/mnode/src/mgmtVgroup.c

@@ -23,12 +23,11 @@
 #include "ttime.h"
 #include "tbalance.h"
 #include "tglobal.h"
+#include "dnode.h"
 #include "tdataformat.h"
 #include "mgmtDef.h"
-#include "mgmtLog.h"
+#include "mgmtInt.h"
 #include "mgmtDb.h"
-#include "mgmtDClient.h"
-#include "mgmtDServer.h"
 #include "mgmtDnode.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
@@ -218,9 +217,9 @@ int32_t mgmtInitVgroups() {
 
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta);
   mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups);
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp);
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp);
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp);
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp);
-  mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg);
+  dnodeAddServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg);
 
   mTrace("table:vgroups is created");
   
@@ -495,11 +494,12 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
         cols++;
       } else {
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-        STR_WITH_SIZE_TO_VARSTR(pWrite, "NULL", 4);
+        const char *src = "NULL";
+        STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src));
         cols++;
         
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-        STR_WITH_SIZE_TO_VARSTR(pWrite, "NULL", 4);
+        STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src));
         cols++;
       }
     }
@@ -560,7 +560,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
   pCfg->commitTime          = htonl(pDb->cfg.commitTime);
   pCfg->precision           = pDb->cfg.precision;
   pCfg->compression         = pDb->cfg.compression;
-  pCfg->commitLog           = pDb->cfg.commitLog;
+  pCfg->walLevel            = pDb->cfg.walLevel;
   pCfg->replications        = (int8_t) pVgroup->numOfVnodes;
   pCfg->wals                = 3;
   pCfg->quorum              = 1;
@@ -584,7 +584,7 @@ SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) {
   };
   for (int i = 0; i < pVgroup->numOfVnodes; ++i) {
     strcpy(ipSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn);
-    ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEMNODE;
+    ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEDNODE;
   }
   return ipSet;
 }
@@ -595,7 +595,7 @@ SRpcIpSet mgmtGetIpSetFromIp(char *ep) {
   ipSet.numOfIps = 1;
   ipSet.inUse = 0;
   taosGetFqdnPortFromEp(ep, ipSet.fqdn[0], &ipSet.port[0]);
-  ipSet.port[0] += TSDB_PORT_DNODEMNODE;
+  ipSet.port[0] += TSDB_PORT_DNODEDNODE;
   return ipSet;
 }
 
@@ -609,7 +609,7 @@ void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
     .code    = 0,
     .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE
   };
-  mgmtSendMsgToDnode(ipSet, &rpcMsg);
+  dnodeSendMsgToDnode(ipSet, &rpcMsg);
 }
 
 void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
@@ -675,7 +675,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
       .code    = 0,
       .msgType = TSDB_MSG_TYPE_MD_DROP_VNODE
   };
-  mgmtSendMsgToDnode(ipSet, &rpcMsg);
+  dnodeSendMsgToDnode(ipSet, &rpcMsg);
 }
 
 static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {

src/os/linux/CMakeLists.txt

@@ -12,3 +12,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   ADD_LIBRARY(os ${SRC})
   TARGET_LINK_LIBRARIES(os m rt)
 ENDIF ()
+
+SET_SOURCE_FILES_PROPERTIES(src/linuxSysPara.c PROPERTIES COMPILE_FLAGS -w)

src/os/linux/inc/os.h

@@ -23,13 +23,6 @@ extern "C" {
 #include <stdio.h>
 #include <stdlib.h>
 
-#ifndef _ALPINE
-  #include <error.h>
-  #include <sys/sysctl.h>
-#else
-  #include <linux/sysctl.h>  
-#endif
-
 #include <argp.h>
 #include <arpa/inet.h>
 #include <assert.h>
@@ -82,6 +75,7 @@ extern "C" {
 #include <fcntl.h>
 #include <sys/utsname.h>
 #include <sys/resource.h>
+#include <error.h>
 
 #define taosCloseSocket(x) \
   {                        \

src/os/linux/inc/tsysctl.h

@@ -13,19 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_DSERVER_H
+#ifndef TDENGINE_TSYSCTL_H
-#define TDENGINE_MGMT_DSERVER_H
+#define TDENGINE_TSYSCTL_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int32_t mgmtInitDServer();
+#ifndef _ALPINE
-void    mgmtCleanupDServer();
+  #include <error.h>
-void    mgmtAddDServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
+  #include <sys/sysctl.h>
-
+#else
-#ifdef __cplusplus
+  #include <linux/sysctl.h>  
-}
 #endif
 
 #endif

src/os/linux/src/linuxSysPara.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "tsysctl.h"
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
@@ -356,10 +357,15 @@ static bool taosGetCardName(char *ip, char *name) {
       break;
     }
 
-    if (strcmp(host, ip) == 0) {
+    if (strcmp(host, "127.0.0.1") == 0) {
-      strcpy(name, ifa->ifa_name);
+      continue;
-      ret = true;
     }
+
+    // TODO: the ip not config
+    // if (strcmp(host, ip) == 0) {
+    strcpy(name, ifa->ifa_name);
+    ret = true;
+    // }
   }
 
   freeifaddrs(ifaddr);

src/plugins/http/src/gcJson.c

@@ -121,10 +121,11 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
 
   for (int k = 0; k < numOfRows; ++k) {
     TAOS_ROW row = taos_fetch_row(result);
+    int32_t* length = taos_fetch_lengths(result);
 
     // for group by
     if (groupFields != -1) {
-      char target[HTTP_GC_TARGET_SIZE];
+      char target[HTTP_GC_TARGET_SIZE] = {0};
       int len;
       len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer);
       for (int i = dataFields + 1; i<num_fields; i++){
@@ -150,7 +151,11 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
             break;
           case TSDB_DATA_TYPE_BINARY:
           case TSDB_DATA_TYPE_NCHAR:
-            len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%s", fields[i].name, (char *)row[i]);
+            if (row[i]!= NULL){            
+              len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name);
+              memcpy(target + len, (char *) row[i], length[i]);
+              len = strlen(target);
+            }
             break;
           default:
             len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%s", fields[i].name, "-");

src/plugins/http/src/httpServer.c

@@ -269,11 +269,12 @@ void httpCleanUpConnect(HttpServer *pServer) {
 
   for (i = 0; i < pServer->numOfThreads; ++i) {
     pThread = pServer->pThreads + i;
+    if (pThread == NULL) continue;
     //taosCloseSocket(pThread->pollFd);
 
-    while (pThread->pHead) {
+    //while (pThread->pHead) {
-      httpCleanUpContext(pThread->pHead, 0);
+    //  httpCleanUpContext(pThread->pHead, 0);
-    }
+    //}
 
     pthread_cancel(pThread->thread);
     pthread_join(pThread->thread, NULL);
@@ -504,8 +505,8 @@ void httpAcceptHttpConnection(void *arg) {
   sockFd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort);
 
   if (sockFd < 0) {
-    httpError("http server:%s, failed to open http socket, ip:%s:%u", pServer->label, pServer->serverIp,
+    httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label, taosIpStr(pServer->serverIp),
-              pServer->serverPort);
+              pServer->serverPort, strerror(errno));
     return;
   } else {
     httpPrint("http service init success at %u", pServer->serverPort);
@@ -645,7 +646,7 @@ bool httpInitConnect(HttpServer *pServer) {
   }
   pthread_attr_destroy(&thattr);
 
-  httpTrace("http server:%s, initialized, ip:%s:%u, numOfThreads:%d", pServer->label, pServer->serverIp,
+  httpTrace("http server:%s, initialized, ip:%s:%u, numOfThreads:%d", pServer->label, taosIpStr(pServer->serverIp),
             pServer->serverPort, pServer->numOfThreads);
   return true;
 }

src/plugins/http/src/httpSystem.c

@@ -117,7 +117,7 @@ void httpCleanUpSystem() {
   httpPrint("http service cleanup");
   httpStopSystem();
 
-#if 0
+//#if 0
   if (httpServer == NULL) {
     return;
   }
@@ -131,7 +131,13 @@ void httpCleanUpSystem() {
     httpServer->timerHandle = NULL;
   }
 
-  httpCleanUpConnect(httpServer);
+  if (httpServer->pThreads != NULL) {
+    httpCleanUpConnect(httpServer);
+    httpServer->pThreads = NULL;
+  }
+  
+
+#if 0
   httpRemoveAllSessions(httpServer);
 
   if (httpServer->pContextPool != NULL) {

src/plugins/http/src/restJson.c

@@ -94,6 +94,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
 
   for (int k = 0; k < numOfRows; ++k) {
     TAOS_ROW row = taos_fetch_row(result);
+    int32_t* length = taos_fetch_lengths(result);
 
     // data row array begin
     httpJsonItemToken(jsonBuf);
@@ -129,7 +130,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
           break;
         case TSDB_DATA_TYPE_BINARY:
         case TSDB_DATA_TYPE_NCHAR:
-          httpJsonStringForTransMean(jsonBuf, row[i], fields[i].bytes);
+          httpJsonStringForTransMean(jsonBuf, row[i], length[i]);
           break;
         case TSDB_DATA_TYPE_TIMESTAMP:
           if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) {

src/plugins/monitor/src/monitorMain.c

@@ -43,8 +43,8 @@
   { taosPrintLog("MON ", 255, __VA_ARGS__); }
 
 #define SQL_LENGTH     1024
-#define LOG_LEN_STR    80
+#define LOG_LEN_STR    100
-#define IP_LEN_STR     15
+#define IP_LEN_STR     18
 #define CHECK_INTERVAL 1000
 
 typedef enum {
@@ -115,6 +115,13 @@ static void monitorInitConn(void *para, void *unused) {
   if (tsMonitorConn.ep[0] == 0) 
     strcpy(tsMonitorConn.ep, tsLocalEp);
 
+  int len = strlen(tsMonitorConn.ep);
+  for (int i = 0; i < len; ++i) {
+    if (tsMonitorConn.ep[i] == ':' || tsMonitorConn.ep[i] == '-') {
+      tsMonitorConn.ep[i] = '_';
+    }
+  }
+
   if (tsMonitorConn.conn == NULL) {
     taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, &tsMonitorConn, &(tsMonitorConn.conn));
   } else {
@@ -141,8 +148,8 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
 
   if (cmd == MONITOR_CMD_CREATE_DB) {
     snprintf(sql, SQL_LENGTH,
-             "create database if not exists %s replica 1 days 10 keep 30 rows 1024 cache 2048 "
+             "create database if not exists %s replica 1 days 10 keep 30 cache 1 "
-             "ablocks 2 tblocks 32 tables 32 precision 'us'",
+             "blocks 2 maxtables 16 precision 'us'",
              tsMonitorDbName);
   } else if (cmd == MONITOR_CMD_CREATE_MT_DN) {
     snprintf(sql, SQL_LENGTH,
@@ -153,11 +160,11 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
              ", band_speed float"
              ", io_read float, io_write float"
              ", req_http int, req_select int, req_insert int"
-             ") tags (ipaddr binary(%d))",
+             ") tags (dnodeid int, fqdn binary(%d))",
-             tsMonitorDbName, IP_LEN_STR + 1);
+             tsMonitorDbName, TSDB_FQDN_LEN + 1);
   } else if (cmd == MONITOR_CMD_CREATE_TB_DN) {
-    snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName,
+    snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName,
-             tsMonitorConn.ep, tsMonitorDbName, tsLocalEp);
+             dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp);
   } else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) {
     snprintf(sql, SQL_LENGTH,
              "create table if not exists %s.acct(ts timestamp "
@@ -340,7 +347,7 @@ static void monitorSaveSystemInfo() {
 
   int64_t ts = taosGetTimestampUs();
   char *  sql = tsMonitorConn.sql;
-  int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.ep, ts);
+  int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts);
 
   pos += monitorBuildCpuSql(sql + pos);
   pos += monitorBuildMemorySql(sql + pos);
@@ -400,7 +407,7 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
 
   if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return;
 
-  int32_t len = snprintf(sql, (size_t)max_length, "import into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName,
+  int32_t len = snprintf(sql, (size_t)max_length, "insert into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName,
                          taosGetTimestampUs(), level);
 
   va_start(argpointer, format);

src/query/inc/qast.h

@@ -48,16 +48,16 @@ typedef struct tQueryInfo {
   int32_t       colIndex; // index of column in schema
   uint8_t       optr;     // expression operator
   SSchema       sch;      // schema of tags
-//  tVariant      q;      // query condition value on the specific schema, filter expression
   char*         q;
   __compar_fn_t compare;  // filter function
+  void*         param;    // STSchema,
 } tQueryInfo;
 
-typedef struct SBinaryFilterSupp {
+typedef struct SExprTraverseSupp {
   __result_filter_fn_t   fp;
   __do_filter_suppl_fn_t setupInfoFn;
   void *                 pExtInfo;
-} SBinaryFilterSupp;
+} SExprTraverseSupp;
 
 typedef struct tExprNode {
   uint8_t nodeType;
@@ -81,7 +81,7 @@ void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len);
 
 void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
 
-void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param);
+void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param);
 
 void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
                                 char *(*cb)(void *, const char*, int32_t));

src/query/inc/qsqlparser.h

@@ -111,13 +111,13 @@ typedef struct SCreateDBInfo {
   SSQLToken dbname;
   int32_t   replica;
   int32_t   cacheBlockSize;
-  int32_t   tablesPerVnode;
+  int32_t   maxTablesPerVnode;
+  int32_t   numOfBlocks;
   int32_t   daysPerFile;
-  int32_t   rowPerFileBlock;
+  int32_t   minRowsPerBlock;
-  float     numOfAvgCacheBlocks;
+  int32_t   maxRowsPerBlock;
-  int32_t   numOfBlocksPerTable;
   int64_t   commitTime;
-  int32_t   commitLog;
+  int32_t   walLevel;
   int32_t   compressionLevel;
   SSQLToken precision;
   bool      ignoreExists;

src/query/inc/queryLog.h

@@ -22,20 +22,20 @@ extern "C" {
 
 #include "tlog.h"
 
-extern int32_t qdebugFlag;
+extern int32_t qDebugFlag;
 
 #define qTrace(...)                               \
-  if (qdebugFlag & DEBUG_TRACE) {                 \
+  if (qDebugFlag & DEBUG_TRACE) {                 \
-    taosPrintLog("DND QRY ", qdebugFlag, __VA_ARGS__); \
+    taosPrintLog("DND QRY ", qDebugFlag, __VA_ARGS__); \
   }
 
 #define qError(...)                                   \
-  if (qdebugFlag & DEBUG_ERROR) {                   \
+  if (qDebugFlag & DEBUG_ERROR) {                   \
-    taosPrintLog("ERROR QRY ", qdebugFlag, __VA_ARGS__); \
+    taosPrintLog("ERROR QRY ", qDebugFlag, __VA_ARGS__); \
   }
 #define qWarn(...)                                    \
-  if (qdebugFlag & DEBUG_WARN) {                    \
+  if (qDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  QRY ", qdebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN  QRY ", qDebugFlag, __VA_ARGS__); \
   }  
 
 #ifdef __cplusplus

src/query/inc/sql.y

@@ -212,31 +212,30 @@ acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K
 %destructor keep {tVariantListDestroy($$);}
 keep(Y)    ::= KEEP tagitemlist(X).           { Y = X; }
 
-tables(Y)  ::= TABLES INTEGER(X).             { Y = X; }
+tables(Y)  ::= MAXTABLES INTEGER(X).          { Y = X; }
 cache(Y)   ::= CACHE INTEGER(X).              { Y = X; }
 replica(Y) ::= REPLICA INTEGER(X).            { Y = X; }
 days(Y)    ::= DAYS INTEGER(X).               { Y = X; }
-rows(Y)    ::= ROWS INTEGER(X).               { Y = X; }
+minrows(Y) ::= MINROWS INTEGER(X).            { Y = X; }
-
+maxrows(Y) ::= MAXROWS INTEGER(X).            { Y = X; }
-ablocks(Y) ::= ABLOCKS ID(X).                 { Y = X; }
+blocks(Y)  ::= BLOCKS INTEGER(X).             { Y = X; }
-tblocks(Y) ::= TBLOCKS INTEGER(X).            { Y = X; }
 ctime(Y)   ::= CTIME INTEGER(X).              { Y = X; }
-clog(Y)    ::= CLOG INTEGER(X).               { Y = X; }
+wal(Y)     ::= WAL INTEGER(X).                { Y = X; }
 comp(Y)    ::= COMP INTEGER(X).               { Y = X; }
 prec(Y)    ::= PRECISION STRING(X).           { Y = X; }
 
 %type db_optr {SCreateDBInfo}
 db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
 
-db_optr(Y) ::= db_optr(Z) tables(X).         { Y = Z; Y.tablesPerVnode = strtol(X.z, NULL, 10); }
+db_optr(Y) ::= db_optr(Z) tables(X).         { Y = Z; Y.maxTablesPerVnode = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) cache(X).          { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) replica(X).        { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) days(X).           { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); }
-db_optr(Y) ::= db_optr(Z) rows(X).           { Y = Z; Y.rowPerFileBlock = strtol(X.z, NULL, 10); }
+db_optr(Y) ::= db_optr(Z) minrows(X).        { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); }
-db_optr(Y) ::= db_optr(Z) ablocks(X).        { Y = Z; Y.numOfAvgCacheBlocks = strtod(X.z, NULL); }
+db_optr(Y) ::= db_optr(Z) maxrows(X).        { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); }
-db_optr(Y) ::= db_optr(Z) tblocks(X).        { Y = Z; Y.numOfBlocksPerTable = strtol(X.z, NULL, 10); }
+db_optr(Y) ::= db_optr(Z) blocks(X).         { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) ctime(X).          { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); }
-db_optr(Y) ::= db_optr(Z) clog(X).           { Y = Z; Y.commitLog = strtol(X.z, NULL, 10); }
+db_optr(Y) ::= db_optr(Z) wal(X).            { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) comp(X).           { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) prec(X).           { Y = Z; Y.precision = X; }
 db_optr(Y) ::= db_optr(Z) keep(X).           { Y = Z; Y.keep = X; }
@@ -245,7 +244,11 @@ db_optr(Y) ::= db_optr(Z) keep(X).           { Y = Z; Y.keep = X; }
 alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
 
 alter_db_optr(Y) ::= alter_db_optr(Z) replica(X).     { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
-alter_db_optr(Y) ::= alter_db_optr(Z) tables(X).      { Y = Z; Y.tablesPerVnode = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) tables(X).      { Y = Z; Y.maxTablesPerVnode = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) keep(X).        { Y = Z; Y.keep = X; }
+alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X).      { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) comp(X).        { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) wal(X).         { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
 
 %type typename {TAOS_FIELD}
 typename(A) ::= ids(X).              { tSQLSetColumnType (&A, &X); }
@@ -652,5 +655,5 @@ cmd ::= KILL QUERY IPTOKEN(X) COLON(Z) INTEGER(Y) COLON(K) INTEGER(F).        {X
   DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD
   LIKE MATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL
   COUNT SUM AVG MIN MAX FIRST LAST TOP BOTTOM STDDEV PERCENTILE APERCENTILE LEASTSQUARES HISTOGRAM DIFF
-  SPREAD TWA INTERP LAST_ROW RATE IRATE SUM_RATE SUM_IRATE AVG_RATE AVG_IRATE NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT
+  SPREAD TWA INTERP LAST_ROW RATE IRATE SUM_RATE SUM_IRATE AVG_RATE AVG_IRATE TBID NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT
   METRIC TBNAME JOIN METRICS STABLE NULL INSERT INTO VALUES.

src/query/inc/tsqlfunction.h

@@ -119,6 +119,7 @@ typedef struct SArithmeticSupport {
   SExprInfo   *pArithExpr;
   int32_t      numOfCols;
   SColumnInfo *colList;
+  SArray*      exprList;   // client side used
   int32_t      offset;
   char**       data;
 } SArithmeticSupport;
@@ -220,7 +221,7 @@ typedef struct SQLAggFuncElem {
 #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
 
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
-                          int16_t *len, int16_t *interResBytes, int16_t extLength, bool isSuperTable);
+                          int16_t *len, int16_t *interBytes, int16_t extLength, bool isSuperTable);
 
 #define IS_STREAM_QUERY_VALID(x)  (((x)&TSDB_FUNCSTATE_STREAM) != 0)
 #define IS_MULTIOUTPUT(x)         (((x)&TSDB_FUNCSTATE_MO) != 0)
@@ -244,21 +245,8 @@ enum {
   BLK_DATA_ALL_NEEDED = 0x3,
 };
 
-#define IS_FILE_BLOCK(x) (((x)&BLK_FILE_BLOCK) != 0)
-
-#define SET_FILE_BLOCK_FLAG(x) \
-  do {                         \
-    (x) &= (~BLK_CACHE_BLOCK); \
-    (x) |= BLK_FILE_BLOCK;     \
-  } while (0);
-
-#define SET_CACHE_BLOCK_FLAG(x) ((x) = BLK_CACHE_BLOCK | BLK_BLOCK_LOADED);
-
 #define SET_DATA_BLOCK_NOT_LOADED(x) ((x) &= (~BLK_BLOCK_LOADED));
 
-#define SET_DATA_BLOCK_LOADED(x) ((x) |= BLK_BLOCK_LOADED);
-#define IS_DATA_BLOCK_LOADED(x) (((x)&BLK_BLOCK_LOADED) != 0)
-
 typedef struct STwaInfo {
   TSKEY   lastKey;
   int8_t  hasResult;  // flag to denote has value
@@ -290,7 +278,6 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi
 
 bool stableQueryFunctChanged(int32_t funcId);
 
-
 void resetResultInfo(SResultInfo *pResInfo);
 void initResultInfo(SResultInfo *pResInfo);
 void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable);

src/query/src/qast.c

@@ -544,13 +544,11 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
   setQueryCond(pQueryInfo, &cond);
 
   if (cond.start != NULL) {
-    iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
+    iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
   } else {
-    iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.end->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+    iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.end->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
   }
   
-  __compar_fn_t func = getKeyComparFunc(pSkipList->keyInfo.type);
-  
   if (cond.start != NULL) {
     int32_t optr = cond.start->optr;
     
@@ -558,7 +556,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
       while(tSkipListIterNext(iter)) {
         SSkipListNode* pNode = tSkipListIterGet(iter);
 
-        int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v);
+        int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v);
         if (ret == 0) {
           taosArrayPush(result, SL_GET_NODE_DATA(pNode));
         } else {
@@ -573,7 +571,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
         SSkipListNode* pNode = tSkipListIterGet(iter);
     
         if (comp) {
-          ret = func(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v);
+          ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v);
           assert(ret >= 0);
         }
         
@@ -600,7 +598,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
         SSkipListNode* pNode = tSkipListIterGet(iter);
       
         if (comp) {
-          ret = func(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v);
+          ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v);
           assert(ret <= 0);
         }
         
@@ -708,7 +706,7 @@ static void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pR
   }
 }
 
-static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *param) {
+static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) {
   tExprNode *pLeft = pExpr->_node.pLeft;
   tExprNode *pRight = pExpr->_node.pRight;
 
@@ -747,7 +745,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *p
  * @param pSchema   tag schemas
  * @param fp        filter callback function
  */
-static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilterSupp *param) {
+static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) {
   size_t size = taosArrayGetSize(pResult);
   
   SArray* array = taosArrayInit(size, POINTER_BYTES);
@@ -763,7 +761,7 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilte
 }
 
 
-static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SBinaryFilterSupp *param ) {
+static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) {
   SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
 
   while (tSkipListIterNext(iter)) {
@@ -813,7 +811,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
 
 
 // post-root order traverse syntax tree
-void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) {
+void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) {
   if (pExpr == NULL) {
     return;
   }
@@ -979,7 +977,8 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
 
     } else if (pRight->nodeType == TSQL_NODE_COL) {  // 12 + columnRight
       // column data specified on right-hand-side
-      char *            pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
+      char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
+      
       _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr);
       fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, order);
 

src/query/src/qparserImpl.c

@@ -52,6 +52,8 @@ int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pStr) {
         Parse(pParser, 0, t0, pSQLInfo);
         goto abort_parse;
       }
+      
+      case TK_QUESTION:
       case TK_ILLEGAL: {
         snprintf(pSQLInfo->pzErrMsg, tListLen(pSQLInfo->pzErrMsg), "unrecognized token: \"%s\"", t0.z);
         pSQLInfo->valid = false;
@@ -818,7 +820,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI
 
   pInfo->pDCLInfo->dbOpt = *pDB;
   pInfo->pDCLInfo->dbOpt.dbname = *pToken;
-  pInfo->pDCLInfo->dbOpt.ignoreExists = (pIgExists != NULL);
+  pInfo->pDCLInfo->dbOpt.ignoreExists = (pIgExists->z != NULL);
 }
 
 void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) {
@@ -886,16 +888,16 @@ void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip) {
 }
 
 void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
-  pDBInfo->numOfBlocksPerTable = 50;
   pDBInfo->compressionLevel = -1;
 
-  pDBInfo->commitLog = -1;
+  pDBInfo->walLevel = -1;
   pDBInfo->commitTime = -1;
-  pDBInfo->tablesPerVnode = -1;
+  pDBInfo->maxTablesPerVnode = -1;
-  pDBInfo->numOfAvgCacheBlocks = -1;
 
   pDBInfo->cacheBlockSize = -1;
-  pDBInfo->rowPerFileBlock = -1;
+  pDBInfo->numOfBlocks = -1;
+  pDBInfo->maxRowsPerBlock = -1;
+  pDBInfo->minRowsPerBlock = -1;
   pDBInfo->daysPerFile = -1;
 
   pDBInfo->replica = -1;

src/query/src/qtokenizer.c

@@ -117,12 +117,13 @@ static SKeyword keywordTable[] = {
     {"KEEP",         TK_KEEP},
     {"REPLICA",      TK_REPLICA},
     {"DAYS",         TK_DAYS},
-    {"ROWS",         TK_ROWS},
+    {"MINROWS",      TK_MINROWS},
+    {"MAXROWS",      TK_MAXROWS},
+    {"BLOCKS",       TK_BLOCKS},
+    {"MAXTABLES",    TK_MAXTABLES},
     {"CACHE",        TK_CACHE},
-    {"ABLOCKS",      TK_ABLOCKS},
-    {"TBLOCKS",      TK_TBLOCKS},
     {"CTIME",        TK_CTIME},
-    {"CLOG",         TK_CLOG},
+    {"WAL",          TK_WAL},
     {"COMP",         TK_COMP},
     {"PRECISION",    TK_PRECISION},
     {"LP",           TK_LP},
@@ -224,6 +225,7 @@ static SKeyword keywordTable[] = {
     {"TBNAME",       TK_TBNAME},
     {"JOIN",         TK_JOIN},
     {"METRICS",      TK_METRICS},
+    {"TBID",         TK_TBID},
     {"STABLE",       TK_STABLE},
     {"FILE",         TK_FILE},
     {"VNODES",       TK_VNODES},
@@ -265,11 +267,15 @@ static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT;
 
 int tSQLKeywordCode(const char* z, int n) {
   pthread_once(&keywordsHashTableInit, doInitKeywordsTable);
-
+  
-  char key[128] = {0};
+  char key[512] = {0};
+  if (n > tListLen(key)) { // too long token, can not be any other token type
+    return TK_ID;
+  }
+  
   for (int32_t j = 0; j < n; ++j) {
     if (z[j] >= 'a' && z[j] <= 'z') {
-      key[j] = (char)(z[j] & 0xDF);  // touppercase and set the null-terminated
+      key[j] = (char)(z[j] & 0xDF);  // to uppercase and set the null-terminated
     } else {
       key[j] = z[j];
     }

src/query/src/queryExecutor.c

@@ -39,10 +39,10 @@
 #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0)
 #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP)
 
-#define IS_MASTER_SCAN(runtime) (((runtime)->scanFlag & 1u) == MASTER_SCAN)
+#define IS_MASTER_SCAN(runtime)        ((runtime)->scanFlag == MASTER_SCAN)
-#define IS_SUPPLEMENT_SCAN(runtime) ((runtime)->scanFlag == SUPPLEMENTARY_SCAN)
+#define IS_REVERSE_SCAN(runtime)       ((runtime)->scanFlag == SUPPLEMENTARY_SCAN)
-#define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN)
+#define SET_MASTER_SCAN_FLAG(runtime)  ((runtime)->scanFlag = MASTER_SCAN)
-#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN)
+#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN)
 
 #define GET_QINFO_ADDR(x) ((void *)((char *)(x)-offsetof(SQInfo, runtimeEnv)))
 
@@ -1101,7 +1101,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
   }
 
   // in the supplementary scan, only the following functions need to be executed
-  if (IS_SUPPLEMENT_SCAN(pRuntimeEnv) &&
+  if (IS_REVERSE_SCAN(pRuntimeEnv) &&
       !(functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST ||
         functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) {
     return false;
@@ -1402,7 +1402,7 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) {
 
 static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery) {
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-    setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interResBytes, isStableQuery);
+    setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interBytes, isStableQuery);
   }
 }
 
@@ -2450,8 +2450,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
   qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d",
          GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order);
 
-  TsdbQueryHandleT pQueryHandle =
+  TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle;
-      pRuntimeEnv->scanFlag == MASTER_SCAN ? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle;
   while (tsdbNextDataBlock(pQueryHandle)) {
     if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
       return 0;
@@ -2835,11 +2834,12 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
       return;  // failed to save data in the disk
     }
 
-    // set current query completed
+    // check if all results has been sent to client
-    //    if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == pQInfo->pSidSet->numOfSubSet) {
+    int32_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
-    //      pQInfo->tableIndex = pQInfo->pSidSet->numOfTables;
+    if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) {
-    //      return;
+      pQInfo->tableIndex = pQInfo->groupInfo.numOfTables;  // set query completed
-    //    }
+      return;
+    }
   }
 
   SQueryRuntimeEnv *   pRuntimeEnv = &pQInfo->runtimeEnv;
@@ -3087,7 +3087,31 @@ void setTableDataInfo(STableQueryInfo *pTableQueryInfo, int32_t tableIndex, int3
   pTableQueryInfo->tableIndex = tableIndex;
 }
 
-static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) {
+static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) {
+  if (pTableQueryInfo == NULL) {
+    return;
+  }
+  
+  // order has change already!
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
+  if (!QUERY_IS_ASC_QUERY(pQuery)) {
+    assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step);
+  } else {
+    assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step);
+  }
+  
+  pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step;
+  
+  SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY);
+  pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
+  
+  SWITCH_ORDER(pTableQueryInfo->cur.order);
+  pTableQueryInfo->cur.vgroupIndex = -1;
+}
+
+static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) {
+  SQuery* pQuery = pQInfo->runtimeEnv.pQuery;
+  
   for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
     SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i);
     if (!pStatus->closed) {
@@ -3108,18 +3132,32 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *
       }
     }
   }
+  
+  int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
+  
+  for(int32_t i = 0; i < numOfGroups; ++i) {
+    SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i);
+    qTrace("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1);
+
+    size_t t = taosArrayGetSize(group);
+    for (int32_t j = 0; j < t; ++j) {
+      SGroupItem *item = taosArrayGet(group, j);
+      updateTableQueryInfoForReverseScan(pQuery, item->info);
+    }
+  }
 }
 
-void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) {
+void disableFuncInReverseScan(SQInfo *pQInfo) {
+  SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *pQuery = pRuntimeEnv->pQuery;
   int32_t order = pQuery->order.order;
 
   // group by normal columns and interval query on normal table
   SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
   if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) {
-    doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order);
+    disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order);
   } else {  // for simple result of table query,
-    for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
+    for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {  // todo refactor
       int32_t functId = pQuery->pSelectExpr[j].base.functionId;
 
       SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j];
@@ -3134,34 +3172,10 @@ void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) {
   }
 }
 
-void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) {
-  SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-  SQuery *          pQuery = pRuntimeEnv->pQuery;
-
-  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-    pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u;
-  }
-
-  if (isIntervalQuery(pQuery)) {
-    //    for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) {
-    //      STableQueryInfo *pTableQueryInfo = pQInfo->pTableQueryInfo[i].pTableQInfo;
-    //      SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo;
-    //
-    //      doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order);
-    //    }
-  } else {
-    SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
-    doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order);
-  }
-
-  pQuery->order.order = (pQuery->order.order) ^ 1u;
-}
-
 void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-    SWITCH_ORDER(pRuntimeEnv->pCtx[i]
+    SWITCH_ORDER(pRuntimeEnv->pCtx[i] .order);
-                     .order);  // = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
   }
 }
 
@@ -3358,7 +3372,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
   SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
 
   SWITCH_ORDER(pQuery->order.order);
-  SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv);
+  SET_REVERSE_SCAN_FLAG(pRuntimeEnv);
 
   STsdbQueryCond cond = {
       .twindow = pQuery->window,
@@ -3376,7 +3390,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
 
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
   switchCtxOrder(pRuntimeEnv);
-  disableFuncInReverseScan(pRuntimeEnv);
+  disableFuncInReverseScan(pQInfo);
 }
 
 static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) {
@@ -3533,28 +3547,6 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols)
   free(pTableQueryInfo);
 }
 
-void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) {
-  if (pTableQueryInfo == NULL) {
-    return;
-  }
-
-  // order has change already!
-  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
-  if (!QUERY_IS_ASC_QUERY(pQuery)) {
-    assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step);
-  } else {
-    assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step);
-  }
-
-  pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step;
-
-  SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY);
-  pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
-
-  pTableQueryInfo->cur.order = pTableQueryInfo->cur.order ^ 1u;
-  pTableQueryInfo->cur.vgroupIndex = -1;
-}
-
 void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
@@ -3943,9 +3935,16 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
     data += bytes * numOfRows;
   }
 
+  
   // all data returned, set query over
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-    setQueryStatus(pQuery, QUERY_OVER);
+    if (pQInfo->runtimeEnv.stableQuery && isIntervalQuery(pQuery)) {
+      if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
+        setQueryStatus(pQuery, QUERY_OVER);
+      }
+    } else {
+      setQueryStatus(pQuery, QUERY_OVER);
+    }
   }
 }
 
@@ -4233,10 +4232,12 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
   
   
   // normal query setup the queryhandle here
-  if (isFirstLastRowQuery(pQuery) && !isSTableQuery) {  // in case of last_row query, invoke a different API.
+  if (!onlyQueryTags(pQuery)) {
-    pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo);
+    if (!isSTableQuery && isFirstLastRowQuery(pQuery)) {  // in case of last_row query, invoke a different API.
-  } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) {
+      pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo);
-    pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
+    } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) {
+      pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
+    }
   }
   
   pQInfo->tsdb = tsdb;
@@ -4366,7 +4367,8 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) {
 
   int64_t st = taosGetTimestampMs();
 
-  TsdbQueryHandleT *pQueryHandle = pRuntimeEnv->pQueryHandle;
+  TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle;
+  
   while (tsdbNextDataBlock(pQueryHandle)) {
     if (isQueryKilled(pQInfo)) {
       break;
@@ -4398,7 +4400,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) {
       }
     }
 
-    assert(pTableQueryInfo != NULL && pTableQueryInfo != NULL);
+    assert(pTableQueryInfo != NULL);
     restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo);
 
     SDataStatis *pStatis = NULL;
@@ -4757,28 +4759,35 @@ static void createTableQueryInfo(SQInfo *pQInfo) {
   }
 }
 
-static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) {
-  //  SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
-
-  //  for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) {
-  //    STableQueryInfo *pTableQueryInfo = pQInfo->pTableQueryInfo[i].pTableQInfo;
-  //    changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo);
-  //  }
-}
-
 static void doSaveContext(SQInfo *pQInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
-  SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv);
+  SET_REVERSE_SCAN_FLAG(pRuntimeEnv);
-  disableFuncForReverseScan(pQInfo, pQuery->order.order);
-
-  if (pRuntimeEnv->pTSBuf != NULL) {
-    pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u;
-  }
-
   SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
-  prepareQueryInfoForReverseScan(pQInfo);
+  SWITCH_ORDER(pQuery->order.order);
+  
+  if (pRuntimeEnv->pTSBuf != NULL) {
+    pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order;
+  }
+  
+  STsdbQueryCond cond = {
+      .twindow = pQuery->window,
+      .order   = pQuery->order.order,
+      .colList = pQuery->colList,
+      .numOfCols = pQuery->numOfCols,
+  };
+  
+  // clean unused handle
+  if (pRuntimeEnv->pSecQueryHandle != NULL) {
+    tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
+  }
+  
+  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo);
+  
+  setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
+  switchCtxOrder(pRuntimeEnv);
+  disableFuncInReverseScan(pQInfo);
 }
 
 static void doRestoreContext(SQInfo *pQInfo) {
@@ -4833,8 +4842,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
       copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
     }
 
-    pQuery->rec.rows += pQuery->rec.rows;
-
     if (pQuery->rec.rows == 0) {
       //      vnodePrintQueryStatistics(pSupporter);
     }
@@ -5540,7 +5547,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo
 
     int32_t param = pExprs[i].base.arg[0].argValue.i64;
     if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes,
-                          &pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
+                          &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
       tfree(pExprs);
       return TSDB_CODE_INVALID_QUERY_MSG;
     }
@@ -5566,7 +5573,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo
 
       int32_t ret =
           getResultDataInfo(pCol->type, pCol->bytes, functId, pExprs[i].base.arg[0].argValue.i64,
-                            &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, tagLen, isSuperTable);
+                            &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable);
       assert(ret == TSDB_CODE_SUCCESS);
     }
   }
@@ -5780,10 +5787,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
   pQuery->rec.threshold = 4000;
 
   for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
-    assert(pExprs[col].interResBytes >= pExprs[col].bytes);
+    assert(pExprs[col].interBytes >= pExprs[col].bytes);
 
     // allocate additional memory for interResults that are usually larger then final results
-    size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interResBytes + sizeof(SData);
+    size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(SData);
     pQuery->sdata[col] = (SData *)calloc(1, size);
     if (pQuery->sdata[col] == NULL) {
       goto _cleanup;
@@ -6285,7 +6292,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
       SGroupItem* item = taosArrayGet(pa, i);
     
       char* output = pQuery->sdata[0]->data + i * rsize;
-      *(int64_t*) output = item->id.uid;  // memory align problem
+      varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);
+      
+      output = varDataVal(output);
+      *(int64_t*) output = item->id.uid;  // memory align problem, todo serialize
       output += sizeof(item->id.uid);
       
       *(int32_t*) output = item->id.tid;

src/rpc/src/rpcMain.c

@@ -226,17 +226,6 @@ void *rpcOpen(const SRpcInit *pInit) {
   pRpc->cfp = pInit->cfp;
   pRpc->afp = pInit->afp;
 
-  pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, 
-                                                  pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
-  pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, 
-                                                  pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
-
-  if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) {
-    tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort);
-    rpcClose(pRpc);
-    return NULL;
-  }
-
   size_t size = sizeof(SRpcConn) * pRpc->sessions;
   pRpc->connList = (SRpcConn *)calloc(1, size);
   if (pRpc->connList == NULL) {
@@ -277,6 +266,17 @@ void *rpcOpen(const SRpcInit *pInit) {
 
   pthread_mutex_init(&pRpc->mutex, NULL);
 
+  pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, 
+                                                  pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
+  pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, 
+                                                  pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
+
+  if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) {
+    tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort);
+    rpcClose(pRpc);
+    return NULL;
+  }
+
   tTrace("%s RPC is openned, numOfThreads:%d", pRpc->label, pRpc->numOfThreads);
 
   return pRpc;
@@ -363,7 +363,7 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
   // connection type is application specific. 
   // for TDengine, all the query, show commands shall have TCP connection
   char type = pMsg->msgType;
-  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_RETRIEVE ||
+  if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH ||
       type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META ||
       type == TSDB_MSG_TYPE_CM_SHOW )
     pContext->connType = RPC_CONN_TCPC;
@@ -793,7 +793,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
 
   pConn->chandle = pRecv->chandle;
   pConn->peerIp = pRecv->ip; 
-  if (pConn->peerPort == 0) pConn->peerPort = pRecv->port;
+  pConn->peerPort = pRecv->port;
   if (pHead->port) pConn->peerPort = htons(pHead->port); 
 
   terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen);
@@ -802,7 +802,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
   pHead->code = htonl(pHead->code);
 
   if (terrno == 0) {
-    if (pHead->msgType != TSDB_MSG_TYPE_REG && pHead->encrypt) {
+    if (pHead->encrypt) {
       // decrypt here
     }
 
@@ -869,9 +869,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   pConn = rpcProcessMsgHead(pRpc, pRecv);
 
   if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) {
-    tTrace("%s %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d",
+    tTrace("%s %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
         pRpc->label, pConn, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, 
-        pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->port);
+        pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
   }
 
   int32_t code = terrno;
@@ -934,11 +934,20 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
     // for UDP, port may be changed by server, the port in ipSet shall be used for cache
     rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType);    
 
+    if (pHead->code == TSDB_CODE_REDIRECT) { 
+      pContext->redirect++;
+      if (pContext->redirect > TSDB_MAX_REPLICA) {
+        pHead->code = TSDB_CODE_NETWORK_UNAVAIL; 
+        tWarn("%s %p, too many redirects, quit", pRpc->label, pConn);
+      }
+    }
+
     if (pHead->code == TSDB_CODE_REDIRECT) {
-      pContext->redirect = 1;
       pContext->numOfTry = 0;
       memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet));
       tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps);
+      for (int i=0; i<pContext->ipSet.numOfIps; ++i) 
+        pContext->ipSet.port[i] = htons(pContext->ipSet.port[i]);
       rpcSendReqToServer(pRpc, pContext);
     } else if (pHead->code == TSDB_CODE_NOT_READY) {
       pContext->code = pHead->code;
@@ -1083,9 +1092,10 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
              pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn,
              pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   } else {
+    if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
     if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
-      tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
+      tTrace( "%s %p, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
-          pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, 
+          pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, 
           htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 

src/rpc/src/rpcTcp.c

@@ -188,7 +188,7 @@ static void taosAcceptTcpConnection(void *arg) {
   sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
   if (sockFd < 0) return; 
 
-  tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
+  tTrace("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port);
 
   while (1) {
     socklen_t addrlen = sizeof(caddr);

src/rpc/test/rserver.c

@@ -151,7 +151,7 @@ int main(int argc, char *argv[]) {
       commit = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
       rpcDebugFlag = atoi(argv[++i]);
-      ddebugFlag = rpcDebugFlag;
+      dDebugFlag = rpcDebugFlag;
       uDebugFlag = rpcDebugFlag;
     } else {
       printf("\nusage: %s [options] \n", argv[0]);

src/tsdb/inc/tsdbMain.h

@@ -21,6 +21,7 @@
 #include "tskiplist.h"
 #include "tutil.h"
 #include "tlog.h"
+#include "tcoding.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -94,6 +95,7 @@ typedef struct STable {
   void *         streamHandler;  // TODO
   TSKEY          lastKey;        // lastkey inserted in this table, initialized as 0, TODO: make a structure
   struct STable *next;           // TODO: remove the next
+  struct STable *prev;
 } STable;
 
 #define TSDB_GET_TABLE_LAST_KEY(pTable) ((pTable)->lastKey)

src/tsdb/src/tsdbFile.c

@@ -178,9 +178,9 @@ void tsdbFitRetention(STsdbRepo *pRepo) {
   SFileGroup *pGroup = pFileH->fGroup;
 
   int mfid =
-      tsdbGetKeyFileId(taosGetTimestamp(pRepo->config.precision), pRepo->config.daysPerFile, pRepo->config.precision);
+      tsdbGetKeyFileId(taosGetTimestamp(pRepo->config.precision), pRepo->config.daysPerFile, pRepo->config.precision) - pFileH->maxFGroups + 3;
 
-  while (pGroup[0].fileId < mfid) {
+  while (pFileH->numOfFGroups > 0 && pGroup[0].fileId < mfid) {
     tsdbRemoveFileGroup(pFileH, pGroup[0].fileId);
   }
 }

src/tsdb/src/tsdbMain.c

@@ -90,6 +90,7 @@ void tsdbFreeCfg(STsdbCfg *pCfg) {
 int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) {
 
   if (mkdir(rootDir, 0755) != 0) {
+    tsdbError("id %d: failed to create rootDir! rootDir %s, reason %s", pCfg->tsdbId, rootDir, strerror(errno));
     if (errno == EACCES) {
       return TSDB_CODE_NO_DISK_PERMISSIONS;
     } else if (errno == ENOSPC) {
@@ -611,14 +612,20 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   if (pCfg->precision == -1) {
     pCfg->precision = TSDB_DEFAULT_PRECISION;
   } else {
-    if (!IS_VALID_PRECISION(pCfg->precision)) return -1;
+    if (!IS_VALID_PRECISION(pCfg->precision)) {
+      tsdbError("id %d: invalid precision configuration! precision %d", pCfg->tsdbId, pCfg->precision);
+      return -1;
+    }
   }
 
   // Check compression
   if (pCfg->compression == -1) {
     pCfg->compression = TSDB_DEFAULT_COMPRESSION;
   } else {
-    if (!IS_VALID_COMPRESSION(pCfg->compression)) return -1;
+    if (!IS_VALID_COMPRESSION(pCfg->compression)) {
+      tsdbError("id %d: invalid compression configuration! compression %d", pCfg->tsdbId, pCfg->precision);
+      return -1;
+    }
   }
 
   // Check tsdbId
@@ -628,29 +635,49 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   if (pCfg->maxTables == -1) {
     pCfg->maxTables = TSDB_DEFAULT_TABLES;
   } else {
-    if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) return -1;
+    if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) {
+      tsdbError("id %d: invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d",
+                pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES);
+      return -1;
+    }
   }
 
   // Check daysPerFile
   if (pCfg->daysPerFile == -1) {
     pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
   } else {
-    if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
+    if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
+      tsdbError(
+          "id %d: invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE "
+          "%d",
+          pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
+      return -1;
+    }
   }
 
   // Check minRowsPerFileBlock and maxRowsPerFileBlock
   if (pCfg->minRowsPerFileBlock == -1) {
     pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
   } else {
-    if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK)
+    if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
+      tsdbError(
+          "id %d: invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d "
+          "TSDB_MAX_MIN_ROW_FBLOCK %d",
+          pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
       return -1;
+    }
   }
 
   if (pCfg->maxRowsPerFileBlock == -1) {
     pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
   } else {
-    if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK)
+    if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
+      tsdbError(
+          "id %d: invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d "
+          "TSDB_MAX_MAX_ROW_FBLOCK %d",
+          pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
       return -1;
+    }
   }
 
   if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) return -1;
@@ -659,7 +686,13 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   if (pCfg->keep == -1) {
     pCfg->keep = TSDB_DEFAULT_KEEP;
   } else {
-    if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) return -1;
+    if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) {
+      tsdbError(
+          "id %d: invalid keep configuration! keep %d TSDB_MIN_KEEP %d "
+          "TSDB_MAX_KEEP %d",
+          pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
+      return -1;
+    }
   }
 
   return 0;
@@ -716,15 +749,22 @@ static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname) {
 }
 
 static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
+  STsdbCfg *pCfg = &pRepo->config;
   if (tsdbSaveConfig(pRepo) < 0) return -1;
 
   char dirName[128] = "\0";
   if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1;
 
   if (mkdir(dirName, 0755) < 0) {
+    tsdbError("id %d: failed to create repository directory! reason %s", pRepo->config.tsdbId, strerror(errno));
     return -1;
   }
 
+  tsdbError(
+      "id %d: set up tsdb environment succeed! cacheBlockSize %d, totalBlocks %d, maxTables %d, daysPerFile %d, keep "
+      "%d, minRowsPerFileBlock %d, maxRowsPerFileBlock %d, precision %d, compression%d",
+      pRepo->config.tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep,
+      pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression);
   return 0;
 }
 
@@ -811,7 +851,8 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY
   STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
   STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId);
   if (pTable == NULL) {
-    tsdbError("failed to get table for insert, uid:%" PRIu64 ", tid:%d", tableId.uid, tableId.tid);
+    tsdbError("id %d: failed to get table for insert, uid:%" PRIu64 ", tid:%d", pRepo->config.tsdbId, pBlock->uid,
+              pBlock->tid);
     return TSDB_CODE_INVALID_TABLE_ID;
   }
 

src/tsdb/src/tsdbMeta.c

@@ -13,10 +13,10 @@
 static int     tsdbFreeTable(STable *pTable);
 static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
 static int     tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx);
-static int     tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbEstimateTableEncodeSize(STable *pTable);
+static int     tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable);
 
 /**
  * Encode a TSDB table object as a binary content
@@ -375,21 +375,9 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) {
   STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid);
   if (pTable == NULL) return -1;
 
-  if (pTable->type == TSDB_SUPER_TABLE) {
+  if (tsdbRemoveTableFromMeta(pMeta, pTable) < 0) return -1;
-    // TODO: implement drop super table
-    return -1;
-  } else {
-    pMeta->tables[pTable->tableId.tid] = NULL;
-    pMeta->nTables--;
-    assert(pMeta->nTables >= 0);
-    if (pTable->type == TSDB_CHILD_TABLE) {
-      tsdbRemoveTableFromIndex(pMeta, pTable);
-    }
-
-    tsdbFreeTable(pTable);
-  }
-
   return 0;
+
 }
 
 // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) {
@@ -445,10 +433,12 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
     if (pMeta->superList == NULL) {
       pMeta->superList = pTable;
       pTable->next = NULL;
+      pTable->prev = NULL;
     } else {
-      STable *pTemp = pMeta->superList;
+      pTable->next = pMeta->superList;
+      pTable->prev = NULL;
+      pTable->next->prev = pTable;
       pMeta->superList = pTable;
-      pTable->next = pTemp;
     }
   } else {
     // add non-super table to the array
@@ -467,22 +457,50 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
     if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes;
   }
 
-  return tsdbAddTableIntoMap(pMeta, pTable);
+  if (taosHashPut(pMeta->map, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), sizeof(pTable)) < 0) {
-}
-
-// static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
-//   // TODO
-//   return 0;
-// }
-
-static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable) {
-  // TODO: add the table to the map
-  int64_t uid = pTable->tableId.uid;
-  if (taosHashPut(pMeta->map, (char *)(&uid), sizeof(uid), (void *)(&pTable), sizeof(pTable)) < 0) {
     return -1;
   }
   return 0;
 }
+
+static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
+  if (pTable->type == TSDB_SUPER_TABLE) {
+    SSkipListIterator  *pIter = tSkipListCreateIter(pTable->pIndex);
+    while (tSkipListIterNext(pIter)) {
+      STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+      ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE);
+
+      pMeta->tables[tTable->tableId.tid] = NULL;
+      taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid));
+      pMeta->nTables--;
+      tsdbFreeTable(tTable);
+    }
+
+    tSkipListDestroyIter(pIter);
+
+    // TODO: Remove the table from the list
+    if (pTable->prev != NULL) {
+      pTable->prev->next = pTable->next;
+      if (pTable->next != NULL) {
+        pTable->next->prev = pTable->prev;
+      }
+    } else {
+      pMeta->superList = pTable->next;
+    }
+  } else {
+    pMeta->tables[pTable->tableId.tid] = NULL;
+    if (pTable->type == TSDB_CHILD_TABLE) {
+      tsdbRemoveTableFromIndex(pMeta, pTable);
+    }
+
+    pMeta->nTables--;
+  }
+
+  tsdbFreeTable(pTable);
+  taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid));
+  return 0;
+}
+
 static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) {
   assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL);
   STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid);
@@ -502,13 +520,19 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) {
   memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES);
   
   tSkipListPut(list, pNode);
-
   return 0;
 }
 
 static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
-  assert(pTable->type == TSDB_CHILD_TABLE);
+  assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL);
-  // TODO
+  
+  STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid);
+  assert(pSTable != NULL);
+  
+  char* key = dataRowTuple(pTable->tagVal); // key
+  bool ret = tSkipListRemove(pSTable->pIndex, key);
+  
+  assert(ret);
   return 0;
 }
 
@@ -535,5 +559,5 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) {
 char *getTupleKey(const void * data) {
   SDataRow row = (SDataRow)data;
 
-  return POINTER_DRIFT(row, TD_DATA_ROW_HEAD_SIZE);
+  return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE);
 }

src/tsdb/src/tsdbRead.c

@@ -335,11 +335,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
         pCheckInfo->compSize = compIndex->len;
       }
       
-      // tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo);
+      tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb);
-      STable* pTable = tsdbGetTableByUid(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->tableId.uid);
-      assert(pTable != NULL);
-      
-      tsdbSetHelperTable(&pQueryHandle->rhelper, pTable, pQueryHandle->pTsdb);
 
       tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
       SCompInfo* pCompInfo = pCheckInfo->pCompInfo;
@@ -472,6 +468,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
       filterDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
     } else {  // the whole block is loaded in to buffer
       pQueryHandle->realNumOfRows = pBlock->numOfPoints;
+      cur->pos = 0;
     }
   } else {
     // query ended in current block
@@ -491,6 +488,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
       filterDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
     } else {
       pQueryHandle->realNumOfRows = pBlock->numOfPoints;
+      cur->pos = pBlock->numOfPoints - 1;
     }
   }
 
@@ -568,7 +566,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
   SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
   
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
-
+  
   int32_t endPos = cur->pos;
   if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
     endPos = blockInfo.rows - 1;
@@ -612,7 +610,6 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
   int32_t reqCols = taosArrayGetSize(pQueryHandle->pColumns);
   
   for (int32_t i = 0; i < reqCols; ++i) {
-//    int16_t colId = *(int16_t*)taosArrayGet(sa, i);
     SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
     int32_t bytes = pCol->info.bytes;
 
@@ -1236,12 +1233,6 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) {
   return TSDB_CODE_SUCCESS;
 }
 
-typedef struct SExprTreeSupporter {
-  SSchema* pTagSchema;
-  int32_t  numOfTags;
-  int32_t  optr;
-} SExprTreeSupporter;
-
 /**
  * convert the result pointer to table id instead of table object pointer
  * @param pRes
@@ -1252,7 +1243,7 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) {
   }
 
   size_t size = taosArrayGetSize(pTableList);
-  for (int32_t i = 0; i < size; ++i) {
+  for (int32_t i = 0; i < size; ++i) {  // todo speedup  by using reserve space.
     STable* pTable = taosArrayGetP(pTableList, i);
     taosArrayPush(pRes, &pTable->tableId);
   }
@@ -1273,16 +1264,15 @@ static void destroyHelper(void* param) {
   free(param);
 }
 
-static int32_t getTagColumnInfo(SExprTreeSupporter* pSupporter, SSchema* pSchema) {
+static int32_t getTagColumnIndex(STSchema* pTSchema, SSchema* pSchema) {
   // filter on table name(TBNAME)
   if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) {
     return TSDB_TBNAME_COLUMN_INDEX;
   }
-
+  
-  for(int32_t i = 0; i < pSupporter->numOfTags; ++i) {
+  for(int32_t i = 0; i < schemaNCols(pTSchema); ++i) {
-    if (pSupporter->pTagSchema[i].bytes == pSchema->bytes &&
+    STColumn* pColumn = &pTSchema->columns[i];
-        pSupporter->pTagSchema[i].type  == pSchema->type  &&
+    if (pColumn->bytes == pSchema->bytes && pColumn->type  == pSchema->type  && pColumn->colId == pSchema->colId) {
-        pSupporter->pTagSchema[i].colId == pSchema->colId) {
       return i;
     }
   }
@@ -1298,21 +1288,22 @@ void filterPrepare(void* expr, void* param) {
 
   int32_t i = 0;
   pExpr->_node.info = calloc(1, sizeof(tQueryInfo));
-
+  
-  SExprTreeSupporter* pSupporter = (SExprTreeSupporter*)param;
+  STSchema* pTSSchema = (STSchema*) param;
 
   tQueryInfo* pInfo = pExpr->_node.info;
   tVariant*   pCond = pExpr->_node.pRight->pVal;
   SSchema*    pSchema = pExpr->_node.pLeft->pSchema;
 
   // todo : if current super table does not change schema yet, this function may failed, add test case
-  int32_t index = getTagColumnInfo(pSupporter, pSchema);
+  int32_t index = getTagColumnIndex(pTSSchema, pSchema);
   assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX));
 
   pInfo->sch      = *pSchema;
   pInfo->colIndex = index;
   pInfo->optr     = pExpr->_node.optr;
   pInfo->compare  = getComparFunc(pSchema->type, pInfo->optr);
+  pInfo->param    = pTSSchema;
   
   if (pInfo->optr == TSDB_RELATION_IN) {
     pInfo->q = (char*) pCond->arr;
@@ -1436,7 +1427,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
 }
 
 bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
-  tQueryInfo* pInfo = (tQueryInfo*)param;
+  tQueryInfo* pInfo = (tQueryInfo*) param;
 
   STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
 
@@ -1447,7 +1438,14 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
     val = pTable->name;
     type = TSDB_DATA_TYPE_BINARY;
   } else {
-    val = dataRowTuple(pTable->tagVal);  // todo not only the first column
+    STSchema* pTSchema = (STSchema*) pInfo->param; // todo table schema is identical to stable schema??
+    
+    int32_t offset = pTSchema->columns[pInfo->colIndex].offset;
+    if (pInfo->sch.type == TSDB_DATA_TYPE_BINARY || pInfo->sch.type == TSDB_DATA_TYPE_NCHAR) {
+      val = tdGetRowDataOfCol(pTable->tagVal, pInfo->sch.type, TD_DATA_ROW_HEAD_SIZE + offset);
+    } else {
+      val = dataRowTuple(pTable->tagVal) + offset;
+    }
   }
 
   int32_t ret = 0;
@@ -1497,19 +1495,11 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
 }
 
 static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) {
-  // query according to the binary expression
+  // query according to the expression tree
-  STSchema* pSchema = pSTable->tagSchema;
+  SExprTraverseSupp supp = {
-  SSchema*  schema = calloc(schemaNCols(pSchema), sizeof(SSchema));
+      .fp = (__result_filter_fn_t) tSkipListNodeFilterCallback,
-  for (int32_t i = 0; i < schemaNCols(pSchema); ++i) {
+      .setupInfoFn = filterPrepare,
-    schema[i].colId = schemaColAt(pSchema, i)->colId;
+      .pExtInfo = pSTable->tagSchema,
-    schema[i].type  = schemaColAt(pSchema, i)->type;
-    schema[i].bytes = schemaColAt(pSchema, i)->bytes;
-  }
-
-  SExprTreeSupporter s = {.pTagSchema = schema, .numOfTags = schemaNCols(pSTable->tagSchema)};
-
-  SBinaryFilterSupp supp = {
-      .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, .setupInfoFn = filterPrepare, .pExtInfo = &s,
       };
 
   SArray* pTableList = taosArrayInit(8, POINTER_BYTES);
@@ -1519,7 +1509,6 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
 
   convertQueryResult(pRes, pTableList);
   taosArrayDestroy(pTableList);
-  free(schema);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1527,10 +1516,17 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC
   const char* tbnameCond, STableGroupInfo *pGroupInfo, SColIndex *pColIndex, int32_t numOfCols) {
   STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
   if (pTable == NULL) {
-    uError("failed to get stable, uid:%, %p" PRIu64, uid);
+    uError("%p failed to get stable, uid:%" PRIu64, tsdb, uid);
     return TSDB_CODE_INVALID_TABLE_ID;
   }
   
+  if (pTable->type != TSDB_SUPER_TABLE) {
+    uError("%p query normal tag not allowed, uid:%, tid:%d, name:%s" PRIu64,
+           tsdb, uid, pTable->tableId.tid, pTable->name);
+    
+    return TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
+  }
+  
   SArray* res = taosArrayInit(8, sizeof(STableId));
   STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pTable);