Merge branch 'develop' into hotfix/taos-tools

* develop: (138 commits)
  [td-225]fix error in global variables.
  [td-225]fix error in global variables.
  [td-225] fix invalid read
  [td-688]
  [td-225] update the error code info
  [td-225] update the error code info
  [td-225] update the error code when system related error happens.
  Fix bug occured in regression test
  fix compile error in arm64
  [TD-543] fix coverity scans
  reference count
  Bump jackson-databind in /tests/comparisonTest/opentsdb/opentsdbtest
  [TD-543] coverity scan
  [td-225] add cancel query support, fix bugs in load file blocks when column has already been updated.
  implement format check for log
  change SIGINT to SIGKILL to make dnode5 be killed.
  scripts
  add comments
  free ID in the last step
  dont reset lockedBy
  ...

.travis.yml

@@ -260,19 +260,75 @@ matrix:
       - cmake .. > /dev/null
       - make > /dev/null
 
-        #  - os: osx
+  - os: linux
-        #    language: c
+    arch: arm64
-        #    compiler: clang
+    dist: bionic
-        #    env: DESC="mac/clang build"
+    language: c
-        #    git:
+    compiler: clang
-        #      - depth: 1
+    env: DESC="linux/clang build"
-        #    addons:
+    git:
-        #      homebrew:
+      - depth: 1
-        #        - cmake
+
-        #
+    addons:
-        #    script:
+      apt:
-        #      - cd ${TRAVIS_BUILD_DIR}
+        packages:
-        #      - mkdir debug
+          - build-essential
-        #      - cd debug
+          - cmake
-        #      - cmake .. > /dev/null
+
-        #      - make > /dev/null
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
+      - mkdir debug
+      - cd debug
+
+    script:
+      - if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
+         cmake .. -DCPUTYPE=aarch64 > /dev/null;
+        else
+         cmake .. > /dev/null;
+        fi
+      - make > /dev/null
+
+  - os: linux
+    arch: arm64
+    dist: trusty
+    language: c
+    git:
+      - depth: 1
+
+    addons:
+      apt:
+        packages:
+          - build-essential
+          - cmake
+    env:
+      - DESC="trusty/gcc-4.8 build"
+
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
+      - mkdir debug
+      - cd debug
+
+    script:
+      - if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
+         cmake .. -DCPUTYPE=aarch64 > /dev/null;
+        else
+         cmake .. > /dev/null;
+        fi
+      - make > /dev/null
+
+#  - os: osx
+#    language: c
+#    compiler: clang
+#    env: DESC="mac/clang build"
+#    git:
+#      - depth: 1
+#    addons:
+#      homebrew:
+#        - cmake
+#
+#    script:
+#      - cd ${TRAVIS_BUILD_DIR}
+#      - mkdir debug
+#      - cd debug
+#      - cmake .. > /dev/null
+#      - make > /dev/null

documentation/webdocs/markdowndocs/Super Table-ch.md

@@ -53,7 +53,7 @@ STable从属于库，一个STable只属于一个库，但一个库可以有一
 
     说明：
 
-    1. TAGS列总长度不能超过64k bytes；
+    1. TAGS列总长度不能超过16k bytes；
     2. TAGS列的数据类型不能是timestamp；
     3. TAGS列名不能与其他列名相同;
     4. TAGS列名不能为预留关键字. 

documentation/webdocs/markdowndocs/Super Table.md

@@ -22,7 +22,7 @@ New keyword "tags" is introduced, where tag_name is the tag name, and tag_type i
 
 Note：
 
-1. The bytes of all tags together shall be less than 64k 
+1. The bytes of all tags together shall be less than 16k 
 2. Tag's data type can not be time stamp
 3. Tag name shall be different from the field name
 4. Tag name shall not be the same as system keywords

documentation/webdocs/markdowndocs/TAOS SQL-ch.md

@@ -63,7 +63,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
 | 3    |  BIGINT   | 8      | 长整型，范围 [-2^63+1,   2^63-1], -2^63用于NULL                                 |
 | 4    |   FLOAT   | 4      | 浮点型，有效位数6-7，范围 [-3.4E38, 3.4E38]                  |
 | 5    |  DOUBLE   | 8      | 双精度浮点型，有效位数15-16，范围 [-1.7E308,   1.7E308]      |
-| 6    |  BINARY   | 自定义 | 用于记录字符串，理论上，最长可以有65526字节，但由于每行数据最多64K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
+| 6    |  BINARY   | 自定义 | 用于记录字符串，理论上，最长可以有16374字节，但由于每行数据最多16K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
 | 7    | SMALLINT  | 2      | 短整型， 范围 [-32767, 32767], -32768用于NULL                                |
 | 8    |  TINYINT  | 1      | 单字节整型，范围 [-127, 127], -128用于NULL                                |
 | 9    |   BOOL    | 1      | 布尔型，{true,   false}                                      |
@@ -106,7 +106,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
     ```mysql
     CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...])
     ```
-    说明：1）表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；2）表的每行长度不能超过64K字节；3）使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节。
+    说明：1）表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；2）表的每行长度不能超过16K字节；3）使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节。
 
 
 - **删除数据表**

documentation/webdocs/markdowndocs/TAOS SQL.md

@@ -39,7 +39,7 @@ The full list of data types is listed below.  For string types of data, we will
 | 6    |   DOUBLE    |    8    | A standard nullable double float type with 15-16 significant digits and a range of [-1.7E308, 1.7E308]​ |
 | 7    |    BOOL     |    1    | A nullable boolean type, [**`true`**, **`false`**]           |
 | 8    |  TIMESTAMP  |    8    | A nullable timestamp type with the same usage as the primary column timestamp |
-| 9    | BINARY(*M*) |   *M*   | A nullable string type whose length is *M*, error should be threw with exceeded chars, the maximum length of *M* is 65526, but as maximum row size is 64K bytes, the actual upper limit will generally less than 65526. This type of string only supports ASCii encoded chars. |
+| 9    | BINARY(*M*) |   *M*   | A nullable string type whose length is *M*, error should be threw with exceeded chars, the maximum length of *M* is 16374, but as maximum row size is 16K bytes, the actual upper limit will generally less than 16374. This type of string only supports ASCii encoded chars. |
 | 10   | NCHAR(*M*)  | 4 * *M* | A nullable string type whose length is *M*, error should be threw with exceeded chars. The **`NCHAR`** type supports Unicode encoded chars. |
 
 All the keywords in a SQL statement are case-insensitive, but strings values are case-sensitive and must be quoted by a pair of `'` or `"`. To quote a `'` or a `"` , you can use the escape character `\`.
@@ -86,7 +86,7 @@ All the keywords in a SQL statement are case-insensitive, but strings values are
     
     1) The first column must be a `timestamp`, and the system will set it as the primary key.
     
-    2) The record size is limited to 64k bytes
+    2) The record size is limited to 16k bytes
     
     3) For `binary` or `nchar` data types, the length must be specified. For example, binary(20) means a binary data type with 20 bytes.
 

src/client/inc/tscLocalMerge.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSCSECONARYMERGE_H
+#ifndef TDENGINE_TSCLOCALMERGE_H
-#define TDENGINE_TSCSECONARYMERGE_H
+#define TDENGINE_TSCLOCALMERGE_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -27,14 +27,7 @@ extern "C" {
 #include "tsclient.h"
 
 #define MAX_NUM_OF_SUBQUERY_RETRY 3
-
+  
-/*
- * @version 0.1
- * @date   2018/01/05
- * @author liaohj
- * management of client-side reducer for metric query
- */
-
 struct SQLFunctionCtx;
 
 typedef struct SLocalDataSource {
@@ -60,7 +53,6 @@ typedef struct SLocalReducer {
   char *                 prevRowOfInput;
   tFilePage *            pResultBuf;
   int32_t                nResultBufSize;
-//  char *                 pBufForInterpo;  // intermediate buffer for interpolation
   tFilePage *            pTempBuffer;
   struct SQLFunctionCtx *pCtx;
   int32_t                rowSize;      // size of each intermediate result.
@@ -81,13 +73,8 @@ typedef struct SLocalReducer {
 } SLocalReducer;
 
 typedef struct SSubqueryState {
-  /*
+  int32_t  numOfRemain;         // the number of remain unfinished subquery
-   * the number of completed retrieval subquery, once this value equals to numOfVnodes,
+  int32_t  numOfTotal;          // the number of total sub-queries
-   * all retrieval are completed.Local merge is launched.
-   */
-  int32_t  numOfCompleted;
-  int32_t  numOfTotal;          // number of total sub-queries
-  int32_t  code;                // code from subqueries
   uint64_t numOfRetrievedRows;  // total number of points in this query
 } SSubqueryState;
 
@@ -128,4 +115,4 @@ int32_t tscDoLocalMerge(SSqlObj *pSql);
 }
 #endif
 
-#endif  // TDENGINE_TSCSECONARYMERGE_H
+#endif  // TDENGINE_TSCLOCALMERGE_H

src/client/inc/tscSubquery.h

@@ -26,11 +26,9 @@ extern "C" {
 void tscFetchDatablockFromSubquery(SSqlObj* pSql);
 
 void tscSetupOutputColumnIndex(SSqlObj* pSql);
-int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql);
 void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
 
 SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index);
-void tscDestroyJoinSupporter(SJoinSupporter* pSupporter);
 
 int32_t tscHandleMasterJoinQuery(SSqlObj* pSql);
 

src/client/inc/tscUtil.h

@@ -28,7 +28,7 @@ extern "C" {
 #include "exception.h"
 #include "qextbuffer.h"
 #include "taosdef.h"
-#include "tscSecondaryMerge.h"
+#include "tscLocalMerge.h"
 #include "tsclient.h"
 
 #define UTIL_TABLE_IS_SUPER_TABLE(metaInfo)  \
@@ -64,7 +64,8 @@ typedef struct SJoinSupporter {
   SSubqueryState* pState;
   SSqlObj*        pObj;           // parent SqlObj
   int32_t         subqueryIndex;  // index of sub query
-  int64_t         interval;       // interval time
+  int64_t         intervalTime;   // interval time
+  int64_t         slidingTime;    // sliding time
   SLimitVal       limit;          // limit info
   uint64_t        uid;            // query meter uid
   SArray*         colList;        // previous query information, no need to use this attribute, and the corresponding attribution
@@ -122,15 +123,13 @@ bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableI
 bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
 bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
 
-bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo);
+bool tscIsProjectionQuery(SQueryInfo* pQueryInfo);
 
 bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
-bool tscQueryOnSTable(SSqlCmd* pCmd);
 bool tscQueryTags(SQueryInfo* pQueryInfo);
-bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd);
 
 void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex,
-                                  SSchema* pColSchema, int16_t isTag);
+                                  SSchema* pColSchema, int16_t colType);
 
 int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql);
 void    tscClearInterpInfo(SQueryInfo* pQueryInfo);
@@ -139,7 +138,7 @@ bool tscIsInsertData(char* sqlstr);
 
 /* use for keep current db info temporarily, for handle table with db prefix */
 // todo remove it
-void tscGetDBInfoFromMeterId(char* tableId, char* db);
+void tscGetDBInfoFromTableFullName(char* tableId, char* db);
 
 int tscAllocPayload(SSqlCmd* pCmd, int size);
 
@@ -253,7 +252,7 @@ void     addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t sub
 
 void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);
 
-int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid);
+int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
 
 void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
 

src/client/inc/tschemautil.h

@@ -64,12 +64,20 @@ SSchema* tscGetTableSchema(const STableMeta* pTableMeta);
 SSchema *tscGetTableTagSchema(const STableMeta *pMeta);
 
 /**
- *
+ * get the column schema according to the column index
  * @param pMeta
- * @param startCol
+ * @param colIndex
  * @return
  */
-SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t startCol);
+SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
+
+/**
+ * get the column schema according to the column id
+ * @param pTableMeta
+ * @param colId
+ * @return
+ */
+SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
 
 /**
  * check if the schema is valid or not, including following aspects:

src/client/inc/tsclient.h

@@ -85,7 +85,7 @@ typedef struct SSqlExpr {
   int16_t   functionId;     // function id in aAgg array
   int16_t   resType;        // return value type
   int16_t   resBytes;       // length of return value
-  int32_t   interBytes;  // inter result buffer size
+  int32_t   interBytes;     // inter result buffer size
   int16_t   numOfParams;    // argument value of each function
   tVariant  param[3];       // parameters are not more than 3
   int32_t   offset;         // sub result column value of arithmetic expression.
@@ -123,7 +123,7 @@ typedef struct SCond {
 typedef struct SJoinNode {
   char     tableId[TSDB_TABLE_ID_LEN];
   uint64_t uid;
-  int16_t  tagCol;
+  int16_t  tagColId;
 } SJoinNode;
 
 typedef struct SJoinInfo {
@@ -155,20 +155,19 @@ typedef struct SParamInfo {
 } SParamInfo;
 
 typedef struct STableDataBlocks {
-  char    tableId[TSDB_TABLE_ID_LEN];
+  char     tableId[TSDB_TABLE_ID_LEN];
-  int8_t  tsSource;     // where does the UNIX timestamp come from, server or client
+  int8_t   tsSource;     // where does the UNIX timestamp come from, server or client
-  bool    ordered;      // if current rows are ordered or not
+  bool     ordered;      // if current rows are ordered or not
-  int64_t vgId;         // virtual group id
+  int64_t  vgId;         // virtual group id
-  int64_t prevTS;       // previous timestamp, recorded to decide if the records array is ts ascending
+  int64_t  prevTS;       // previous timestamp, recorded to decide if the records array is ts ascending
-  int32_t numOfTables;  // number of tables in current submit block
+  int32_t  numOfTables;  // number of tables in current submit block
-
+  int32_t  rowSize;      // row size for current table
-  int32_t  rowSize;  // row size for current table
   uint32_t nAllocSize;
-  uint32_t headerSize;  // header for metadata (submit metadata)
+  uint32_t headerSize;   // header for table info (uid, tid, submit metadata)
   uint32_t size;
 
   /*
-   * the metermeta for current table, the metermeta will be used during submit stage, keep a ref
+   * the table meta of table, the table meta will be used during submit, keep a ref
    * to avoid it to be removed from cache
    */
   STableMeta *pTableMeta;
@@ -191,32 +190,28 @@ typedef struct SDataBlockList {  // todo remove
 } SDataBlockList;
 
 typedef struct SQueryInfo {
-  int16_t          command;  // the command may be different for each subclause, so keep it seperately.
+  int16_t          command;       // the command may be different for each subclause, so keep it seperately.
-  uint32_t         type;     // query/insert/import type
+  uint32_t         type;          // query/insert/import type
   char             slidingTimeUnit;
-                   
   STimeWindow      window;
   int64_t          intervalTime;  // aggregation time interval
   int64_t          slidingTime;   // sliding window in mseconds
   SSqlGroupbyExpr  groupbyExpr;   // group by tags info
-
+  SArray *         colList;       // SArray<SColumn*>
-  SArray *         colList;      // SArray<SColumn*>
   SFieldInfo       fieldsInfo;
-  SArray *         exprList;    // SArray<SSqlExpr*>
+  SArray *         exprList;      // SArray<SSqlExpr*>
   SLimitVal        limit;
   SLimitVal        slimit;
   STagCond         tagCond;
   SOrderVal        order;
-  int16_t          fillType;    // final result fill type
+  int16_t          fillType;      // final result fill type
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
   struct STSBuf *  tsBuf;
-  int64_t *        fillVal;      // default value for fill
+  int64_t *        fillVal;       // default value for fill
-  char *           msg;          // pointer to the pCmd->payload to keep error message temporarily
+  char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
-  int64_t          clauseLimit;  // limit for current sub clause
+  int64_t          clauseLimit;   // limit for current sub clause
-
+  int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
-  // offset value in the original sql expression, NOT sent to virtual node, only applied at client side
-  int64_t          prjOffset;
 } SQueryInfo;
 
 typedef struct {
@@ -299,11 +294,12 @@ typedef struct STscObj {
 } STscObj;
 
 typedef struct SSqlObj {
-  void *   signature;
+  void            *signature;
-  STscObj *pTscObj;
+  STscObj         *pTscObj;
-  void (*fp)();
+  void            *SRpcReqContext;
-  void (*fetchFp)();
+  void            (*fp)();
-  void *           param;
+  void            (*fetchFp)();
+  void            *param;
   int64_t          stime;
   uint32_t         queryId;
   void *           pStream;
@@ -431,7 +427,7 @@ extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
 
 int32_t tscCompareTidTags(const void* p1, const void* p2);
-void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables);
+void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
 
 #ifdef __cplusplus
 }

src/client/src/TSDBJNIConnector.c

@@ -151,8 +151,8 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp
 JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) {
   if (jconfigDir != NULL) {
     const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL);
-    if (confDir && strlen(configDir) != 0) {
+    if (confDir && strlen(confDir) != 0) {
-      strcpy(configDir, confDir);
+      tstrncpy(configDir, confDir, TSDB_FILENAME_LEN);
     }
     (*env)->ReleaseStringUTFChars(env, jconfigDir, confDir);
   }
@@ -227,12 +227,12 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn
   }
 
   if (user == NULL) {
-    jniTrace("jobj:%p, user is null, use tsDefaultUser", jobj);
+    jniTrace("jobj:%p, user is null, use default user %s", jobj, TSDB_DEFAULT_USER);
-    user = tsDefaultUser;
+    user = TSDB_DEFAULT_USER;
   }
   if (pass == NULL) {
-    jniTrace("jobj:%p, pass is null, use tsDefaultPass", jobj);
+    jniTrace("jobj:%p, pass is null, use default password", jobj);
-    pass = tsDefaultPass;
+    pass = TSDB_DEFAULT_PASS;
   }
 
   /*
@@ -252,8 +252,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn
 
   if (host != NULL) (*env)->ReleaseStringUTFChars(env, jhost, host);
   if (dbname != NULL) (*env)->ReleaseStringUTFChars(env, jdbName, dbname);
-  if (user != NULL && user != tsDefaultUser) (*env)->ReleaseStringUTFChars(env, juser, user);
+  if (user != NULL && user != (const char *)TSDB_DEFAULT_USER) (*env)->ReleaseStringUTFChars(env, juser, user);
-  if (pass != NULL && pass != tsDefaultPass) (*env)->ReleaseStringUTFChars(env, jpass, pass);
+  if (pass != NULL && pass != (const char *)TSDB_DEFAULT_PASS) (*env)->ReleaseStringUTFChars(env, jpass, pass);
 
   return ret;
 }
@@ -465,7 +465,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
   int         num_fields = taos_num_fields(result);
 
   if (num_fields == 0) {
-    jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, (void *)res, num_fields);
+    jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, (void*)res, num_fields);
     return JNI_NUM_OF_FIELDS_0;
   }
 
@@ -473,7 +473,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
   if (row == NULL) {
     int tserrno = taos_errno(result);
     if (tserrno == 0) {
-      jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, (void *)res, num_fields);
+      jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, (void*)res, num_fields);
       return JNI_FETCH_END;
     } else {
       jniTrace("jobj:%p, conn:%p, interruptted query", jobj, tscon);
@@ -571,9 +571,9 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
   sub = (jlong)tsub;
 
   if (sub == 0) {
-    jniTrace("jobj:%p, failed to subscribe: topic:%s", jobj, jtopic);
+    jniTrace("jobj:%p, failed to subscribe: topic:%s", jobj, topic);
   } else {
-    jniTrace("jobj:%p, successfully subscribe: topic: %s", jobj, jtopic);
+    jniTrace("jobj:%p, successfully subscribe: topic: %s", jobj, topic);
   }
 
   if (topic != NULL) (*env)->ReleaseStringUTFChars(env, jtopic, topic);
@@ -583,7 +583,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
 }
 
 JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) {
-  jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%ld", jobj, sub);
+  jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%lld", jobj, sub);
   jniGetGlobalMethod(env);
 
   TAOS_SUB *tsub = (TAOS_SUB *)sub;

src/client/src/tscAsync.c

@@ -19,9 +19,8 @@
 #include "tnote.h"
 #include "trpc.h"
 #include "tscLog.h"
-#include "tscProfile.h"
 #include "tscSubquery.h"
-#include "tscSecondaryMerge.h"
+#include "tscLocalMerge.h"
 #include "tscUtil.h"
 #include "tsched.h"
 #include "tschemautil.h"
@@ -46,7 +45,8 @@ int doAsyncParseSql(SSqlObj* pSql) {
   int32_t code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
   if (code != TSDB_CODE_SUCCESS) {
     tscError("failed to malloc payload");
-    tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_TSC_OUT_OF_MEMORY);
+    tscQueueAsyncRes(pSql);
+//    tscQueueAsyncRes(pSql->fp, pSql->param, TSDB_CODE_TSC_OUT_OF_MEMORY);
     return code;
   }
   
@@ -211,7 +211,8 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
 
   if (pRes->qhandle == 0) {
     tscError("qhandle is NULL");
-    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_INVALID_QHANDLE);
+    pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;
+    tscQueueAsyncRes(pSql);
     return;
   }
 
@@ -285,7 +286,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW),
     
     tscProcessSql(pSql);
   } else {
-    SSchedMsg schedMsg;
+    SSchedMsg schedMsg = { 0 };
     schedMsg.fp = tscProcessFetchRow;
     schedMsg.ahandle = pSql;
     schedMsg.thandle = pRes->tsrow;
@@ -368,7 +369,9 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) {
     pSql->fp = pSql->fetchFp;
   }
 
-  (*pSql->fp)(pSql->param, taosres, code);
+  if (pSql->fp) {
+    (*pSql->fp)(pSql->param, taosres, code);
+  }
 
   if (shouldFree) {
     tscTrace("%p sqlObj is automatically freed in async res", pSql);
@@ -385,7 +388,7 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) {
   int32_t* c = malloc(sizeof(int32_t));
   *c = code;
   
-  SSchedMsg schedMsg;
+  SSchedMsg schedMsg = { 0 };
   schedMsg.fp = tscProcessAsyncError;
   schedMsg.ahandle = fp;
   schedMsg.thandle = param;
@@ -401,7 +404,7 @@ void tscQueueAsyncRes(SSqlObj *pSql) {
     tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
   }
 
-  SSchedMsg schedMsg;
+  SSchedMsg schedMsg = { 0 };
   schedMsg.fp = tscProcessAsyncRes;
   schedMsg.ahandle = pSql;
   schedMsg.thandle = (void *)1;
@@ -418,7 +421,7 @@ void tscProcessAsyncFree(SSchedMsg *pMsg) {
 void tscQueueAsyncFreeResult(SSqlObj *pSql) {
   tscTrace("%p sqlObj put in queue to async free", pSql);
 
-  SSchedMsg schedMsg;
+  SSchedMsg schedMsg = { 0 };
   schedMsg.fp = tscProcessAsyncFree;
   schedMsg.ahandle = pSql;
   schedMsg.thandle = (void *)1;

src/client/src/tscFunctionImpl.c

@@ -153,7 +153,7 @@ typedef struct SRateInfo {
 
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
                           int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) {
-  if (!isValidDataType(dataType, dataBytes)) {
+  if (!isValidDataType(dataType)) {
     tscError("Illegal data type %d or data type length %d", dataType, dataBytes);
     return TSDB_CODE_TSC_INVALID_SQL;
   }
@@ -1853,26 +1853,14 @@ static void last_row_function(SQLFunctionCtx *pCtx) {
 static void last_row_finalizer(SQLFunctionCtx *pCtx) {
   // do nothing at the first stage
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-  if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
+  if (pResInfo->hasResult != DATA_SET_FLAG) {
-    if (pResInfo->hasResult != DATA_SET_FLAG) {
+    if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
-      if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
-        setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+    } else {
-      } else {
+      setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
-        setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
-      }
-      
-      return;
-    }
-  } else {
-    if (pResInfo->hasResult != DATA_SET_FLAG) {
-      if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
-        setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
-      } else {
-        setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
-      }
-      
-      return;
     }
+    
+    return;
   }
   
   GET_RES_INFO(pCtx)->numOfRes = 1;
@@ -2989,12 +2977,12 @@ static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) {
  */
 static void tag_function(SQLFunctionCtx *pCtx) {
   SET_VAL(pCtx, 1, 1);
-  tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true);
+  tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true);
 }
 
 static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   SET_VAL(pCtx, 1, 1);
-  tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true);
+  tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true);
 }
 
 static void copy_function(SQLFunctionCtx *pCtx) {
@@ -3903,7 +3891,7 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
   STSCompInfo *pInfo = pResInfo->interResultBuf;
   
-  pInfo->pTSBuf = tsBufCreate(false);
+  pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
   pInfo->pTSBuf->tsOrder = pCtx->order;
   return true;
 }
@@ -3925,7 +3913,6 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) {
   }
   
   SET_VAL(pCtx, pCtx->size, 1);
-  
   pResInfo->hasResult = DATA_SET_FLAG;
 }
 

src/client/src/tscLocal.c

@@ -22,7 +22,6 @@
 #include "taosdef.h"
 #include "tscLog.h"
 #include "qextbuffer.h"
-#include "tscSecondaryMerge.h"
 #include "tschemautil.h"
 #include "tname.h"
 
@@ -132,14 +131,14 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
   for (int32_t i = 0; i < numOfRows; ++i) {
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
     char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
+    STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, pField->bytes);
 
     char *type = tDataTypeDesc[pSchema[i].type].aName;
 
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
     dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i;
     
-    STR_TO_VARSTR(dst, type);
+    STR_WITH_MAXSIZE_TO_VARSTR(dst, type, pField->bytes);
     
     int32_t bytes = pSchema[i].bytes;
     if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
@@ -157,7 +156,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) {
       char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
       const char *src = "TAG";
-      STR_WITH_SIZE_TO_VARSTR(output, src, strlen(src));
+      STR_WITH_MAXSIZE_TO_VARSTR(output, src, pField->bytes);
     }
   }
 
@@ -171,7 +170,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     // field name
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
     char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
+    STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, pField->bytes);
 
     // type name
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
@@ -183,8 +182,12 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     // type length
     int32_t bytes = pSchema[i].bytes;
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2);
-    if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+    if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
-      bytes = bytes / TSDB_NCHAR_SIZE;
+      bytes -= VARSTR_HEADER_SIZE;
+      
+      if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+        bytes = bytes / TSDB_NCHAR_SIZE;
+      }
     }
 
     *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes;
@@ -193,7 +196,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3);
     char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
     const char *src = "TAG";
-    STR_WITH_SIZE_TO_VARSTR(target, src, strlen(src));
+    STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes);
 
     pTagValue += pSchema[i].bytes;
   }
@@ -220,15 +223,15 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
   
   rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
 
-  f.bytes = typeColLength;
+  f.bytes = typeColLength + VARSTR_HEADER_SIZE;
   f.type = TSDB_DATA_TYPE_BINARY;
   tstrncpy(f.name, "Type", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
-  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength,
+  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength + VARSTR_HEADER_SIZE,
       typeColLength, false);
   
-  rowLen += typeColLength;
+  rowLen += typeColLength + VARSTR_HEADER_SIZE;
 
   f.bytes = sizeof(int32_t);
   f.type = TSDB_DATA_TYPE_INT;
@@ -240,15 +243,15 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
   
   rowLen += sizeof(int32_t);
 
-  f.bytes = noteColLength;
+  f.bytes = noteColLength + VARSTR_HEADER_SIZE;
   f.type = TSDB_DATA_TYPE_BINARY;
   tstrncpy(f.name, "Note", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
-  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength,
+  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength + VARSTR_HEADER_SIZE,
       noteColLength, false);
   
-  rowLen += noteColLength;
+  rowLen += noteColLength + VARSTR_HEADER_SIZE;
   return rowLen;
 }
 

src/client/src/tscLocalMerge.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tscSecondaryMerge.h"
 #include "os.h"
 #include "tlosertree.h"
 #include "tscUtil.h"
@@ -21,6 +20,7 @@
 #include "tsclient.h"
 #include "tutil.h"
 #include "tscLog.h"
+#include "tscLocalMerge.h"
 
 typedef struct SCompareParam {
   SLocalDataSource **pLocalData;
@@ -230,6 +230,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
       if (ds == NULL) {
         tscError("%p failed to create merge structure", pSql);
         pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+        tfree(pReducer);
         return;
       }
       
@@ -266,6 +267,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   
   // no data actually, no need to merge result.
   if (idx == 0) {
+    tfree(pReducer);
     return;
   }
 
@@ -282,6 +284,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
   pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
   if (pReducer->pLoserTree == NULL || pRes->code != 0) {
+    tfree(pReducer);
     return;
   }
 
@@ -325,7 +328,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
     tfree(pReducer->pResultBuf);
     tfree(pReducer->pFinalRes);
     tfree(pReducer->prevRowOfInput);
-
+    tfree(pReducer);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return;
   }
@@ -353,7 +356,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pRes->numOfGroups = 0;
 
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);;
+  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
   
   TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey);
   int64_t revisedSTime =
@@ -410,13 +413,13 @@ static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor
 }
 
 int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, int32_t orderType) {
-  int32_t ret = tscFlushTmpBufferImpl(pMemoryBuf, pDesc, pPage, orderType);
+  int32_t ret = 0;
-  if (ret != 0) {
+  if ((ret = tscFlushTmpBufferImpl(pMemoryBuf, pDesc, pPage, orderType)) != 0) {
-    return -1;
+    return ret;
   }
 
-  if (!tExtMemBufferFlush(pMemoryBuf)) {
+  if ((ret = tExtMemBufferFlush(pMemoryBuf)) != 0) {
-    return -1;
+    return ret;
   }
 
   return 0;
@@ -437,9 +440,9 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
 
   // current buffer is full, need to flushed to disk
   assert(pPage->num == pModel->capacity);
-  int32_t ret = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType);
+  int32_t code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType);
-  if (ret != 0) {
+  if (code != 0) {
-    return -1;
+    return code;
   }
 
   int32_t remain = numOfRows - numOfRemainEntries;
@@ -455,8 +458,8 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
     tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows);
 
     if (pPage->num == pModel->capacity) {
-      if (tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType) != TSDB_CODE_SUCCESS) {
+      if ((code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType)) != TSDB_CODE_SUCCESS) {
-        return -1;
+        return code;
       }
     } else {
       pPage->num = numOfWriteElems;
@@ -685,6 +688,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
   if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    tfree(pSchema);
     return pRes->code;
   }
 
@@ -1092,14 +1096,6 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
   
   size_t size = tscSqlExprNumOfExprs(pQueryInfo);
   for (int32_t j = 0; j < size; ++j) {
-    //    SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[j];
-    //    if (pExpr == NULL) {
-    //      assert(pQueryInfo->fieldsInfo.pExpr[j] != NULL);
-    //
-    //      maxOutput = 1;
-    //      continue;
-    //    }
-
     /*
      * ts, tag, tagprj function can not decide the output number of current query
      * the number of output result is decided by main output
@@ -1109,8 +1105,9 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
       continue;
     }
 
-    if (maxOutput < GET_RES_INFO(&pCtx[j])->numOfRes) {
+    SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
-      maxOutput = GET_RES_INFO(&pCtx[j])->numOfRes;
+    if (maxOutput < pResInfo->numOfRes) {
+      maxOutput = pResInfo->numOfRes;
     }
   }
 
@@ -1260,7 +1257,6 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
 
 #ifdef _DEBUG_VIEW
   printf("final result before interpo:\n");
-  assert(0);
 //  tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num);
 #endif
   

src/client/src/tscParseInsert.c

@@ -252,7 +252,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
         numType = tscToInteger(pToken, &iv, &endptr);
         if (TK_ILLEGAL == numType) {
           return tscInvalidSQLErrMsg(msg, "invalid bigint data", pToken->z);
-        } else if (errno == ERANGE || iv > INT64_MAX || iv <= INT64_MIN) {
+        } else if (errno == ERANGE || iv == INT64_MIN) {
           return tscInvalidSQLErrMsg(msg, "bigint data overflow", pToken->z);
         }
 
@@ -594,7 +594,6 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
   size_t    remain = pDataBlock->nAllocSize - pDataBlock->size;
   const int factor = 5;
   uint32_t nAllocSizeOld = pDataBlock->nAllocSize;
-  assert(pDataBlock->headerSize >= 0);
   
   // expand the allocated size
   if (remain < rowSize * factor) {
@@ -1328,12 +1327,14 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
   int32_t ret = TSDB_CODE_SUCCESS;
   
   if (initialParse) {
+    assert(!pSql->cmd.parseFinished);
+
     char* p = pSql->sqlstr;
     pSql->sqlstr = NULL;
     
     tscPartiallyFreeSqlObj(pSql);
     pSql->sqlstr = p;
-  } else {
+  } else if (!pSql->cmd.parseFinished) {
     tscTrace("continue parse sql: %s", pSql->cmd.curSql);
   }
   

src/client/src/tscSQLParser.c

@@ -98,8 +98,6 @@ static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killTy
 static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField);
 static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo);
 
-static void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex);
-
 static int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql);
 static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql);
 static int32_t getColumnIndexByName(const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex);
@@ -640,17 +638,11 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
 
+  SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX};
+  tstrncpy(s.name, aAggs[TSDB_FUNC_TS].aName, sizeof(s.name));
+
   SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
-  SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE,
+  tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL);
-      TSDB_KEYSIZE, false);
-
-  SColumnList ids = getColumnList(1, 0, PRIMARYKEY_TIMESTAMP_COL_INDEX);
-
-  int32_t ret =
-      insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].aName, pExpr);
-  if (ret != TSDB_CODE_SUCCESS) {
-    return ret;
-  }
 
   if (parseSlidingClause(pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_TSC_INVALID_SQL;
@@ -760,6 +752,10 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
 
   int32_t nLen = 0;
   for (int32_t i = 0; i < pFieldList->nField; ++i) {
+    if (pFieldList->p[i].bytes == 0) {
+      invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
+      return false;
+    }
     nLen += pFieldList->p[i].bytes;
   }
 
@@ -816,6 +812,10 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
 
   int32_t nLen = 0;
   for (int32_t i = 0; i < pTagsList->nField; ++i) {
+    if (pTagsList->p[i].bytes == 0) {
+      invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
+      return false;
+    }
     nLen += pTagsList->p[i].bytes;
   }
 
@@ -1241,11 +1241,11 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
     return invalidSqlErrMsg(pQueryInfo->msg, msg2);
   }
 
+  /*
+   * transfer sql functions that need secondary merge into another format
+   * in dealing with metric queries such as: count/first/last
+   */
   if (isSTable) {
-    /*
-     * transfer sql functions that need secondary merge into another format
-     * in dealing with metric queries such as: count/first/last
-     */
     tscTansformSQLFuncForSTableQuery(pQueryInfo);
 
     if (hasUnsupportFunctionsForSTableQuery(pQueryInfo)) {
@@ -1272,7 +1272,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi
   }
   
   TAOS_FIELD f = tscCreateField(type, fieldName, bytes);
-  SFieldSupInfo* pInfo =tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f);
+  SFieldSupInfo* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f);
   pInfo->pSqlExpr = pSqlExpr;
   
   return TSDB_CODE_SUCCESS;
@@ -1324,8 +1324,9 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
 
 void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
                                   SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) {
-  SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, pIndex, pColSchema->type,
+  SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type,
                                      pColSchema->bytes, pColSchema->bytes, flag);
+  tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName));
 
   SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex);
   if (TSDB_COL_IS_TAG(flag)) {
@@ -1403,7 +1404,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI
 
     if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
       SSchema colSchema = tGetTableNameColumnSchema();
-      tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, true);
+      tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG);
     } else {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
       STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
@@ -1445,7 +1446,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
   }
 
   if (aliasName != NULL) {
-    strcpy(columnName, aliasName);
+    tstrncpy(columnName, aliasName, sizeof(columnName));
   } else {
     getRevisedName(columnName, functionID, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
   }
@@ -1846,7 +1847,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
       } else {
         tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true);
 
-        int64_t nTop = *((int32_t*)val);
+        int64_t nTop = GET_INT32_VAL(val);
         if (nTop <= 0 || nTop > 100) {  // todo use macro
           return invalidSqlErrMsg(pQueryInfo->msg, msg5);
         }
@@ -1856,12 +1857,14 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
           return TSDB_CODE_TSC_INVALID_SQL;
         }
 
+        // todo REFACTOR
         // set the first column ts for top/bottom query
         SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
         pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE,
             TSDB_KEYSIZE, false);
+        tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].aName, sizeof(pExpr->aliasName));
 
-        const int32_t TS_COLUMN_INDEX = 0;
+        const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX;
         SColumnList   ids = getColumnList(1, 0, TS_COLUMN_INDEX);
         insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP,
                           aAggs[TSDB_FUNC_TS].aName, pExpr);
@@ -2221,7 +2224,6 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     // db prefix in tagCond, show table conds in payload
     SSQLToken* pDbPrefixToken = &pShowInfo->prefix;
     if (pDbPrefixToken->type != 0) {
-      assert(pDbPrefixToken->n >= 0);
 
       if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) {  // db name is too long
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
@@ -2470,62 +2472,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) {
   return true;
 }
 
-void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) {
-//  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
-//
-//  // update tags column index for expression
-//  size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-//  for (int32_t i = 0; i < size; ++i) {
-//    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-//
-//    if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) {  // not tags, continue
-//      continue;
-//    }
-//
-//    // not belongs to this table
-//    if (pExpr->uid != pTableMetaInfo->pTableMeta->uid) {
-//      continue;
-//    }
-
-//    for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) {
-//      if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) {
-//        pExpr->colInfo.colIndex = j;
-//        break;
-//      }
-//    }
-//  }
-
-  // update join condition tag column index
-//  SJoinInfo* pJoinInfo = &pQueryInfo->tagCond.joinInfo;
-//  if (!pJoinInfo->hasJoin) {  // not join query
-//    return;
-//  }
-//
-//  assert(pJoinInfo->left.uid != pJoinInfo->right.uid);
-//
-//  // the join condition expression node belongs to this table(super table)
-//  assert(0);
-//  if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->left.uid) {
-//    for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) {
-//      if (pJoinInfo->left.tagCol == pTableMetaInfo->tagColumnIndex[i]) {
-//        pJoinInfo->left.tagCol = i;
-//      }
-//    }
-//  }
-//
-//  if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->right.uid) {
-//    for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) {
-//      if (pJoinInfo->right.tagCol == pTableMetaInfo->tagColumnIndex[i]) {
-//        pJoinInfo->right.tagCol = i;
-//      }
-//    }
-//  }
-}
-
 int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) {
   const char* msg1 = "too many columns in group by clause";
   const char* msg2 = "invalid column name in group by clause";
-  const char* msg3 = "group by columns must belong to one table";
+//  const char* msg3 = "group by columns must belong to one table";
   const char* msg7 = "not support group by expression";
   const char* msg8 = "not allowed column type for group by";
   const char* msg9 = "tags not allowed for table query";
@@ -2561,10 +2511,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
       return invalidSqlErrMsg(pQueryInfo->msg, msg2);
     }
 
-    if (tableIndex != index.tableIndex && tableIndex >= 0) {
-      return invalidSqlErrMsg(pQueryInfo->msg, msg3);
-    }
-
     tableIndex = index.tableIndex;
 
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
@@ -2621,7 +2567,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
   }
 
   pQueryInfo->groupbyExpr.tableIndex = tableIndex;
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2677,7 +2622,7 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn
     tVariantDump(&pRight->val, (char*)&pColumnFilter->upperBndd, colType, false);
   } else {  // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColumn->lowerBndd
     if (colType == TSDB_DATA_TYPE_BINARY) {
-      pColumnFilter->pz = (int64_t)calloc(1, pRight->val.nLen + 1);
+      pColumnFilter->pz = (int64_t)calloc(1, pRight->val.nLen + TSDB_NCHAR_SIZE);
       pColumnFilter->len = pRight->val.nLen;
 
       tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType, false);
@@ -3051,14 +2996,17 @@ static int32_t getColumnQueryCondInfo(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, i
 }
 
 static int32_t getJoinCondInfo(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) {
-  const char* msg = "invalid join query condition";
+  const char* msg1 = "invalid join query condition";
+  const char* msg2 = "join on binary/nchar not supported";
+  const char* msg3 = "type of join columns must be identical";
+  const char* msg4 = "invalid column name in join condition";
 
   if (pExpr == NULL) {
     return TSDB_CODE_SUCCESS;
   }
 
   if (!isExprDirectParentOfLeaftNode(pExpr)) {
-    return invalidSqlErrMsg(pQueryInfo->msg, msg);
+    return invalidSqlErrMsg(pQueryInfo->msg, msg1);
   }
 
   STagCond*  pTagCond = &pQueryInfo->tagCond;
@@ -3067,28 +3015,36 @@ static int32_t getJoinCondInfo(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) {
 
   SColumnIndex index = COLUMN_INDEX_INITIALIZER;
   if (getColumnIndexByName(&pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_TSC_INVALID_SQL;
+    return invalidSqlErrMsg(pQueryInfo->msg, msg4);
   }
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
-  int16_t         tagColIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
+  SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
 
   pLeft->uid = pTableMetaInfo->pTableMeta->uid;
-  pLeft->tagCol = tagColIndex;
+  pLeft->tagColId = pTagSchema1->colId;
   strcpy(pLeft->tableId, pTableMetaInfo->name);
 
   index = (SColumnIndex)COLUMN_INDEX_INITIALIZER;
   if (getColumnIndexByName(&pExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_TSC_INVALID_SQL;
+    return invalidSqlErrMsg(pQueryInfo->msg, msg4);
   }
 
   pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
-  tagColIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
+  SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
 
   pRight->uid = pTableMetaInfo->pTableMeta->uid;
-  pRight->tagCol = tagColIndex;
+  pRight->tagColId = pTagSchema2->colId;
   strcpy(pRight->tableId, pTableMetaInfo->name);
 
+  if (pTagSchema1->type != pTagSchema2->type) {
+    return invalidSqlErrMsg(pQueryInfo->msg, msg3);
+  }
+
+  if (pTagSchema1->type == TSDB_DATA_TYPE_BINARY || pTagSchema1->type == TSDB_DATA_TYPE_NCHAR) {
+    return invalidSqlErrMsg(pQueryInfo->msg, msg2);
+  }
+
   pTagCond->joinInfo.hasJoin = true;
   return TSDB_CODE_SUCCESS;
 }
@@ -3816,6 +3772,10 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr,
   
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i);
+    if (p1 == NULL) {  // no query condition on this table
+      continue;
+    }
+
     tExprNode* p = NULL;
   
     SArray* colList = taosArrayInit(10, sizeof(SColIndex));
@@ -4508,7 +4468,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) {
       len = tDataTypeDesc[pTagsSchema->type].nSize;
     } else {
-      len = varDataLen(pUpdateMsg->data);
+      len = varDataTLen(pUpdateMsg->data);
     }
     
     pUpdateMsg->tagValLen = htonl(len);  // length may be changed after dump data
@@ -4765,7 +4725,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL*
   pQueryInfo->clauseLimit = pQueryInfo->limit.limit;
   pQueryInfo->slimit = pQuerySql->slimit;
   
-  tscTrace("%p limit:%d, offset:%" PRId64 " slimit:%d, soffset:%" PRId64, pSql, pQueryInfo->limit.limit,
+  tscTrace("%p limit:%" PRId64 ", offset:%" PRId64 " slimit:%" PRId64 ", soffset:%" PRId64, pSql, pQueryInfo->limit.limit,
       pQueryInfo->limit.offset, pQueryInfo->slimit.limit, pQueryInfo->slimit.offset);
   
   if (pQueryInfo->slimit.offset < 0 || pQueryInfo->limit.offset < 0) {
@@ -4949,25 +4909,25 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
   return TSDB_CODE_SUCCESS;
 }
 
-void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) {
+//void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) {
-  // the first column not timestamp column, add it
+//  // the first column not timestamp column, add it
-  SSqlExpr* pExpr = NULL;
+//  SSqlExpr* pExpr = NULL;
-  if (tscSqlExprNumOfExprs(pQueryInfo) > 0) {
+//  if (tscSqlExprNumOfExprs(pQueryInfo) > 0) {
-    pExpr = tscSqlExprGet(pQueryInfo, 0);
+//    pExpr = tscSqlExprGet(pQueryInfo, 0);
-  }
+//  }
-
+//
-  if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) {
+//  if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) {
-    SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
+//    SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
-
+//
-    pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false);
+//    pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false);
-    pExpr->colInfo.flag = TSDB_COL_NORMAL;
+//    pExpr->colInfo.flag = TSDB_COL_NORMAL;
-
+//
-    // NOTE: tag column does not add to source column list
+//    // NOTE: tag column does not add to source column list
-    SColumnList ids = getColumnList(1, tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX);
+//    SColumnList ids = getColumnList(1, tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX);
-
+//
-    insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, "ts", pExpr);
+//    insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, "ts", pExpr);
-  }
+//  }
-}
+//}
 
 void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) {
   SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentObj->cmd, subClauseIndex);
@@ -4980,7 +4940,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau
 
     if (pExpr->functionId != TSDB_FUNC_TAG) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
-      int16_t         columnInfo = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+      int16_t         columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
       SColumnIndex    index = {.tableIndex = 0, .columnIndex = columnInfo};
       SSchema*        pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
 
@@ -5016,27 +4976,17 @@ static void doLimitOutputNormalColOfGroupby(SSqlExpr* pExpr) {
 
 void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) {
   SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, tagIndex);
-  int32_t index = pColIndex->colIndex;
+  size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-  
+
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  SSchema*     pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index);
+  SSchema*     pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex);
-  SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index};
+  SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex};
-  
-  size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-  SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes,
-      pSchema->bytes, false);
 
-  pExpr->colInfo.flag = TSDB_COL_NORMAL;
+  tscAddSpecialColumnForSelect(pQueryInfo, size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL);
-  doLimitOutputNormalColOfGroupby(pExpr);
 
-  // NOTE: tag column does not add to source column list
+  SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size);
-  SColumnList list = {0};
+  doLimitOutputNormalColOfGroupby(pInfo->pSqlExpr);
-  list.num = 1;
-  list.ids[0] = colIndex;
-
-  insertResultField(pQueryInfo, size, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr);
-  SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size - 1);
   pInfo->visible = false;
 }
 
@@ -5248,6 +5198,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
+  SSchema s = tGetTableNameColumnSchema();
   SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
   int16_t  bytes = 0;
   int16_t  type = 0;
@@ -5255,10 +5206,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) {
 
   for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
     SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i);
-  
     int16_t colIndex = pColIndex->colIndex;
     if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
-      SSchema s = tGetTableNameColumnSchema();
       type  = s.type;
       bytes = s.bytes;
       name  = s.name;
@@ -5955,10 +5904,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
 
   setColumnOffsetValueInResultset(pQueryInfo);
 
-  for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
-    updateTagColumnIndex(pQueryInfo, i);
-  }
-
   /*
    * fill options are set at the end position, when all columns are set properly
    * the columns may be increased due to group by operation
@@ -6082,11 +6027,14 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
       }
     }
 
+
     if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
       if (pRight->nodeType == TSQL_NODE_VALUE) {
-        if (  pRight->pVal->nType == TSDB_DATA_TYPE_BOOL 
+        if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) {
-           || pRight->pVal->nType == TSDB_DATA_TYPE_BINARY 
+          return TSDB_CODE_TSC_INVALID_SQL;
-           || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) {
+        }
+        if ((pRight->pVal->nType == TSDB_DATA_TYPE_BINARY || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR)
+            && (*pExpr)->_node.optr != TSDB_RELATION_LIKE) {
           return TSDB_CODE_TSC_INVALID_SQL;
         }
       }

src/client/src/tscSchemaUtil.c

@@ -32,7 +32,6 @@ int32_t tscGetNumOfTags(const STableMeta* pTableMeta) {
   }
   
   if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) {
-    assert(tinfo.numOfTags >= 0);
     return tinfo.numOfTags;
   }
   
@@ -64,14 +63,6 @@ SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) {
 
 STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
-
-#if 0
-  if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
-    assert (pTableMeta->pSTable != NULL);
-    return pTableMeta->pSTable->tableInfo;
-  }
-#endif
-
   return pTableMeta->tableInfo;
 }
 
@@ -119,11 +110,24 @@ bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) {
   return (rowLen <= TSDB_MAX_BYTES_PER_ROW);
 }
 
-SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol) {
+SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
   assert(pTableMeta != NULL);
   
   SSchema* pSchema = (SSchema*) pTableMeta->schema;
-  return &pSchema[startCol];
+  return &pSchema[colIndex];
+}
+
+// TODO for large number of columns, employ the binary search method
+SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
+  STableComInfo tinfo = tscGetTableInfo(pTableMeta);
+
+  for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) {
+    if (pTableMeta->schema[i].colId == colId) {
+      return &pTableMeta->schema[i];
+    }
+  }
+
+  return NULL;
 }
 
 struct SSchema tscGetTbnameColumnSchema() {

src/client/src/tscServer.c

@@ -14,20 +14,18 @@
  */
 
 #include "os.h"
+#include "qsqltype.h"
 #include "tcache.h"
 #include "trpc.h"
+#include "tscLocalMerge.h"
+#include "tscLog.h"
 #include "tscProfile.h"
-#include "tscSecondaryMerge.h"
-#include "tscSubquery.h"
 #include "tscUtil.h"
 #include "tschemautil.h"
 #include "tsclient.h"
-#include "tsocket.h"
 #include "ttime.h"
 #include "ttimer.h"
 #include "tutil.h"
-#include "tscLog.h"
-#include "qsqltype.h"
 
 #define TSC_MGMT_VNODE 999
 
@@ -198,7 +196,9 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .handle  = pSql,
       .code    = 0
   };
-  rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
+
+  pSql->SRpcReqContext = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
+  assert(pSql->SRpcReqContext != NULL);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -274,7 +274,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) {
     pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_RPC_NETWORK_UNAVAIL;
   } else {
-    tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code));
+    tscTrace("%p query is cancelled, code:%s", pSql, tstrerror(pRes->code));
   }
 
   if (pRes->code == TSDB_CODE_SUCCESS) {
@@ -414,7 +414,6 @@ void tscKillSTableQuery(SSqlObj *pSql) {
 
   for (int i = 0; i < pSql->numOfSubs; ++i) {
     SSqlObj *pSub = pSql->pSubs[i];
-
     if (pSub == NULL) {
       continue;
     }
@@ -424,7 +423,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
      * sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
      */
     pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-//    taosStopRpcConn(pSql->pSubs[i]->);
+    rpcCancelRequest(pSql->pSubs[i]->SRpcReqContext);
   }
 
   /*
@@ -644,14 +643,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols);
   pQueryMsg->numOfTags      = htonl(numOfTags);
   pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
-  pQueryMsg->queryType      = htons(pQueryInfo->type);
+  pQueryMsg->queryType      = htonl(pQueryInfo->type);
   
   size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
   pQueryMsg->numOfOutput = htons(numOfOutput);
-  if (numOfOutput < 0) {
-    tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput);
-    return -1;
-  }
 
   // set column list ids
   size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
@@ -665,7 +660,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL ||
         pColSchema->type > TSDB_DATA_TYPE_NCHAR) {
       tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s",
-          pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex,
+          pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex,
                pColSchema->name);
 
       return TSDB_CODE_TSC_INVALID_SQL;
@@ -723,6 +718,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pMsg += sizeof(SSqlFuncMsg);
 
     for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
+      // todo add log
       pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType);
       pSqlFuncExpr->arg[j].argBytes = htons(pExpr->param[j].nLen);
 
@@ -784,7 +780,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
           (pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) {
         tscError("%p sid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s",
                  pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, total, numOfTagColumns,
-                 pCol->colIndex, pColSchema->name);
+                 pCol->colIndex.columnIndex, pColSchema->name);
 
         return TSDB_CODE_TSC_INVALID_SQL;
       }
@@ -800,6 +796,27 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     }
   }
 
+  // serialize tag column query condition
+  if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
+    STagCond* pTagCond = &pQueryInfo->tagCond;
+    
+    SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid);
+    if (pCond != NULL && pCond->cond != NULL) {
+      pQueryMsg->tagCondLen = htons(pCond->len);
+      memcpy(pMsg, pCond->cond, pCond->len);
+      
+      pMsg += pCond->len;
+    }
+  }
+  
+  if (pQueryInfo->tagCond.tbnameCond.cond == NULL) {
+    *pMsg = 0;
+    pMsg++;
+  } else {
+    strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond);
+    pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1;
+  }
+
   // compressed ts block
   pQueryMsg->tsOffset = htonl(pMsg - pStart);
   int32_t tsLen = 0;
@@ -824,27 +841,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
   }
 
-  // serialize tag column query condition
-  if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
-    STagCond* pTagCond = &pQueryInfo->tagCond;
-    
-    SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid);
-    if (pCond != NULL && pCond->cond != NULL) {
-      pQueryMsg->tagCondLen = htons(pCond->len);
-      memcpy(pMsg, pCond->cond, pCond->len);
-      
-      pMsg += pCond->len;
-    }
-  }
-  
-  if (pQueryInfo->tagCond.tbnameCond.cond == NULL) {
-    *pMsg = 0;
-    pMsg++;
-  } else {
-    strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond);
-    pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1;
-  }
-
   int32_t msgLen = pMsg - pStart;
 
   tscTrace("%p msg built success,len:%d bytes", pSql, msgLen);
@@ -983,7 +979,7 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload;
 
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  strncpy(pDropDbMsg->db, pTableMetaInfo->name, tListLen(pDropDbMsg->db));
+  tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db));
   pDropDbMsg->ignoreNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0;
 
   pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DB;
@@ -1053,7 +1049,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  strcpy(pDropMsg->user, pTableMetaInfo->name);
+  tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1175,7 +1171,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   strcpy(pCreateTableMsg->tableId, pTableMetaInfo->name);
 
   // use dbinfo from table id without modifying current db info
-  tscGetDBInfoFromMeterId(pTableMetaInfo->name, pCreateTableMsg->db);
+  tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateTableMsg->db);
 
   SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo;
 
@@ -1252,7 +1248,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   }
   
   SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload;
-  tscGetDBInfoFromMeterId(pTableMetaInfo->name, pAlterTableMsg->db);
+  tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db);
 
   strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
   pAlterTableMsg->type = htons(pAlterInfo->type);
@@ -1473,7 +1469,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pMsg += len;
   }
 
-  pCmd->payloadLen = pMsg - (char*)pInfoMsg;;
+  pCmd->payloadLen = pMsg - (char*)pInfoMsg;
   pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META;
 
   tfree(tmpData);
@@ -1577,7 +1573,7 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pMsg = pStart;
 
   SMgmtHead *pMgmt = (SMgmtHead *)pMsg;
-  tscGetDBInfoFromMeterId(pTableMetaInfo->name, pMgmt->db);
+  tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pMgmt->db);
 
   pMsg += sizeof(SMgmtHead);
 
@@ -1813,6 +1809,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   
   // todo handle out of memory case
   if (pTableMetaInfo->pTableMeta == NULL) {
+    free(pTableMeta);
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
@@ -1932,113 +1929,6 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
 }
 
 int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
-#if 0
-  void **      metricMetaList = NULL;
-  int32_t *    sizes = NULL;
-  
-  int32_t num = htons(*(int16_t *)rsp);
-  rsp += sizeof(int16_t);
-
-  metricMetaList = calloc(1, POINTER_BYTES * num);
-  sizes = calloc(1, sizeof(int32_t) * num);
-
-  // return with error code
-  if (metricMetaList == NULL || sizes == NULL) {
-    tfree(metricMetaList);
-    tfree(sizes);
-    pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-
-    return pSql->res.code;
-  }
-
-  for (int32_t k = 0; k < num; ++k) {
-    pMeta = (SSuperTableMeta *)rsp;
-
-    size_t size = (size_t)pSql->res.rspLen - 1;
-    rsp = rsp + sizeof(SSuperTableMeta);
-
-    pMeta->numOfTables = htonl(pMeta->numOfTables);
-    pMeta->numOfVnodes = htonl(pMeta->numOfVnodes);
-    pMeta->tagLen = htons(pMeta->tagLen);
-
-    size += pMeta->numOfVnodes * sizeof(SVnodeSidList *) + pMeta->numOfTables * sizeof(STableIdInfo *);
-
-    char *pBuf = calloc(1, size);
-    if (pBuf == NULL) {
-      pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto _error_clean;
-    }
-
-    SSuperTableMeta *pNewMetricMeta = (SSuperTableMeta *)pBuf;
-    metricMetaList[k] = pNewMetricMeta;
-
-    pNewMetricMeta->numOfTables = pMeta->numOfTables;
-    pNewMetricMeta->numOfVnodes = pMeta->numOfVnodes;
-    pNewMetricMeta->tagLen = pMeta->tagLen;
-
-    pBuf = pBuf + sizeof(SSuperTableMeta) + pNewMetricMeta->numOfVnodes * sizeof(SVnodeSidList *);
-
-    for (int32_t i = 0; i < pMeta->numOfVnodes; ++i) {
-      SVnodeSidList *pSidLists = (SVnodeSidList *)rsp;
-      memcpy(pBuf, pSidLists, sizeof(SVnodeSidList));
-
-      pNewMetricMeta->list[i] = pBuf - (char *)pNewMetricMeta;  // offset value
-      SVnodeSidList *pLists = (SVnodeSidList *)pBuf;
-
-      tscTrace("%p metricmeta:vid:%d,numOfTables:%d", pSql, i, pLists->numOfSids);
-
-      pBuf += sizeof(SVnodeSidList) + sizeof(STableIdInfo *) * pSidLists->numOfSids;
-      rsp += sizeof(SVnodeSidList);
-
-      size_t elemSize = sizeof(STableIdInfo) + pNewMetricMeta->tagLen;
-      for (int32_t j = 0; j < pSidLists->numOfSids; ++j) {
-        pLists->pSidExtInfoList[j] = pBuf - (char *)pLists;
-        memcpy(pBuf, rsp, elemSize);
-
-        ((STableIdInfo *)pBuf)->uid = htobe64(((STableIdInfo *)pBuf)->uid);
-        ((STableIdInfo *)pBuf)->sid = htonl(((STableIdInfo *)pBuf)->sid);
-
-        rsp += elemSize;
-        pBuf += elemSize;
-      }
-    }
-
-    sizes[k] = pBuf - (char *)pNewMetricMeta;
-  }
-
-  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
-  for (int32_t i = 0; i < num; ++i) {
-    char name[TSDB_MAX_TAGS_LEN + 1] = {0};
-
-    STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
-    tscGetMetricMetaCacheKey(pQueryInfo, name, pTableMetaInfo->pTableMeta->uid);
-
-#ifdef _DEBUG_VIEW
-    printf("generate the metric key:%s, index:%d\n", name, i);
-#endif
-
-    // release the used metricmeta
-    taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false);
-    pTableMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i],
-                                                                      sizes[i], tsMetricMetaKeepTimer);
-    tfree(metricMetaList[i]);
-
-    // failed to put into cache
-    if (pTableMetaInfo->pMetricMeta == NULL) {
-      pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto _error_clean;
-    }
-  }
-
-_error_clean:
-  // free allocated resource
-  for (int32_t i = 0; i < num; ++i) {
-    tfree(metricMetaList[i]);
-  }
-
-  free(sizes);
-  free(metricMetaList);
-#endif
   SSqlRes* pRes = &pSql->res;
   
   // NOTE: the order of several table must be preserved.
@@ -2291,7 +2181,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) {
   }
 
   pRes->row = 0;
-  tscTrace("%p numOfRows:%d, offset:%d, complete:%d", pSql, pRes->numOfRows, pRes->offset, pRes->completed);
+  tscTrace("%p numOfRows:%" PRId64 ", offset:%" PRId64 ", complete:%d", pSql, pRes->numOfRows, pRes->offset, pRes->completed);
 
   return 0;
 }
@@ -2432,6 +2322,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
   
   SQueryInfo *pNewQueryInfo = NULL;
   if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) {
+    tscFreeSqlObj(pNew);
     return code;
   }
   

src/client/src/tscSql.c

@@ -20,14 +20,9 @@
 #include "tnote.h"
 #include "trpc.h"
 #include "tscLog.h"
-#include "tscProfile.h"
-#include "tscSecondaryMerge.h"
 #include "tscSubquery.h"
 #include "tscUtil.h"
 #include "tsclient.h"
-#include "tscompression.h"
-#include "tsocket.h"
-#include "ttimer.h"
 #include "ttokendef.h"
 #include "tutil.h"
 
@@ -589,7 +584,7 @@ char *taos_errstr(TAOS_RES *tres) {
 
 void taos_config(int debug, char *log_path) {
   uDebugFlag = debug;
-  strcpy(tsLogDir, log_path);
+  tstrncpy(tsLogDir, log_path, TSDB_FILENAME_LEN);
 }
 
 char *taos_get_server_info(TAOS *taos) {
@@ -612,7 +607,9 @@ int* taos_fetch_lengths(TAOS_RES *res) {
 char *taos_get_client_info() { return version; }
 
 void taos_stop_query(TAOS_RES *res) {
-  if (res == NULL) return;
+  if (res == NULL) {
+    return;
+  }
 
   SSqlObj *pSql = (SSqlObj *)res;
   SSqlCmd *pCmd = &pSql->cmd;
@@ -632,7 +629,7 @@ void taos_stop_query(TAOS_RES *res) {
     return;
   }
 
-  //taosStopRpcConn(pSql->thandle);
+  rpcCancelRequest(pSql->SRpcReqContext);
   tscTrace("%p query is cancelled", res);
 }
 
@@ -724,6 +721,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
   if (sqlLen > tsMaxSQLStringLen) {
     tscError("%p sql too long", pSql);
     pRes->code = TSDB_CODE_TSC_INVALID_SQL;
+    tfree(pSql);
     return pRes->code;
   }
 
@@ -732,6 +730,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("%p failed to malloc sql string buffer", pSql);
     tscTrace("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
+    tfree(pSql);
     return pRes->code;
   }
 
@@ -856,6 +855,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   if (tblListLen > MAX_TABLE_NAME_LENGTH) {
     tscError("%p tableNameList too long, length:%d, maximum allowed:%d", pSql, tblListLen, MAX_TABLE_NAME_LENGTH);
     pRes->code = TSDB_CODE_TSC_INVALID_SQL;
+    tfree(pSql);
     return pRes->code;
   }
 
@@ -863,6 +863,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   if (str == NULL) {
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("%p failed to malloc sql string buffer", pSql);
+    tfree(pSql);
     return pRes->code;
   }
 
@@ -878,6 +879,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   free(str);
 
   if (pRes->code != TSDB_CODE_SUCCESS) {
+    tscFreeSqlObj(pSql);
     return pRes->code;
   }
 

src/client/src/tscStream.c

@@ -125,7 +125,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
   }
 
   // launch stream computing in a new thread
-  SSchedMsg schedMsg;
+  SSchedMsg schedMsg = { 0 };
   schedMsg.fp = tscProcessStreamLaunchQuery;
   schedMsg.ahandle = pStream;
   schedMsg.thandle = (void *)1;
@@ -239,7 +239,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
         /* no resuls in the query range, retry */
         // todo set retry dynamic time
         int32_t retry = tsProjectExecInterval;
-        tscError("%p stream:%p, retrieve no data, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retry);
+        tscError("%p stream:%p, retrieve no data, code:%d, retry in %" PRId32 "ms", pSql, pStream, numOfRows, retry);
 
         tscSetRetryTimer(pStream, pStream->pSql, retry);
         return;
@@ -250,7 +250,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
       }
     }
 
-    tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%d", pSql, pStream, pTableMetaInfo->name,
+    tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name,
              pStream->numOfRes);
 
     // release the metric/meter meta information reference, so data in cache can be updated
@@ -499,7 +499,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
   if (pSql->sqlstr == NULL) {
     tscError("%p failed to malloc sql string buffer", pSql);
     tscFreeSqlObj(pSql);
-    return NULL;;
+    return NULL;
   }
   strtolower(pSql->sqlstr, sqlstr);
 

src/client/src/tscSub.c

@@ -238,7 +238,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
 
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     taosArraySort( tables, tscCompareTidTags );
-    tscBuildVgroupTableInfo( pTableMetaInfo, tables );
+    tscBuildVgroupTableInfo(pSql, pTableMetaInfo, tables);
   }
   taosArrayDestroy(tables);
 
@@ -291,7 +291,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
   fclose(fp);
 
   taosArraySort(progress, tscCompareSubscriptionProgress);
-  tscTrace("subscription progress loaded, %d tables: %s", taosArrayGetSize(progress), pSub->topic);
+  tscTrace("subscription progress loaded, %zu tables: %s", taosArrayGetSize(progress), pSub->topic);
   return 1;
 }
 
@@ -350,7 +350,7 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
 
   pSub->interval = interval;
   if (fp != NULL) {
-    tscTrace("asynchronize subscription, create new timer", topic);
+    tscTrace("asynchronize subscription, create new timer: %s", topic);
     pSub->fp = fp;
     pSub->param = param;
     taosTmrReset(tscProcessSubscriptionTimer, interval, pSub, tscTmr, &pSub->pTimer);
@@ -435,7 +435,9 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
   } else {
     char path[256];
     sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
-    remove(path);
+    if (remove(path) != 0) {
+      tscError("failed to remove progress file, topic = %s, error = %s", pSub->topic, strerror(errno));
+    }
   }
 
   tscFreeSqlObj(pSub->pSql);

src/client/src/tscSystem.c

@@ -220,7 +220,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
 
         if (strlen(tsLocale) == 0) { // locale does not set yet
           char* defaultLocale = setlocale(LC_CTYPE, "");
-          strcpy(tsLocale, defaultLocale);
+          tstrncpy(tsLocale, defaultLocale, sizeof(tsLocale));
         }
 
         // set the user specified locale
@@ -304,7 +304,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       assert(cfg != NULL);
     
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        strcpy(tsTimezone, pStr);
+        tstrncpy(tsTimezone, pStr, sizeof(tsTimezone));
         tsSetTimeZone();
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscTrace("timezone set:%s, input:%s by taos_options", tsTimezone, pStr);

src/client/src/tscUtil.c

@@ -22,7 +22,7 @@
 #include "tkey.h"
 #include "tmd5.h"
 #include "tscProfile.h"
-#include "tscSecondaryMerge.h"
+#include "tscLocalMerge.h"
 #include "tscSubquery.h"
 #include "tschemautil.h"
 #include "tsclient.h"
@@ -32,7 +32,7 @@
 
 static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
 static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache);
-  
+
   SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
   if (pTagCond->pCond == NULL) {
     return NULL;
@@ -70,13 +70,6 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
   taosArrayPush(pTagCond->pCond, &cond);
 }
 
-bool tscQueryOnSTable(SSqlCmd* pCmd) {
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-
-  return ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) == TSDB_QUERY_TYPE_STABLE_QUERY) &&
-         (pCmd->msgType == TSDB_MSG_TYPE_QUERY);
-}
-
 bool tscQueryTags(SQueryInfo* pQueryInfo) {
   for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
@@ -95,32 +88,8 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) {
   return true;
 }
 
-bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) {
+// todo refactor, extract methods and move the common module
-  bool    hasTags = false;
+void tscGetDBInfoFromTableFullName(char* tableId, char* db) {
-  int32_t numOfSelectivity = 0;
-
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId;
-    if (functId == TSDB_FUNC_TAG_DUMMY) {
-      hasTags = true;
-      continue;
-    }
-
-    if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
-      numOfSelectivity++;
-    }
-  }
-
-  if (numOfSelectivity > 0 && hasTags) {
-    return true;
-  }
-
-  return false;
-}
-
-void tscGetDBInfoFromMeterId(char* tableId, char* db) {
   char* st = strstr(tableId, TS_PATH_DELIMITER);
   if (st != NULL) {
     char* end = strstr(st + 1, TS_PATH_DELIMITER);
@@ -181,8 +150,14 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   
   for (int32_t i = 0; i < numOfExprs; ++i) {
     int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
-    if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG &&
+
-        functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) {
+    if (functionId != TSDB_FUNC_PRJ &&
+        functionId != TSDB_FUNC_TAGPRJ &&
+        functionId != TSDB_FUNC_TAG &&
+        functionId != TSDB_FUNC_TS &&
+        functionId != TSDB_FUNC_ARITHM &&
+        functionId != TSDB_FUNC_TS_COMP &&
+        functionId != TSDB_FUNC_TID_TAG) {
       return false;
     }
   }
@@ -209,10 +184,14 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde
   return pQueryInfo->order.orderColId >= 0;
 }
 
-bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) {
+bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+  size_t size = tscSqlExprNumOfExprs(pQueryInfo);
+
+  for (int32_t i = 0; i < size; ++i) {
     int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
-    if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TS) {
+
+    if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG &&
+        functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) {
       return false;
     }
   }
@@ -225,9 +204,10 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
   
   for (int32_t i = 0; i < size; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-    if (pExpr == NULL) {
+    assert(pExpr != NULL);
-      return false;
+//    if (pExpr == NULL) {
-    }
+//      return false;
+//    }
 
     int32_t functionId = pExpr->functionId;
     if (functionId == TSDB_FUNC_TAG) {
@@ -238,6 +218,7 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
       return false;
     }
   }
+
   return true;
 }
 
@@ -1451,8 +1432,6 @@ bool tscShouldBeFreed(SSqlObj* pSql) {
     return false;
   }
   
-  assert(pSql->fp != NULL);
-  
   STscObj* pTscObj = pSql->pTscObj;
   if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) {
     return false;
@@ -1774,7 +1753,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     SQueryInfo* pPrevQueryInfo = tscGetQueryInfoDetail(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex);
     pNewQueryInfo->type = pPrevQueryInfo->type;
   } else {
-    pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY;  // it must be the subquery
+    TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery
   }
 
   uint64_t uid = pTableMetaInfo->pTableMeta->uid;
@@ -1799,19 +1778,26 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     }
 
     // make sure the the sqlExpr for each fields is correct
-// todo handle the agg arithmetic expression
+    // todo handle the agg arithmetic expression
+    numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo);
+
     for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) {
       TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f);
-      numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo);
+      bool matched = false;
-      
+
       for(int32_t k1 = 0; k1 < numOfExprs; ++k1) {
         SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1);
-        
+
-        if (strcmp(field->name, pExpr1->aliasName) == 0) {  // eatablish link according to the result field name
+        if (strcmp(field->name, pExpr1->aliasName) == 0) {  // establish link according to the result field name
           SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f);
           pInfo->pSqlExpr = pExpr1;
+
+          matched = true;
+          break;
         }
       }
+
+      assert(matched);
     }
   
     tscFieldInfoUpdateOffset(pNewQueryInfo);
@@ -1849,12 +1835,12 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     assert(pFinalInfo->vgroupList != NULL);
   }
-  
+
   if (cmd == TSDB_SQL_SELECT) {
     size_t size = taosArrayGetSize(pNewQueryInfo->colList);
     
     tscTrace(
-        "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d,"
+        "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%zu, colList:%zu,"
         "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64,
         pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo),
         size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey,
@@ -1900,16 +1886,21 @@ void tscDoQuery(SSqlObj* pSql) {
     }
   
     if (QUERY_IS_JOIN_QUERY(type)) {
-      if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0) {
+      if (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) {
         tscHandleMasterJoinQuery(pSql);
-        return;
+      } else { // for first stage sub query, iterate all vnodes to get all timestamp
-      } else {
+        if (!TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) {
-        // for first stage sub query, iterate all vnodes to get all timestamp
+          tscProcessSql(pSql);
-        if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) {
+        } else { // secondary stage join query.
-//          doProcessSql(pSql);
+          if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {  // super table query
-          assert(0);
+            tscHandleMasterSTableQuery(pSql);
+          } else {
+            tscProcessSql(pSql);
+          }
         }
       }
+
+      return;
     } else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {  // super table query
       tscHandleMasterSTableQuery(pSql);
       return;
@@ -1919,11 +1910,13 @@ void tscDoQuery(SSqlObj* pSql) {
   }
 }
 
-int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid) {
+int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) {
   if (pTagCond->joinInfo.left.uid == uid) {
-    return pTagCond->joinInfo.left.tagCol;
+    return pTagCond->joinInfo.left.tagColId;
+  } else if (pTagCond->joinInfo.right.uid == uid) {
+    return pTagCond->joinInfo.right.tagColId;
   } else {
-    return pTagCond->joinInfo.right.tagCol;
+    assert(0);
   }
 }
 
@@ -1980,11 +1973,10 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) {
     return false;
   }
 
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  
-  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   assert(pRes->completed);
-  
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
   // for normal table, no need to try any more if results are all retrieved from one vnode
   if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || (pTableMetaInfo->vgroupList == NULL)) {
     return false;
@@ -2006,12 +1998,11 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
    * if case of: multi-vnode super table projection query
    */
   assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes));
-
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
   int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
   while (++pTableMetaInfo->vgroupIndex < totalVgroups) {
-    tscTrace("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%d", pSql,
+    tscTrace("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql,
              pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal);
 
     /*

src/client/tests/timeParseTest.cpp

@@ -24,19 +24,19 @@ TEST(testCase, parse_time) {
 
   int64_t time = 0, time1 = 0;
 
-  taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 1514739661952);
 
-  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
 
   char t2[] = "2018-1-1T1:1:1.952Z";
-  taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0);
 
   EXPECT_EQ(time, 1514739661952 + 28800000);
 
   char t3[] = "2018-1-1 1:01:01.952";
-  taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 1514739661952);
 
   char t4[] = "2018-1-1 1:01:01.9";
@@ -45,122 +45,122 @@ TEST(testCase, parse_time) {
   char t7[] = "2018-01-01 01:01:01.9";
   char t8[] = "2018-01-01 01:01:01.9007865";
 
-  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
-  taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, time1);
 
-  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
-  taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, time1);
 
-  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
-  taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, time1);
 
-  taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0);
-  taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, time1);
 
   char t9[] = "2017-4-3 1:1:2.980";
   char t10[] = "2017-4-3T2:1:2.98+9:00";
-  taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI, 0);
-  taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, time1);
 
   char t11[] = "2017-4-3T2:1:2.98+09:00";
-  taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0);
-  taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, time1);
 
   char t12[] = "2017-4-3T2:1:2.98+0900";
-  taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0);
-  taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, time1);
 
   taos_options(TSDB_OPTION_TIMEZONE, "UTC");  
   deltaToUtcInitOnce();
   
-  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 0);
 
   taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
   deltaToUtcInitOnce();
   
   char t14[] = "1970-1-1T0:0:0Z";
-  taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 0);
 
   char t40[] = "1970-1-1 0:0:0.999999999";
-  taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
 
   char t41[] = "1997-1-1 0:0:0.999999999";
-  taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 852048000999);
 
   int64_t k = timezone;
   char    t42[] = "1997-1-1T0:0:0.999999999Z";
-  taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
 
   ////////////////////////////////////////////////////////////////////
   // illegal timestamp format
   char t15[] = "2017-12-33 0:0:0";
-  EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1);
+  EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI, 0), -1);
 
   char t16[] = "2017-12-31 99:0:0";
-  EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1);
+  EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI, 0), -1);
 
   char t17[] = "2017-12-31T9:0:0";
-  EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1);
+  EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI, 0), -1);
 
   char t18[] = "2017-12-31T9:0:0.Z";
-  EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1);
+  EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI, 0), -1);
 
   char t19[] = "2017-12-31 9:0:0.-1";
-  EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1);
+  EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI, 0), -1);
 
   char t20[] = "2017-12-31 9:0:0.1+12:99";
-  EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0);
+  EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI, 0), 0);
   EXPECT_EQ(time, 1514682000100);
 
   char t21[] = "2017-12-31T9:0:0.1+12:99";
-  EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1);
+  EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI, 0), -1);
 
   char t22[] = "2017-12-31 9:0:0.1+13:1";
-  EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0);
+  EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI, 0), 0);
 
   char t23[] = "2017-12-31T9:0:0.1+13:1";
-  EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0);
+  EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI, 0), 0);
 
 
   //======================== add some case ============================//
   
   char b1[] = "9999-12-31 23:59:59.999";
-  taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI,0);
   EXPECT_EQ(time, 253402271999999);
 
 
   char b2[] = "2020-01-01 01:01:01.321";
-  taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 1577811661321);
 
   taos_options(TSDB_OPTION_TIMEZONE, "America/New_York");  
   deltaToUtcInitOnce();
   
-  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND);
 
   taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo");  
   deltaToUtcInitOnce();
   
-  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND);
 
   taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
   deltaToUtcInitOnce();
   
-  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND);
 }
 

src/common/inc/tdataformat.h

@@ -36,7 +36,7 @@ extern "C" {
 
 #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)      \
   do {                                                 \
-    char *_e = stpncpy(varDataVal(x), (str), (_maxs)); \
+    char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \
     varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE));   \
   } while (0)
 

src/common/inc/tglobal.h

@@ -92,10 +92,6 @@ extern int32_t tsNumOfMnodes;
 extern int32_t tsMaxShellConns;
 extern int32_t tsMaxTables;
 
-extern char tsDefaultDB[];
-extern char tsDefaultUser[];
-extern char tsDefaultPass[];
-
 extern char tsMqttBrokerAddress[];
 extern char tsMqttBrokerClientId[];
 

src/common/src/tdataformat.c

@@ -105,7 +105,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
 }
 
 int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int32_t bytes) {
-  if (!isValidDataType(type, 0)) return -1;
+  if (!isValidDataType(type)) return -1;
 
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
@@ -588,7 +588,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) {
         if (kvRowNCols(nrow) - colIdx - 1 > 0) {
           for (int i = colIdx + 1; i < kvRowNCols(nrow); i++) {
             kvRowColIdxAt(nrow, i)->colId = kvRowColIdxAt(row, i)->colId;
-            kvRowColIdxAt(nrow, i)->offset += diff;
+            kvRowColIdxAt(nrow, i)->offset = kvRowColIdxAt(row, i)->offset + diff;
           }
           memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx + 1)), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1)),
                  POINTER_DISTANCE(kvRowEnd(row), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1))));

src/common/src/tglobal.c

@@ -109,10 +109,6 @@ int32_t tsReplications  = TSDB_DEFAULT_REPLICA_NUM;
 int16_t tsAffectedRowsMod = 0;
 int32_t tsNumOfMnodes = 3;
 int32_t tsMaxShellConns = 5000;
-
-char    tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
-char    tsDefaultUser[64] = "root";
-char    tsDefaultPass[64] = "taosdata";
 int32_t tsMaxConnections = 5000;
 
 int32_t tsBalanceInterval = 300;  // seconds
@@ -713,37 +709,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  // login configs
-  cfg.option = "defaultDB";
-  cfg.ptr = tsDefaultDB;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
-  cfg.option = "defaultUser";
-  cfg.ptr = tsDefaultUser;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_USER_LEN - 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
-  cfg.option = "defaultPass";
-  cfg.ptr = tsDefaultPass;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_NOT_PRINT;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_PASSWORD_LEN - 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   cfg.option = "mqttBrokerAddress";
   cfg.ptr = tsMqttBrokerAddress;
   cfg.valType = TAOS_CFG_VTYPE_STRING;

src/common/src/ttypes.c

@@ -363,16 +363,8 @@ char tTokenTypeSwitcher[13] = {
     TSDB_DATA_TYPE_NCHAR,   // TK_NCHAR
 };
 
-bool isValidDataType(int32_t type, int32_t length) {
+bool isValidDataType(int32_t type) {
-  if (type < TSDB_DATA_TYPE_NULL || type > TSDB_DATA_TYPE_NCHAR) {
+  return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR;
-    return false;
-  }
-
-  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-//    return length >= 0 && length <= TSDB_MAX_BINARY_LEN;
-  }
-
-  return true;
 }
 
 bool isNull(const char *val, int32_t type) {

src/cq/src/cqMain.c

@@ -256,11 +256,30 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
   SDataRow trow = (SDataRow)pBlk->data;
   tdInitDataRow(trow, pSchema);
 
+  union {
+    char buf[sizeof(int64_t)];
+    tstr str;
+  } nullVal;
+
   for (int32_t i = 0; i < pSchema->numOfCols; i++) {
     STColumn *c = pSchema->columns + i;
     char* val = (char*)row[i];
     if (IS_VAR_DATA_TYPE(c->type)) {
-      val -= sizeof(VarDataLenT);
+      if (val == NULL) {
+        val = nullVal.buf;
+        if (c->type == TSDB_DATA_TYPE_BINARY) {
+          setNull(nullVal.str.data, TSDB_DATA_TYPE_BINARY, 1);
+          nullVal.str.len = 1;
+        } else {
+          setNull(nullVal.str.data, TSDB_DATA_TYPE_NCHAR, 4);
+          nullVal.str.len = 4;
+        }
+      } else {
+        val -= sizeof(VarDataLenT);
+      }
+    } else if (val == NULL) {
+      val = nullVal.buf;
+      setNull(val, c->type, c->bytes);
     }
     tdAppendColVal(trow, val, c->type, c->bytes, c->offset);
   }

src/cq/test/cqtest.c

@@ -48,8 +48,8 @@ int main(int argc, char *argv[]) {
   taosInitLog("cq.log", 100000, 10);
 
   SCqCfg cqCfg;
-  strcpy(cqCfg.user, "root");
+  strcpy(cqCfg.user, TSDB_DEFAULT_USER);
-  strcpy(cqCfg.pass, "taosdata");
+  strcpy(cqCfg.pass, TSDB_DEFAULT_PASS);
   cqCfg.vgId = 2;
   cqCfg.cqWrite = writeToQueue;
 

src/dnode/src/dnodeMWrite.c

@@ -118,6 +118,8 @@ void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
 
   SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
   mnodeCreateMsg(pWrite, pMsg);
+
+  dTrace("app:%p:%p, msg:%s is put into mwrite queue", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType]);
   taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
 }
 
@@ -128,6 +130,7 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
 
 void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) {
   SMnodeMsg *pWrite = pRaw;
+  if (pWrite == NULL) return;
   if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
   if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
     dnodeReprocessMnodeWriteMsg(pWrite);
@@ -146,19 +149,21 @@ void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) {
 }
 
 static void *dnodeProcessMnodeWriteQueue(void *param) {
-  SMnodeMsg *pWriteMsg;
+  SMnodeMsg *pWrite;
   int32_t    type;
   void *     unUsed;
   
   while (1) {
-    if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWriteMsg, &unUsed) == 0) {
+    if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWrite, &unUsed) == 0) {
       dTrace("dnodeProcessMnodeWriteQueue: got no message from qset, exiting...");
       break;
     }
 
-    dTrace("%p, msg:%s will be processed in mwrite queue", pWriteMsg->rpcMsg.ahandle, taosMsg[pWriteMsg->rpcMsg.msgType]);    
+    dTrace("app:%p:%p, msg:%s will be processed in mwrite queue", pWrite->rpcMsg.ahandle, pWrite,
-    int32_t code = mnodeProcessWrite(pWriteMsg);    
+           taosMsg[pWrite->rpcMsg.msgType]);
-    dnodeSendRpcMnodeWriteRsp(pWriteMsg, code);
+
+    int32_t code = mnodeProcessWrite(pWrite);
+    dnodeSendRpcMnodeWriteRsp(pWrite, code);
   }
 
   return NULL;
@@ -168,9 +173,15 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
   SMnodeMsg *pWrite = pMsg;
 
   if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
+    dTrace("app:%p:%p, msg:%s is redirected for mnode not running, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
+           taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
+
     dnodeSendRedirectMsg(pMsg, true);
     dnodeFreeMnodeWriteMsg(pWrite);
-  } else {    
+  } else {
+    dTrace("app:%p:%p, msg:%s is reput into mwrite queue, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
+           taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
+
     taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
   }
 }

src/dnode/src/dnodeMgmt.c

@@ -407,11 +407,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
     pMnodeInfo->nodeId   = htonl(pMnodeInfo->nodeId);
   }
 
-  SDMVgroupAccess *pVgAcccess = pStatusRsp->vgAccess;
+  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
-  for (int32_t i = 0; i < pCfg->numOfVnodes; ++i) {
-    pVgAcccess[i].vgId = htonl(pVgAcccess[i].vgId);
-  }
-  
   dnodeProcessModuleStatus(pCfg->moduleStatus);
   dnodeUpdateDnodeCfg(pCfg);
 
@@ -616,6 +612,16 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
   pStatus->diskAvailable    = tsAvailDataDirGB;
   pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
+
+  // fill cluster cfg parameters
+  pStatus->clusterCfg.numOfMnodes        = tsNumOfMnodes;
+  pStatus->clusterCfg.mnodeEqualVnodeNum = tsMnodeEqualVnodeNum;
+  pStatus->clusterCfg.offlineThreshold   = tsOfflineThreshold;
+  pStatus->clusterCfg.statusInterval     = tsStatusInterval;
+  strcpy(pStatus->clusterCfg.arbitrator, tsArbitrator);
+  strcpy(pStatus->clusterCfg.timezone, tsTimezone);
+  strcpy(pStatus->clusterCfg.locale, tsLocale);
+  strcpy(pStatus->clusterCfg.charset, tsCharset);  
   
   vnodeBuildStatusMsg(pStatus);
   contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);

src/inc/query.h

@@ -32,11 +32,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs
 
 /**
  * Destroy QInfo object
- *
+ * @param qinfo  qhandle
- * @param qinfo
+ * @param fp     destroy callback function, while the qhandle is destoried, invoke the fp
- * @return
+ * @param param  free callback params
  */
-void qDestroyQueryInfo(qinfo_t qinfo);
+void qDestroyQueryInfo(qinfo_t qinfo, void (*fp)(void*), void* param);
 
 /**
  * the main query execution function, including query on both table and multitables,
@@ -45,7 +45,7 @@ void qDestroyQueryInfo(qinfo_t qinfo);
  * @param qinfo
  * @return
  */
-void qTableQuery(qinfo_t qinfo);
+void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param);
 
 /**
  * Retrieve the produced results information, if current query is not paused or completed,
@@ -80,9 +80,12 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo);
 
 /**
  * kill current ongoing query and free query handle automatically
- * @param qinfo
+ * @param qinfo  qhandle
+ * @param fp     destroy callback function, while the qhandle is destoried, invoke the fp
+ * @param param  free callback params
+ * @return
  */
-int32_t qKillQuery(qinfo_t qinfo);
+int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param);
 
 #ifdef __cplusplus
 }

src/inc/taos.h

@@ -55,7 +55,7 @@ typedef enum {
 typedef struct taosField {
   char     name[65];
   uint8_t  type;
-  short    bytes;
+  int16_t  bytes;
 } TAOS_FIELD;
 
 #ifdef _TD_GO_DLL_

src/inc/taosdef.h

@@ -83,6 +83,9 @@ extern const int32_t TYPE_BYTES[11];
 #define TSDB_DATA_NULL_STR              "NULL"
 #define TSDB_DATA_NULL_STR_L            "null"
 
+#define TSDB_DEFAULT_USER               "root"
+#define TSDB_DEFAULT_PASS               "taosdata"
+
 #define TSDB_TRUE   1
 #define TSDB_FALSE  0
 #define TSDB_OK     0
@@ -156,7 +159,7 @@ typedef struct tDataTypeDescriptor {
 extern tDataTypeDescriptor tDataTypeDesc[11];
 #define POINTER_BYTES sizeof(void *)  // 8 by default  assert(sizeof(ptrdiff_t) == sizseof(void*)
 
-bool isValidDataType(int32_t type, int32_t length);
+bool isValidDataType(int32_t type);
 bool isNull(const char *val, int32_t type);
 
 void setVardataNull(char* val, int32_t type);
@@ -209,8 +212,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MAX_SQL_SHOW_LEN     256
 #define TSDB_MAX_ALLOWED_SQL_LEN  (8*1024*1024U)          // sql length should be less than 8mb
 
-#define TSDB_MAX_BYTES_PER_ROW    65535
+#define TSDB_MAX_BYTES_PER_ROW    16384
-#define TSDB_MAX_TAGS_LEN         65535
+#define TSDB_MAX_TAGS_LEN         16384
 #define TSDB_MAX_TAGS             128
 
 #define TSDB_AUTH_LEN             16

src/inc/taoserror.h

@@ -113,14 +113,19 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID,         0, 0x030C, "mnode inva
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID,        0, 0x030D, "mnode invalid stream id")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID,          0, 0x030E, "mnode invalid connection")
 
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE,    0, 0x0320, "mnode object already there")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE,    0, 0x0320, "sdb object already there")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR,                0, 0x0321, "mnode sdb error")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR,                0, 0x0321, "sdb app error")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE,   0, 0x0322, "sdb invalid table type")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE,        0, 0x0323, "sdb object not there")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW,      0, 0x0324, "sdb invalid meta row")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE,      0, 0x0325, "sdb invalid key type")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST,      0, 0x0330, "mnode dnode already exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST,          0, 0x0331, "mnode dnode not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST,         0, 0x0332, "mnode vgroup not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER,         0, 0x0333, "mnode cant not remove master")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES,         0, 0x0334, "mnode no enough dnodes")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "mnode cluster cfg inconsistent")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST,       0, 0x0340, "mnode accounts already exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT,             0, 0x0341, "mnode invalid account")
@@ -174,6 +179,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS,      0, 0x0506, "vnode no d
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR,      0, 0x0507, "vnode no such file or directory")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY,            0, 0x0508, "vnode out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR,                0, 0x0509, "vnode app error")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0214, "vnode no write auth")
 
 // tsdb
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         0, 0x0600, "tsdb invalid table id")
@@ -195,6 +201,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG,              0, 0x0701, "query inva
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE,             0, 0x0702, "query no diskspace")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY,            0, 0x0703, "query out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR,                0, 0x0704, "query app error")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY,             0, 0x0705, "query duplicated join key")
 
 // grant
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                0, 0x0800, "grant expired")

src/inc/taosmsg.h

@@ -455,7 +455,7 @@ typedef struct {
   int16_t     orderType;        // used in group by xx order by xxx
   int64_t     limit;
   int64_t     offset;
-  uint16_t    queryType;        // denote another query process
+  uint32_t    queryType;        // denote another query process
   int16_t     numOfOutput;  // final output columns numbers
   int16_t     tagNameRelType;   // relation of tag criteria and tbname criteria
   int16_t     fillType;      // interpolate type
@@ -543,6 +543,7 @@ typedef struct {
   int32_t  dnodeId;
   uint32_t moduleStatus;
   uint32_t numOfVnodes;
+  uint32_t reserved;
 } SDMDnodeCfg;
 
 typedef struct {
@@ -557,18 +558,30 @@ typedef struct {
 } SDMMnodeInfos;
 
 typedef struct {
-  uint32_t   version;
+  int32_t  numOfMnodes;               // tsNumOfMnodes
-  int32_t    dnodeId;
+  int32_t  mnodeEqualVnodeNum;        // tsMnodeEqualVnodeNum
-  char       dnodeEp[TSDB_EP_LEN];
+  int32_t  offlineThreshold;          // tsOfflineThreshold
-  uint32_t   moduleStatus;
+  int32_t  statusInterval;            // tsStatusInterval
-  uint32_t   lastReboot;        // time stamp for last reboot
+  char     arbitrator[TSDB_EP_LEN];   // tsArbitrator
-  uint16_t   numOfTotalVnodes;  // from config file
+  char     timezone[64];              // tsTimezone
-  uint16_t   openVnodes;
+  char     locale[TSDB_LOCALE_LEN];   // tsLocale
-  uint16_t   numOfCores;
+  char     charset[TSDB_LOCALE_LEN];  // tsCharset
-  float      diskAvailable;  // GB
+} SClusterCfg;
-  uint8_t    alternativeRole;
+
-  uint8_t    reserve[15];
+typedef struct {
-  SVnodeLoad load[];
+  uint32_t    version;
+  int32_t     dnodeId;
+  char        dnodeEp[TSDB_EP_LEN];
+  uint32_t    moduleStatus;
+  uint32_t    lastReboot;        // time stamp for last reboot
+  uint16_t    numOfTotalVnodes;  // from config file
+  uint16_t    openVnodes;
+  uint16_t    numOfCores;
+  float       diskAvailable;  // GB
+  uint8_t     alternativeRole;
+  uint8_t     reserve[15];
+  SClusterCfg clusterCfg;
+  SVnodeLoad  load[];
 } SDMStatusMsg;
 
 typedef struct {

src/inc/trpc.h

@@ -84,7 +84,7 @@ void  rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet);
 int   rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void  rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp);
 int   rpcReportProgress(void *pConn, char *pCont, int contLen);
-void  rpcCanelRequest(void *pContext);
+void  rpcCancelRequest(void *pContext);
 
 #ifdef __cplusplus
 }

src/inc/vnode.h

@@ -58,6 +58,7 @@ void*   vnodeGetWal(void *pVnode);
 
 int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
 void    vnodeBuildStatusMsg(void * param);
+void    vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes);
 
 int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg);
 

src/kit/shell/src/shellEngine.c

@@ -56,11 +56,11 @@ TAOS *shellInit(SShellArguments *args) {
   if (args->is_use_passwd) {
     if (args->password == NULL) args->password = getpass("Enter password: ");
   } else {
-    args->password = tsDefaultPass;
+    args->password = TSDB_DEFAULT_PASS;
   }
 
   if (args->user == NULL) {
-    args->user = tsDefaultUser;
+    args->user = TSDB_DEFAULT_USER;
   }
 
   taos_init();
@@ -276,6 +276,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
   st = taosGetTimestampUs();
 
   TAOS_RES* pSql = taos_query(con, command);
+  result = pSql;  // set it into the global variable
+
   if (taos_errno(pSql)) {
     taos_error(pSql);
     return;
@@ -284,7 +286,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
   if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) {
     fprintf(stdout, "Database changed.\n\n");
     fflush(stdout);
-    
+
+    result = NULL;
     taos_free_result(pSql);
     return;
   }
@@ -294,6 +297,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
     int error_no = 0;
     int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode);
     if (numOfRows < 0) {
+      result = NULL;
       taos_free_result(pSql);
       return;
     }
@@ -315,7 +319,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
   if (fname != NULL) {
     wordfree(&full_path);
   }
-  
+
+  result = NULL;
   taos_free_result(pSql);
 }
 
@@ -419,8 +424,8 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_
   }
 }
 
-static int dumpResultToFile(const char* fname, TAOS_RES* result) {
+static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
-  TAOS_ROW row = taos_fetch_row(result);
+  TAOS_ROW row = taos_fetch_row(tres);
   if (row == NULL) {
     return 0;
   }
@@ -441,9 +446,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
 
   wordfree(&full_path);
 
-  int num_fields = taos_num_fields(result);
+  int num_fields = taos_num_fields(tres);
-  TAOS_FIELD *fields = taos_fetch_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(tres);
-  int precision = taos_result_precision(result);
+  int precision = taos_result_precision(tres);
 
   for (int col = 0; col < num_fields; col++) {
     if (col > 0) {
@@ -455,7 +460,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
   
   int numOfRows = 0;
   do {
-    int32_t* length = taos_fetch_lengths(result);
+    int32_t* length = taos_fetch_lengths(tres);
     for (int i = 0; i < num_fields; i++) {
       if (i > 0) {
         fputc(',', fp);
@@ -465,10 +470,13 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
     fputc('\n', fp);
 
     numOfRows++;
-    row = taos_fetch_row(result);
+    row = taos_fetch_row(tres);
   } while( row != NULL);
 
+  result = NULL;
+  taos_free_result(tres);
   fclose(fp);
+
   return numOfRows;
 }
 
@@ -769,8 +777,7 @@ void write_history() {
 
 void taos_error(TAOS_RES *tres) {
   fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres));
-
+  result = NULL;
-  /* free local resouce: allocated memory/metric-meta refcnt */
   taos_free_result(tres);
 }
 
@@ -845,9 +852,9 @@ void shellGetGrantInfo(void *con) {
 
   char sql[] = "show grants";
 
-  result = taos_query(con, sql);
+  TAOS_RES* tres = taos_query(con, sql);
 
-  int code = taos_errno(result);
+  int code = taos_errno(tres);
   if (code != TSDB_CODE_SUCCESS) {
     if (code == TSDB_CODE_COM_OPS_NOT_SUPPORT) {
       fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con));
@@ -857,18 +864,18 @@ void shellGetGrantInfo(void *con) {
     return;
   }
 
-  int num_fields = taos_field_count(result);
+  int num_fields = taos_field_count(tres);
   if (num_fields == 0) {
     fprintf(stderr, "\nInvalid grant information.\n");
     exit(0);
   } else {
-    if (result == NULL) {
+    if (tres == NULL) {
       fprintf(stderr, "\nGrant information is null.\n");
       exit(0);
     }
 
-    TAOS_FIELD *fields = taos_fetch_fields(result);
+    TAOS_FIELD *fields = taos_fetch_fields(tres);
-    TAOS_ROW row = taos_fetch_row(result);
+    TAOS_ROW row = taos_fetch_row(tres);
     if (row == NULL) {
       fprintf(stderr, "\nFailed to get grant information from server. Abort.\n");
       exit(0);
@@ -888,8 +895,8 @@ void shellGetGrantInfo(void *con) {
       fprintf(stdout, "Server is Enterprise %s Edition, version is %s and will expire at %s.\n", serverVersion, taos_get_server_info(con), expiretime);
     }
 
-    taos_free_result(result);
     result = NULL;
+    taos_free_result(tres);
   }
 
   fprintf(stdout, "\n");

src/kit/shell/src/shellMain.c

@@ -15,7 +15,6 @@
 
 #include "os.h"
 #include "shell.h"
-#include "tsclient.h"
 
 pthread_t pid;
 
@@ -23,14 +22,6 @@ pthread_t pid;
 void interruptHandler(int signum) {
 #ifdef LINUX
   taos_stop_query(result);
-  if (result != NULL) {
-    /*
-     * we need to free result in async model, in order to avoid free
-     * results while the master thread is waiting for server response.
-     */
-    tscQueueAsyncFreeResult(result);
-  }
-  
   result = NULL;
 #else
   printf("\nReceive ctrl+c or other signal, quit shell.\n");

src/kit/taosdemo/taosdemo.c

@@ -306,7 +306,7 @@ double getCurrentTime();
 void callBack(void *param, TAOS_RES *res, int code);
 
 int main(int argc, char *argv[]) {
-  SDemoArguments arguments = {NULL,            // host
+  SDemoArguments arguments = {  NULL,            // host
                                 0,               // port
                                 "root",          // user
                                 "taosdata",      // password

src/kit/taosdump/taosdump.c

@@ -34,6 +34,8 @@
 #include "taosdef.h"
 #include "tutil.h"
 
+#include "tglobal.h"
+
 #define COMMAND_SIZE 65536
 #define DEFAULT_DUMP_FILE "taosdump.sql"
 
@@ -293,7 +295,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 static struct argp argp = {options, parse_opt, args_doc, doc};
 
 TAOS *taos = NULL;
-TAOS_RES *result = NULL;
 char *command = NULL;
 char *lcommand = NULL;
 char *buffer = NULL;
@@ -324,7 +325,7 @@ void taosFreeDbInfos();
 int main(int argc, char *argv[]) {
   SDumpArguments arguments = {
     // connection option
-    NULL, "root", "taosdata", 0,
+    NULL, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS, 0,
     // output file
     DEFAULT_DUMP_FILE, DEFAULT_DUMP_FILE, NULL,
     // dump unit option
@@ -463,10 +464,10 @@ int taosDumpOut(SDumpArguments *arguments) {
   taosDumpCharset(fp);
 
   sprintf(command, "show databases");
-  result = taos_query(taos, command);
+  TAOS_RES* result = taos_query(taos, command);
   int32_t code = taos_errno(result);
   if (code != 0) {
-    fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos));
+    fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(result));
     taos_free_result(result);
     goto _exit_failure;
   }
@@ -502,7 +503,7 @@ int taosDumpOut(SDumpArguments *arguments) {
     }
 
     strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
-    if (strcmp(arguments->user, "root") == 0) {
+    if (strcmp(arguments->user, TSDB_DEFAULT_USER) == 0) {
       dbInfos[count]->replica = (int)(*((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]));
       dbInfos[count]->days = (int)(*((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]));
       dbInfos[count]->keep = *((int *)row[TSDB_SHOW_DB_KEEP_INDEX]);
@@ -613,7 +614,7 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
   fprintf(fp, "USE %s\n\n", dbInfo->name);
 
   sprintf(command, "show tables");
-  result = taos_query(taos,command);
+  TAOS_RES* result = taos_query(taos,command);
   int32_t code = taos_errno(result);
   if (code != 0) {
     fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -717,7 +718,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
 
     sprintf(command, "select %s from %s limit 1", tableDes->cols[counter].field, tableDes->name);
 
-    result = taos_query(taos, command);
+    TAOS_RES* result = taos_query(taos, command);
     int32_t code = taos_errno(result);
     if (code != 0) {
       fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -795,7 +796,7 @@ int taosGetTableDes(char *table, STableDef *tableDes) {
 
   sprintf(command, "describe %s", table);
 
-  result = taos_query(taos, command);
+  TAOS_RES* result = taos_query(taos, command);
   int32_t code = taos_errno(result);
   if (code != 0) {
     fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -875,7 +876,7 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
   tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
 
   sprintf(command, "select tbname from %s", metric);
-  result = taos_query(taos, command);
+  TAOS_RES* result = taos_query(taos, command);
   int32_t code = taos_errno(result);
   if (code != 0) {
     fprintf(stderr, "failed to run command %s, error: %s\n", command, taos_errstr(result));
@@ -928,7 +929,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) {
   sprintf(command, "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", tbname, arguments->start_time,
           arguments->end_time);
           
-  result = taos_query(taos, command);
+  TAOS_RES* result = taos_query(taos, command);
   int32_t code = taos_errno(result);
   if (code != 0) {
     fprintf(stderr, "failed to run command %s, reason: %s\n", command, taos_errstr(result));
@@ -1177,9 +1178,13 @@ int taosDumpIn(SDumpArguments *arguments) {
           tcommand = command;
         }
         taosReplaceCtrlChar(tcommand);
-        if (taos_query(taos, tcommand) == NULL)
+
+        TAOS_RES* result = taos_query(taos, tcommand);
+        if (taos_errno(result) != 0){
           fprintf(stderr, "linenu: %" PRId64 " failed to run command %s reason:%s \ncontinue...\n", linenu, command,
-                  taos_errstr(taos));
+                  taos_errstr(result));
+          taos_free_result(result);
+        }
 
         pstr = command;
         pstr[0] = '\0';
@@ -1225,12 +1230,12 @@ int taosDumpIn(SDumpArguments *arguments) {
         tcommand = command;
       }
       taosReplaceCtrlChar(tcommand);
-      result = taos_query(taos, tcommand);
+      TAOS_RES* result = taos_query(taos, tcommand);
       int32_t code = taos_errno(result);
       if (code != 0) 
       {
         fprintf(stderr, "linenu:%" PRId64 " failed to run command %s reason: %s \ncontinue...\n", linenu, command,
-                taos_errstr(taos));
+                taos_errstr(result));
       }
       taos_free_result(result);
     }

src/mnode/inc/mnodeDef.h

@@ -122,7 +122,8 @@ typedef struct SVgObj {
   int32_t        lbDnodeId;
   int32_t        lbTime;
   int8_t         inUse;
-  int8_t         reserved[13];
+  int8_t         accessState;
+  int8_t         reserved[12];
   int8_t         updateEnd[1];
   int32_t        refCount;
   struct SVgObj *prev, *next;

src/mnode/inc/mnodeDnode.h

@@ -47,7 +47,7 @@ void    mnodeDecDnodeRef(SDnodeObj *pDnode);
 void *  mnodeGetDnode(int32_t dnodeId);
 void *  mnodeGetDnodeByEp(char *ep);
 void    mnodeUpdateDnode(SDnodeObj *pDnode);
-int32_t mnodeDropDnode(SDnodeObj *pDnode);
+int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg);
 
 extern int32_t tsAccessSquence;
 

src/mnode/inc/mnodeInt.h

@@ -36,10 +36,10 @@ extern int32_t sdbDebugFlag;
 #define mLWarn(...)  { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__)  }
 #define mLPrint(...) { monitorSaveLog(0, __VA_ARGS__); mPrint(__VA_ARGS__) }
 
-#define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND-SDB ", 255, __VA_ARGS__); }}
+#define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR SDB ", 255, __VA_ARGS__); }}
-#define sdbWarn(...)  { if (sdbDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); }}
+#define sdbWarn(...)  { if (sdbDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN SDB ", sdbDebugFlag, __VA_ARGS__); }}
-#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("MND-SDB ", sdbDebugFlag, __VA_ARGS__);}}
+#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__);}}
-#define sdbPrint(...) { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); }
+#define sdbPrint(...) { taosPrintLog("SDB ", 255, __VA_ARGS__); }
 
 #define sdbLError(...) { monitorSaveLog(2, __VA_ARGS__); sdbError(__VA_ARGS__) }
 #define sdbLWarn(...)  { monitorSaveLog(1, __VA_ARGS__); sdbWarn(__VA_ARGS__)  }

src/mnode/inc/mnodeSdb.h

@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+struct SMnodeMsg;
+
 typedef enum {
   SDB_TABLE_DNODE   = 0,
   SDB_TABLE_MNODE   = 1,
@@ -48,8 +50,11 @@ typedef struct {
   ESdbOper type;
   void *   table;
   void *   pObj;
-  int32_t  rowSize;
   void *   rowData;
+  int32_t  rowSize;
+  int32_t  retCode; // for callback in sdb queue
+  int32_t  (*cb)(struct SMnodeMsg *pMsg, int32_t code);
+  struct SMnodeMsg *pMsg;
 } SSdbOper;
 
 typedef struct {

src/mnode/inc/mnodeUser.h

@@ -28,7 +28,8 @@ void *    mnodeGetNextUser(void *pIter, SUserObj **pUser);
 void      mnodeIncUserRef(SUserObj *pUser);
 void      mnodeDecUserRef(SUserObj *pUser);
 SUserObj *mnodeGetUserFromConn(void *pConn);
-int32_t   mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass);
+char *    mnodeGetUserFromMsg(void *pMnodeMsg);
+int32_t   mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg);
 void      mnodeDropAllUsers(SAcctObj *pAcct);
 
 #ifdef __cplusplus

src/mnode/inc/mnodeVgroup.h

@@ -34,7 +34,8 @@ void    mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
 
 void *  mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup);
 void    mnodeUpdateVgroup(SVgObj *pVgroup);
-void    mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *dnodeId, SVnodeLoad *pVload);
+void    mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload);
+void    mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes);
 
 int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg, SDbObj *pDb);
 void    mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);

src/mnode/src/mnodeAcct.c

@@ -25,6 +25,8 @@
 #include "mnodeSdb.h"
 #include "mnodeUser.h"
 
+#include "tglobal.h"
+
 void *  tsAcctSdb = NULL;
 static int32_t tsAcctUpdateSize;
 static int32_t mnodeCreateRootAcct();
@@ -39,6 +41,7 @@ static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
 static int32_t mnodeAcctActionInsert(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
   memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo));
+  pAcct->acctInfo.accessState = TSDB_VN_ALL_ACCCESS;
   pthread_mutex_init(&pAcct->mutex, NULL);
   return TSDB_CODE_SUCCESS;
 }
@@ -78,7 +81,9 @@ static int32_t mnodeAcctActionDecode(SSdbOper *pOper) {
 }
 
 static int32_t mnodeAcctActionRestored() {
-  if (dnodeIsFirstDeploy()) {
+  int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb);
+  if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
+    mPrint("dnode first deploy, create root acct");
     int32_t code = mnodeCreateRootAcct();
     if (code != TSDB_CODE_SUCCESS) {
       mError("failed to create root account, reason:%s", tstrerror(code));
@@ -171,8 +176,8 @@ static int32_t mnodeCreateRootAcct() {
 
   SAcctObj *pAcct = malloc(sizeof(SAcctObj));
   memset(pAcct, 0, sizeof(SAcctObj));
-  strcpy(pAcct->user, "root");
+  strcpy(pAcct->user, TSDB_DEFAULT_USER);
-  taosEncryptPass((uint8_t*)"taosdata", strlen("taosdata"), pAcct->pass);
+  taosEncryptPass((uint8_t *)TSDB_DEFAULT_PASS, strlen(TSDB_DEFAULT_PASS), pAcct->pass);
   pAcct->cfg = (SAcctCfg){
     .maxUsers           = 10,
     .maxDbs             = 64,
@@ -204,4 +209,4 @@ int32_t acctInit() { return TSDB_CODE_SUCCESS; }
 void    acctCleanUp() {}
 int32_t acctCheck(void *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
 
-#endif
+#endif

src/mnode/src/mnodeDb.c

@@ -41,7 +41,7 @@
 static void *  tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;
 
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg);
 static int32_t mnodeDropDb(SMnodeMsg *newMsg);
 static int32_t mnodeSetDbDropping(SDbObj *pDb);
 static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@@ -311,7 +311,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->replications < 0) pCfg->replications = tsReplications;
 }
 
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg) {
   int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
   if (code != 0) return code;
 
@@ -364,12 +364,15 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
     .table   = tsDbSdb,
     .pObj    = pDb,
     .rowSize = sizeof(SDbObj),
+    .pMsg    = pMsg
   };
 
   code = sdbInsertRow(&oper);
   if (code != TSDB_CODE_SUCCESS) {
     tfree(pDb);
-    code = TSDB_CODE_MND_SDB_ERROR;
+  } else {
+    mLPrint("db:%s, is created by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
@@ -475,7 +478,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
   cols++;
 
 #ifndef __CLOUD_VERSION__
-  if (strcmp(pUser->user, "root") == 0) {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
 #endif
     pShow->bytes[cols] = 4;
     pSchema[cols].type = TSDB_DATA_TYPE_INT;
@@ -487,7 +490,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
 #endif
 
 #ifndef __CLOUD_VERSION__
-  if (strcmp(pUser->user, "root") == 0) {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
 #endif
     pShow->bytes[cols] = 2;
     pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
@@ -511,7 +514,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
   cols++;
 
 #ifndef __CLOUD_VERSION__
-  if (strcmp(pUser->user, "root") == 0) {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
 #endif
     pShow->bytes[cols] = 4;
     pSchema[cols].type = TSDB_DATA_TYPE_INT;
@@ -613,7 +616,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     
     char* name = mnodeGetDbStr(pDb->name);
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN - 1);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -625,7 +628,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
     cols++;
 
 #ifndef __CLOUD_VERSION__
-    if (strcmp(pUser->user, "root") == 0) {
+    if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
 #endif
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int32_t *)pWrite = pDb->numOfVgroups;
@@ -635,7 +638,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
 #endif
 
 #ifndef __CLOUD_VERSION__
-    if (strcmp(pUser->user, "root") == 0) {
+    if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
 #endif
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int16_t *)pWrite = pDb->cfg.replications;
@@ -656,7 +659,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
     cols++;
 
 #ifndef __CLOUD_VERSION__
-    if (strcmp(pUser->user, "root") == 0) {
+    if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
 #endif
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int32_t *)pWrite = pDb->cfg.maxTables;  // table num can be created should minus 1
@@ -771,12 +774,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
   } else if (!pMsg->pUser->writeAuth) {
     code = TSDB_CODE_MND_NO_RIGHTS;
   } else {
-    code = mnodeCreateDb(pMsg->pUser->pAcct, pCreate);
+    code = mnodeCreateDb(pMsg->pUser->pAcct, pCreate, pMsg);
-    if (code == TSDB_CODE_SUCCESS) {
-      mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user);
-    } else {
-      mError("db:%s, failed to create, reason:%s", pCreate->db, tstrerror(code));
-    }
   }
 
   return code;
@@ -893,7 +891,31 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
   return newCfg;
 }
 
-static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
+static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
+  if (code != TSDB_CODE_SUCCESS) return code;
+  SDbObj *pDb = pMsg->pDb;
+
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup = NULL;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+    if (pVgroup->pDb == pDb) {
+      mnodeSendCreateVgroupMsg(pVgroup, NULL);
+    }
+    mnodeDecVgroupRef(pVgroup);
+  }
+  sdbFreeIter(pIter);
+
+  mTrace("db:%s, all vgroups is altered", pDb->name);
+  mLPrint("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+
+  balanceNotify();
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
   SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter);
   if (terrno != TSDB_CODE_SUCCESS) {
     return terrno;
@@ -904,38 +926,24 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     return code;
   }
 
-  int32_t oldReplica = pDb->cfg.replications;
-
   if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
     pDb->cfg = newCfg;
     pDb->cfgVersion++;
     SSdbOper oper = {
-      .type = SDB_OPER_GLOBAL,
+      .type  = SDB_OPER_GLOBAL,
       .table = tsDbSdb,
-      .pObj = pDb
+      .pObj  = pDb,
+      .pMsg  = pMsg,
+      .cb    = mnodeAlterDbCb
     };
 
-    int32_t code = sdbUpdateRow(&oper);
+    code = sdbUpdateRow(&oper);
-    if (code != TSDB_CODE_SUCCESS) {
+    if (code == TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_MND_SDB_ERROR;
+      if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
     }
   }
 
-  void *pIter = NULL;
+  return code;
-  while (1) {
-    SVgObj *pVgroup = NULL;
-    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
-    if (pVgroup == NULL) break;   
-    mnodeSendCreateVgroupMsg(pVgroup, NULL);
-    mnodeDecVgroupRef(pVgroup);
-  }
-  sdbFreeIter(pIter);
-
-  if (oldReplica != pDb->cfg.replications) {
-    balanceNotify();
-  }
-
-  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
@@ -948,28 +956,26 @@ static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
     return TSDB_CODE_MND_INVALID_DB;
   }
 
-  int32_t code = mnodeAlterDb(pMsg->pDb, pAlter);
+  return mnodeAlterDb(pMsg->pDb, pAlter, pMsg);
-  if (code != TSDB_CODE_SUCCESS) {
-    mError("db:%s, failed to alter, invalid db option", pAlter->db);
-    return code;
-  }
-
-  mTrace("db:%s, all vgroups is altered", pMsg->pDb->name);
-  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
+  if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
+  
   SDbObj *pDb = pMsg->pDb;
   mPrint("db:%s, drop db from sdb", pDb->name);
 
   SSdbOper oper = {
-    .type = SDB_OPER_GLOBAL,
+    .type  = SDB_OPER_GLOBAL,
     .table = tsDbSdb,
-    .pObj = pDb
+    .pObj  = pDb,
+    .pMsg  = pMsg
   };
+
   int32_t code = sdbDeleteRow(&oper);
-  if (code != 0) {
+  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_SDB_ERROR;
+    mLPrint("db:%s, is dropped by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;

src/mnode/src/mnodeDnode.c

@@ -44,7 +44,7 @@ static int32_t tsDnodeUpdateSize = 0;
 extern void *  tsMnodeSdb;
 extern void *  tsVgroupSdb;
 
-static int32_t mnodeCreateDnode(char *ep);
+static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg);
 static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg);
@@ -90,11 +90,12 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
 static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
   SDnodeObj *pSaved = mnodeGetDnode(pDnode->dnodeId);
-  if (pDnode != pSaved && pDnode != NULL && pSaved != NULL) {
+  if (pSaved != NULL && pDnode != pSaved) {
     memcpy(pSaved, pDnode, pOper->rowSize);
     free(pDnode);
+    mnodeDecDnodeRef(pSaved);
   }
-  mnodeDecDnodeRef(pSaved);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -117,10 +118,13 @@ static int32_t mnodeDnodeActionDecode(SSdbOper *pOper) {
 static int32_t mnodeDnodeActionRestored() {
   int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb);
   if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
-    mnodeCreateDnode(tsLocalEp);
+    mPrint("dnode first deploy, create dnode:%s", tsLocalEp);
+    mnodeCreateDnode(tsLocalEp, NULL);
     SDnodeObj *pDnode = mnodeGetDnodeByEp(tsLocalEp);
-    mnodeAddMnode(pDnode->dnodeId);
+    if (pDnode != NULL) {
-    mnodeDecDnodeRef(pDnode);
+      mnodeAddMnode(pDnode->dnodeId);
+      mnodeDecDnodeRef(pDnode);
+    }
   }
 
   return TSDB_CODE_SUCCESS;
@@ -250,7 +254,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
     // TODO temporary disabled for compiling: strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep); 
   }
 
-  if (strcmp(pMsg->pUser->user, "root") != 0) {
+  if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
     return TSDB_CODE_MND_NO_RIGHTS;
   }
 
@@ -277,6 +281,20 @@ static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
   mPrint("cfg dnode rsp is received");
 }
 
+static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
+  if (clusterCfg->numOfMnodes        != tsNumOfMnodes)        return false;
+  if (clusterCfg->mnodeEqualVnodeNum != tsMnodeEqualVnodeNum) return false;
+  if (clusterCfg->offlineThreshold   != tsOfflineThreshold)   return false;
+  if (clusterCfg->statusInterval     != tsStatusInterval)     return false;
+
+  if (0 != strncasecmp(clusterCfg->arbitrator, tsArbitrator, strlen(tsArbitrator))) return false;
+  if (0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone)))       return false;
+  if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale)))              return false;
+  if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset)))           return false;
+    
+  return true;
+}
+
 static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont;
   pStatus->dnodeId      = htonl(pStatus->dnodeId);
@@ -312,7 +330,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pDnode->alternativeRole  = pStatus->alternativeRole;
   pDnode->totalVnodes      = pStatus->numOfTotalVnodes; 
   pDnode->moduleStatus     = pStatus->moduleStatus;
-  pDnode->lastAccess       = tsAccessSquence;
   
   if (pStatus->dnodeId == 0) {
     mTrace("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp);
@@ -321,6 +338,19 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   }
  
   int32_t openVnodes = htons(pStatus->openVnodes);
+  int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess);
+  SDMStatusRsp *pRsp = rpcMallocCont(contLen);
+  if (pRsp == NULL) {
+    mnodeDecDnodeRef(pDnode);
+    return TSDB_CODE_MND_OUT_OF_MEMORY;
+  }
+
+  pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
+  pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
+  pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
+  mnodeGetMnodeInfos(&pRsp->mnodes);
+  SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
+
   for (int32_t j = 0; j < openVnodes; ++j) {
     SVnodeLoad *pVload = &pStatus->load[j];
     pVload->vgId = htonl(pVload->vgId);
@@ -333,42 +363,42 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
       mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
     } else {
       mnodeUpdateVgroupStatus(pVgroup, pDnode, pVload);
+      pAccess->vgId = htonl(pVload->vgId);
+      pAccess->accessState = pVgroup->accessState;
       mnodeDecVgroupRef(pVgroup);
     }
   }
 
   if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+    // Verify whether the cluster parameters are consistent when status change from offline to ready
+    bool ret = mnodeCheckClusterCfgPara(&(pStatus->clusterCfg));
+    if (false == ret) {
+      mnodeDecDnodeRef(pDnode);
+      rpcFreeCont(pRsp);
+      mError("dnode %s cluster cfg parameters inconsistent", pStatus->dnodeEp);
+      return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
+    }
+    
     mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
     pDnode->status = TAOS_DN_STATUS_READY;
     balanceUpdateMnode();
     balanceNotify();
   }
 
-  mnodeDecDnodeRef(pDnode);
+  if (openVnodes != pDnode->openVnodes) {
-
+    mnodeCheckUnCreatedVgroup(pDnode, pStatus->load, openVnodes);
-  int32_t contLen = sizeof(SDMStatusRsp) + TSDB_MAX_VNODES * sizeof(SDMVgroupAccess);
-  SDMStatusRsp *pRsp = rpcMallocCont(contLen);
-  if (pRsp == NULL) {
-    return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
-  mnodeGetMnodeInfos(&pRsp->mnodes);
+  pDnode->lastAccess = tsAccessSquence;
+  mnodeDecDnodeRef(pDnode);
 
-  pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
-  pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
-  pRsp->dnodeCfg.numOfVnodes = 0;
-  
-  contLen = sizeof(SDMStatusRsp);
-
-  //TODO: set vnode access
-  
   pMsg->rpcRsp.len = contLen;
   pMsg->rpcRsp.rsp =  pRsp;
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeCreateDnode(char *ep) {
+static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
   int32_t grantCode = grantCheck(TSDB_GRANT_DNODE);
   if (grantCode != TSDB_CODE_SUCCESS) {
     return grantCode;
@@ -392,7 +422,8 @@ static int32_t mnodeCreateDnode(char *ep) {
     .type = SDB_OPER_GLOBAL,
     .table = tsDnodeSdb,
     .pObj = pDnode,
-    .rowSize = sizeof(SDnodeObj)
+    .rowSize = sizeof(SDnodeObj),
+    .pMsg = pMsg
   };
 
   int32_t code = sdbInsertRow(&oper);
@@ -400,30 +431,32 @@ static int32_t mnodeCreateDnode(char *ep) {
     int dnodeId = pDnode->dnodeId;
     tfree(pDnode);
     mError("failed to create dnode:%d, result:%s", dnodeId, tstrerror(code));
-    return TSDB_CODE_MND_SDB_ERROR;
+  } else {
+    mPrint("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code));
+    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
-  mPrint("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code));
   return code;
 }
 
-int32_t mnodeDropDnode(SDnodeObj *pDnode) {
+int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsDnodeSdb,
-    .pObj = pDnode
+    .pObj = pDnode,
+    .pMsg = pMsg
   };
 
-  int32_t code = sdbDeleteRow(&oper); 
+  int32_t code = sdbDeleteRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_SDB_ERROR;
+    mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
+    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
-  mLPrint("dnode:%d, is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
   return code;
 }
 
-static int32_t mnodeDropDnodeByEp(char *ep) {
+static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
   SDnodeObj *pDnode = mnodeGetDnodeByEp(ep);
   if (pDnode == NULL) {
     mError("dnode:%s, is not exist", ep);
@@ -438,7 +471,7 @@ static int32_t mnodeDropDnodeByEp(char *ep) {
 
   mPrint("dnode:%d, start to drop it", pDnode->dnodeId);
 #ifndef _SYNC
-  return mnodeDropDnode(pDnode);
+  return mnodeDropDnode(pDnode, pMsg);
 #else
   return balanceDropDnode(pDnode);
 #endif
@@ -447,38 +480,20 @@ static int32_t mnodeDropDnodeByEp(char *ep) {
 static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
   SCMCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
 
-  if (strcmp(pMsg->pUser->user, "root") != 0) {
+  if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
     return TSDB_CODE_MND_NO_RIGHTS;
   } else {
-    int32_t code = mnodeCreateDnode(pCreate->ep);
+    return mnodeCreateDnode(pCreate->ep, pMsg);
-
-    if (code == TSDB_CODE_SUCCESS) {
-      SDnodeObj *pDnode = mnodeGetDnodeByEp(pCreate->ep);
-      mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user);
-      mnodeDecDnodeRef(pDnode);
-    } else {
-      mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(code));
-    }
-
-    return code;
   }
 }
 
 static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) {
   SCMDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;
 
-  if (strcmp(pMsg->pUser->user, "root") != 0) {
+  if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
     return TSDB_CODE_MND_NO_RIGHTS;
   } else {
-    int32_t code = mnodeDropDnodeByEp(pDrop->ep);
+    return mnodeDropDnodeByEp(pDrop->ep, pMsg);
-
-    if (code == TSDB_CODE_SUCCESS) {
-      mLPrint("dnode:%s is dropped by %s", pDrop->ep, pMsg->pUser->user);
-    } else {
-      mError("failed to drop dnode:%s, reason:%s", pDrop->ep, tstrerror(code));
-    }
-
-    return code;
   }
 }
 
@@ -486,7 +501,7 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
-  if (strcmp(pUser->pAcct->user, "root") != 0) {
+  if (strcmp(pUser->pAcct->user, TSDB_DEFAULT_USER) != 0) {
     mnodeDecUserRef(pUser);
     return TSDB_CODE_MND_NO_RIGHTS;
   }
@@ -570,7 +585,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols]);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -615,7 +630,7 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
-  if (strcmp(pUser->user, "root") != 0)  {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0)  {
     mnodeDecUserRef(pUser);
     return TSDB_CODE_MND_NO_RIGHTS;
   }
@@ -725,7 +740,7 @@ static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
-  if (strcmp(pUser->user, "root") != 0)  {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0)  {
     mnodeDecUserRef(pUser);
     return TSDB_CODE_MND_NO_RIGHTS;
   }
@@ -812,7 +827,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
   
-  if (strcmp(pUser->user, "root") != 0)  {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0)  {
     mnodeDecUserRef(pUser);
     return TSDB_CODE_MND_NO_RIGHTS;
   }

src/mnode/src/mnodeMnode.c

@@ -31,6 +31,8 @@
 #include "mnodeShow.h"
 #include "mnodeUser.h"
 
+#include "tglobal.h"
+
 static void *        tsMnodeSdb = NULL;
 static int32_t       tsMnodeUpdateSize = 0;
 static SRpcIpSet     tsMnodeIpSetForShell;
@@ -333,7 +335,7 @@ static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
-  if (strcmp(pUser->pAcct->user, "root") != 0)  {
+  if (strcmp(pUser->pAcct->user, TSDB_DEFAULT_USER) != 0)  {
     mnodeDecUserRef(pUser);
     return TSDB_CODE_MND_NO_RIGHTS;
   }
@@ -401,9 +403,9 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
     
     SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
     if (pDnode != NULL) {
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols]);
     } else {
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols]);
     }
     mnodeDecDnodeRef(pDnode);
 
@@ -411,7 +413,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     char* roles = mnodeGetMnodeRoleStr(pMnode->role);
-    STR_TO_VARSTR(pWrite, roles);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, roles, pShow->bytes[cols]);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

src/mnode/src/mnodeProfile.c

@@ -39,6 +39,7 @@
 #define CONN_KEEP_TIME  (tsShellActivityTimer * 3)
 #define CONN_CHECK_TIME (tsShellActivityTimer * 2)
 #define QUERY_ID_SIZE   20
+#define QUERY_STREAM_SAVE_SIZE 20
 
 extern void *tsMnodeTmr;
 static SCacheObj *tsMnodeConnCache = NULL;
@@ -108,8 +109,8 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) {
 }
 
 void mnodeReleaseConn(SConnObj *pConn) {
-  if(pConn == NULL) return;
+  if (pConn == NULL) return;
-  taosCacheRelease(tsMnodeConnCache, (void**)&pConn, false);
+  taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false);
 }
 
 SConnObj *mnodeAccquireConn(uint32_t connId, char *user, uint32_t ip, uint16_t port) {
@@ -138,7 +139,7 @@ SConnObj *mnodeAccquireConn(uint32_t connId, char *user, uint32_t ip, uint16_t p
 static void mnodeFreeConn(void *data) {
   SConnObj *pConn = data;
   tfree(pConn->pQueries);
-  tfree(pConn->pQueries);
+  tfree(pConn->pStreams);
 
   mTrace("connId:%d, is destroyed", pConn->connId);
 }
@@ -168,7 +169,7 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
 static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
-  if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
   
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
@@ -235,14 +236,12 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    size_t size = sizeof(pConnObj->user);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]);
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, size);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
-    size = sizeof(ipStr);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, size);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -266,16 +265,27 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
 // not thread safe, need optimized
 int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) {
   pConn->numOfQueries = htonl(pHBMsg->numOfQueries);
-  if (pConn->numOfQueries > 0 && pConn->numOfQueries < 20) {
+  if (pConn->numOfQueries > 0) {
-    pConn->pQueries = calloc(sizeof(SQueryDesc), pConn->numOfQueries);
+    if (pConn->pQueries == NULL) {
-    memcpy(pConn->pQueries, pHBMsg->pData, pConn->numOfQueries * sizeof(SQueryDesc));
+      pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE);
+    }
+
+    int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfQueries) * sizeof(SQueryDesc);
+    if (saveSize > 0 && pConn->pQueries != NULL) {
+      memcpy(pConn->pQueries, pHBMsg->pData, saveSize);
+    }
   }
 
   pConn->numOfStreams = htonl(pHBMsg->numOfStreams);
-  if (pConn->numOfStreams > 0 && pConn->numOfStreams < 20) {
+  if (pConn->numOfStreams > 0) {
-    pConn->pStreams = calloc(sizeof(SStreamDesc), pConn->numOfStreams);
+    if (pConn->pStreams == NULL) {
-    memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc),
+      pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE);
-           pConn->numOfStreams * sizeof(SStreamDesc));
+    }
+
+    int32_t saveSize = MIN(QUERY_STREAM_SAVE_SIZE, pConn->numOfStreams) * sizeof(SStreamDesc);
+    if (saveSize > 0 && pConn->pStreams != NULL) {
+      memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc), saveSize);
+    }
   }
 
   return TSDB_CODE_SUCCESS;
@@ -284,7 +294,7 @@ int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) {
 static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
-  if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
 
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
@@ -356,18 +366,16 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
 
       snprintf(ipStr, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->queryId));
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, QUERY_ID_SIZE);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      size_t size = sizeof(pConnObj->user);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]);
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, size);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
-      size = sizeof(ipStr);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, size);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -379,7 +387,7 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, TSDB_SHOW_SQL_LEN);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]);
       cols++;
 
       numOfRows++;
@@ -395,7 +403,7 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
 static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
-  if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
 
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
@@ -479,18 +487,16 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
 
       snprintf(ipStr, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->streamId));
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, QUERY_ID_SIZE);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      size_t size = sizeof(pConnObj->user);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]);
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, size);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
-      size = sizeof(ipStr);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, size);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -506,7 +512,7 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, TSDB_SHOW_SQL_LEN);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -525,7 +531,7 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
 
 static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) {
   SUserObj *pUser = pMsg->pUser;
-  if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
 
   SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
   mPrint("kill query msg is received, queryId:%s", pKill->queryId);
@@ -555,7 +561,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) {
 
 static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) {
   SUserObj *pUser = pMsg->pUser;
-  if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
 
   SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
   mPrint("kill stream msg is received, streamId:%s", pKill->queryId);
@@ -585,7 +591,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) {
 
 static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) {
   SUserObj *pUser = pMsg->pUser;
-  if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_MND_NO_RIGHTS;
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
 
   SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont;
   SConnObj *      pConn = taosCacheAcquireByName(tsMnodeConnCache, pKill->queryId);

src/mnode/src/mnodeSdb.c

@@ -24,6 +24,7 @@
 #include "tsync.h"
 #include "tglobal.h"
 #include "dnode.h"
+#include "mnode.h"
 #include "mnodeDef.h"
 #include "mnodeInt.h"
 #include "mnodeMnode.h"
@@ -31,6 +32,7 @@
 #include "mnodeSdb.h"
 
 #define SDB_TABLE_LEN 12
+#define SDB_SYNC_HACK 16
 
 typedef enum {
   SDB_ACTION_INSERT,
@@ -83,8 +85,29 @@ typedef struct {
   void *  row;
 } SSdbRow;
 
+typedef struct {
+  pthread_t thread;
+  int32_t   workerId;
+} SSdbWriteWorker;
+
+typedef struct {
+  int32_t num;
+  SSdbWriteWorker *writeWorker;
+} SSdbWriteWorkerPool;
+
 static SSdbObject tsSdbObj = {0};
-static int sdbWrite(void *param, void *data, int type);
+static taos_qset  tsSdbWriteQset;
+static taos_qall  tsSdbWriteQall;
+static taos_queue tsSdbWriteQueue;
+static SSdbWriteWorkerPool tsSdbPool;
+
+static int     sdbWrite(void *param, void *data, int type);
+static int     sdbWriteToQueue(void *param, void *data, int type);
+static void *  sdbWorkerFp(void *param);
+static int32_t sdbInitWriteWorker();
+static void    sdbCleanupWriteWorker();
+static int32_t sdbAllocWriteQueue();
+static void    sdbFreeWritequeue();
 
 int32_t sdbGetId(void *handle) {
   return ((SSdbTable *)handle)->autoIndex;
@@ -302,7 +325,7 @@ void sdbUpdateSync() {
   syncInfo.ahandle = NULL;
   syncInfo.getWalInfo = sdbGetWalInfo;
   syncInfo.getFileInfo = sdbGetFileInfo;
-  syncInfo.writeToCache = sdbWrite;
+  syncInfo.writeToCache = sdbWriteToQueue;
   syncInfo.confirmForward = sdbConfirmForward; 
   syncInfo.notifyRole = sdbNotifyRole;
   tsSdbObj.cfg = syncCfg;
@@ -319,10 +342,14 @@ int32_t sdbInit() {
   pthread_mutex_init(&tsSdbObj.mutex, NULL);
   sem_init(&tsSdbObj.sem, 0, 0);
 
+  if (sdbInitWriteWorker() != 0) {
+    return -1;
+  }
+
   if (sdbInitWal() != 0) {
     return -1;
   }
-  
+
   sdbRestoreTables();
 
   if (mnodeGetMnodesNum() == 1) {
@@ -340,6 +367,8 @@ void sdbCleanUp() {
 
   tsSdbObj.status = SDB_STATUS_CLOSING;
   
+  sdbCleanupWriteWorker();
+
   if (tsSdbObj.sync) {
     syncStop(tsSdbObj.sync);
     tsSdbObj.sync = NULL;
@@ -475,7 +504,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
   pTable->numOfRows--;
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 "version:%" PRIu64, pTable->tableName,
+  sdbTrace("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName,
            sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion());
 
   int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
@@ -494,9 +523,10 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
 }
 
 static int sdbWrite(void *param, void *data, int type) {
+  SSdbOper *pOper = param;
   SWalHead *pHead = data;
-  int32_t   tableId = pHead->msgType / 10;
+  int32_t tableId = pHead->msgType / 10;
-  int32_t   action = pHead->msgType % 10;
+  int32_t action = pHead->msgType % 10;
 
   SSdbTable *pTable = sdbGetTableFromId(tableId);
   assert(pTable != NULL);
@@ -531,21 +561,22 @@ static int sdbWrite(void *param, void *data, int type) {
     pthread_mutex_unlock(&tsSdbObj.mutex);
     return code;
   }
-  walFsync(tsSdbObj.wal);
+  
-
   code = sdbForwardToPeer(pHead);
   pthread_mutex_unlock(&tsSdbObj.mutex);
 
   // from app, oper is created
-  if (param != NULL) {
+  if (pOper != NULL) {
-    //sdbTrace("request from app is disposed, version:%" PRIu64 " code:%s", pHead->version, tstrerror(code));
+    sdbTrace("record from app is disposed, version:%" PRIu64 " result:%s", pHead->version, tstrerror(code));
     return code;
   }
   
   // from wal or forward msg, oper not created, should add into hash
   if (tsSdbObj.sync != NULL) {
-    sdbTrace("forward request is received, version:%" PRIu64 " result:%s, confirm it", pHead->version, tstrerror(code));
+    sdbTrace("record from wal forward is disposed, version:%" PRIu64 " confirm it", pHead->version);
     syncConfirmForward(tsSdbObj.sync, pHead->version, code);
+  } else {
+    sdbTrace("record from wal restore is disposed, version:%" PRIu64 , pHead->version);
   }
 
   if (action == SDB_ACTION_INSERT) {
@@ -568,7 +599,7 @@ static int sdbWrite(void *param, void *data, int type) {
 
 int32_t sdbInsertRow(SSdbOper *pOper) {
   SSdbTable *pTable = (SSdbTable *)pOper->table;
-  if (pTable == NULL) return -1;
+  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
 
   if (sdbGetRowFromObj(pTable, pOper->pObj)) {
     sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj));
@@ -587,98 +618,146 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
     pthread_mutex_unlock(&pTable->mutex);
   }
 
-  if (pOper->type == SDB_OPER_GLOBAL) {
+  int32_t code = sdbInsertHash(pTable, pOper);
-    int32_t   size = sizeof(SWalHead) + pTable->maxRowSize;
+  if (code != TSDB_CODE_SUCCESS) {
-    SWalHead *pHead = taosAllocateQitem(size);
+    sdbError("table:%s, failed to insert into hash", pTable->tableName);
-    pHead->version = 0;
+    return code;
-    pHead->len = pOper->rowSize;
-    pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT;
-
-    pOper->rowData = pHead->cont;
-    (*pTable->encodeFp)(pOper);
-    pHead->len = pOper->rowSize;
-
-    int32_t code = sdbWrite(pOper, pHead, pHead->msgType);
-    taosFreeQitem(pHead);
-    if (code < 0) return code;
   }
 
-  return sdbInsertHash(pTable, pOper);
+  // just insert data into memory
+  if (pOper->type != SDB_OPER_GLOBAL) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
+  SSdbOper *pNewOper = taosAllocateQitem(size);
+  
+  SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+  pHead->version = 0;
+  pHead->len = pOper->rowSize;
+  pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT;
+
+  pOper->rowData = pHead->cont;
+  (*pTable->encodeFp)(pOper);
+  pHead->len = pOper->rowSize;
+
+  memcpy(pNewOper, pOper, sizeof(SSdbOper));
+
+  if (pNewOper->pMsg != NULL) {
+    sdbTrace("app:%p:%p, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg);
+  }
+
+  taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t sdbDeleteRow(SSdbOper *pOper) {
   SSdbTable *pTable = (SSdbTable *)pOper->table;
-  if (pTable == NULL) return -1;
+  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
 
   SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj);
   if (pMeta == NULL) {
     sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
-    return -1;
+    return TSDB_CODE_MND_SDB_OBJ_NOT_THERE;
   }
 
-  void * pMetaRow = pMeta->row;
+  void *pMetaRow = pMeta->row;
-  assert(pMetaRow != NULL);
+  if (pMetaRow == NULL) {
-
+    sdbError("table:%s, record meta is null", pTable->tableName);
-  if (pOper->type == SDB_OPER_GLOBAL) {
+    return TSDB_CODE_MND_SDB_INVAID_META_ROW;
-    void *  key = sdbGetObjKey(pTable, pOper->pObj);
-    int32_t keySize = 0;
-    switch (pTable->keyType) {
-      case SDB_KEY_STRING:
-      case SDB_KEY_VAR_STRING:
-        keySize = strlen((char *)key) + 1;
-        break;
-      case SDB_KEY_INT:
-      case SDB_KEY_AUTO:
-        keySize = sizeof(uint32_t);
-        break;
-      default:
-        return -1;
-    }
-
-    int32_t   size = sizeof(SWalHead) + keySize;
-    SWalHead *pHead = taosAllocateQitem(size);
-    pHead->version = 0;
-    pHead->len = keySize;
-    pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
-    memcpy(pHead->cont, key, keySize);
-
-    int32_t code = sdbWrite(pOper, pHead, pHead->msgType);
-    taosFreeQitem(pHead);
-    if (code < 0) return code;
   }
 
-  return sdbDeleteHash(pTable, pOper);
+  int32_t code = sdbDeleteHash(pTable, pOper);
+  if (code != TSDB_CODE_SUCCESS) {
+    sdbError("table:%s, failed to delete from hash", pTable->tableName);
+    return code;
+  }
+
+  // just delete data from memory
+  if (pOper->type != SDB_OPER_GLOBAL) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  void *  key = sdbGetObjKey(pTable, pOper->pObj);
+  int32_t keySize = 0;
+  switch (pTable->keyType) {
+    case SDB_KEY_STRING:
+    case SDB_KEY_VAR_STRING:
+      keySize = strlen((char *)key) + 1;
+      break;
+    case SDB_KEY_INT:
+    case SDB_KEY_AUTO:
+      keySize = sizeof(uint32_t);
+      break;
+    default:
+      return TSDB_CODE_MND_SDB_INVAID_KEY_TYPE;
+  }
+
+  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + keySize + SDB_SYNC_HACK;
+  SSdbOper *pNewOper = taosAllocateQitem(size);
+
+  SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+  pHead->version = 0;
+  pHead->len = keySize;
+  pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
+  memcpy(pHead->cont, key, keySize);
+
+  memcpy(pNewOper, pOper, sizeof(SSdbOper));
+
+  if (pNewOper->pMsg != NULL) {
+    sdbTrace("app:%p:%p, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg);
+  }
+
+  taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t sdbUpdateRow(SSdbOper *pOper) {
   SSdbTable *pTable = (SSdbTable *)pOper->table;
-  if (pTable == NULL) return -1;
+  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
 
   SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj);
   if (pMeta == NULL) {
-    sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
+    sdbTrace("table:%s, record is not there, update failed", pTable->tableName);
-    return -1;
+    return TSDB_CODE_MND_SDB_OBJ_NOT_THERE;
   }
 
-  void * pMetaRow = pMeta->row;
+  void *pMetaRow = pMeta->row;
-  assert(pMetaRow != NULL);
+  if (pMetaRow == NULL) {
+    sdbError("table:%s, record meta is null", pTable->tableName);
+    return TSDB_CODE_MND_SDB_INVAID_META_ROW;
+  }
 
-  if (pOper->type == SDB_OPER_GLOBAL) {
+  int32_t code = sdbUpdateHash(pTable, pOper);
-    int32_t   size = sizeof(SWalHead) + pTable->maxRowSize;
+  if (code != TSDB_CODE_SUCCESS) {
-    SWalHead *pHead = taosAllocateQitem(size);
+    sdbError("table:%s, failed to update hash", pTable->tableName);
-    pHead->version = 0;
+    return code;
-    pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE;
+  }
 
-    pOper->rowData = pHead->cont;
+  // just update data in memory
-    (*pTable->encodeFp)(pOper);
+  if (pOper->type != SDB_OPER_GLOBAL) {
-    pHead->len = pOper->rowSize;
+    return TSDB_CODE_SUCCESS;
+  }
 
-    int32_t code = sdbWrite(pOper, pHead, pHead->msgType);
+  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
-    taosFreeQitem(pHead);
+  SSdbOper *pNewOper = taosAllocateQitem(size);
-    if (code < 0) return code;
+
-  } 
+  SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
-  
+  pHead->version = 0;
-  return sdbUpdateHash(pTable, pOper);
+  pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE;
+
+  pOper->rowData = pHead->cont;
+  (*pTable->encodeFp)(pOper);
+  pHead->len = pOper->rowSize;
+
+  memcpy(pNewOper, pOper, sizeof(SSdbOper));
+
+  if (pNewOper->pMsg != NULL) {
+    sdbTrace("app:%p:%p, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle, pNewOper->pMsg);
+  }
+
+  taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
+  return TSDB_CODE_SUCCESS;
 }
 
 void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
@@ -775,3 +854,158 @@ void sdbCloseTable(void *handle) {
   free(pTable);
 }
 
+int32_t sdbInitWriteWorker() {
+  tsSdbPool.num = 1;
+  tsSdbPool.writeWorker = (SSdbWriteWorker *)calloc(sizeof(SSdbWriteWorker), tsSdbPool.num);
+
+  if (tsSdbPool.writeWorker == NULL) return -1;
+  for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    pWorker->workerId = i;
+  }
+
+  sdbAllocWriteQueue();
+  
+  mPrint("sdb write is opened");
+  return 0;
+}
+
+void sdbCleanupWriteWorker() {
+  for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    if (pWorker->thread) {
+      taosQsetThreadResume(tsSdbWriteQset);
+    }
+  }
+
+  for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    if (pWorker->thread) {
+      pthread_join(pWorker->thread, NULL);
+    }
+  }
+
+  sdbFreeWritequeue();
+
+  mPrint("sdb write is closed");
+}
+
+int32_t sdbAllocWriteQueue() {
+  tsSdbWriteQueue = taosOpenQueue();
+  if (tsSdbWriteQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
+
+  tsSdbWriteQset = taosOpenQset();
+  if (tsSdbWriteQset == NULL) {
+    taosCloseQueue(tsSdbWriteQueue);
+    return TSDB_CODE_MND_OUT_OF_MEMORY;
+  }
+  taosAddIntoQset(tsSdbWriteQset, tsSdbWriteQueue, NULL);
+
+  tsSdbWriteQall = taosAllocateQall();
+  if (tsSdbWriteQall == NULL) {
+    taosCloseQset(tsSdbWriteQset);
+    taosCloseQueue(tsSdbWriteQueue);
+    return TSDB_CODE_MND_OUT_OF_MEMORY;
+  }
+  
+  for (int32_t i = 0; i < tsSdbPool.num; ++i) {
+    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    pWorker->workerId = i;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&pWorker->thread, &thAttr, sdbWorkerFp, pWorker) != 0) {
+      mError("failed to create thread to process sdb write queue, reason:%s", strerror(errno));
+      taosFreeQall(tsSdbWriteQall);
+      taosCloseQset(tsSdbWriteQset);
+      taosCloseQueue(tsSdbWriteQueue);
+      return TSDB_CODE_MND_OUT_OF_MEMORY;
+    }
+
+    pthread_attr_destroy(&thAttr);
+    mTrace("sdb write worker:%d is launched, total:%d", pWorker->workerId, tsSdbPool.num);
+  }
+
+  mTrace("sdb write queue:%p is allocated", tsSdbWriteQueue);
+  return TSDB_CODE_SUCCESS;
+}
+
+void sdbFreeWritequeue() {
+  taosCloseQset(tsSdbWriteQueue);
+  taosFreeQall(tsSdbWriteQall);
+  taosCloseQset(tsSdbWriteQset);
+  tsSdbWriteQall = NULL;
+  tsSdbWriteQset = NULL;
+  tsSdbWriteQueue = NULL;
+}
+
+int sdbWriteToQueue(void *param, void *data, int type) {
+  SWalHead *pHead = data;
+  int size = sizeof(SWalHead) + pHead->len;
+  SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
+  memcpy(pWal, pHead, size);
+
+  taosWriteQitem(tsSdbWriteQueue, type, pWal);
+  return 0;
+}
+
+static void *sdbWorkerFp(void *param) {
+  SWalHead *pHead;
+  SSdbOper *pOper;
+  int32_t   type;
+  int32_t   numOfMsgs;
+  void *    item;
+  void *    unUsed;
+
+  while (1) {
+    numOfMsgs = taosReadAllQitemsFromQset(tsSdbWriteQset, tsSdbWriteQall, &unUsed);
+    if (numOfMsgs == 0) {
+      sdbTrace("sdbWorkerFp: got no message from qset, exiting...");
+      break;
+    }
+
+    for (int32_t i = 0; i < numOfMsgs; ++i) {
+      taosGetQitem(tsSdbWriteQall, &type, &item);
+      if (type == TAOS_QTYPE_RPC) {
+        pOper = (SSdbOper *)item;
+        pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
+      } else {
+        pHead = (SWalHead *)item;
+        pOper = NULL;
+      }
+
+      if (pOper != NULL && pOper->pMsg != NULL) {
+        sdbTrace("app:%p:%p, will be processed in sdb queue", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg);
+      }
+
+      int32_t code = sdbWrite(pOper, pHead, type);
+      if (pOper) pOper->retCode = code;
+    }
+
+    walFsync(tsSdbObj.wal);
+
+    // browse all items, and process them one by one
+    taosResetQitems(tsSdbWriteQall);
+    for (int32_t i = 0; i < numOfMsgs; ++i) {
+      taosGetQitem(tsSdbWriteQall, &type, &item);
+      if (type == TAOS_QTYPE_RPC) {
+        pOper = (SSdbOper *)item;
+        if (pOper != NULL && pOper->cb != NULL) {
+          pOper->retCode = (*pOper->cb)(pOper->pMsg, pOper->retCode);
+        }
+          
+        if (pOper != NULL && pOper->pMsg != NULL) {
+          sdbTrace("app:%p:%p, msg is processed, result:%s", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg,
+                   tstrerror(pOper->retCode));
+        }
+
+        dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode);
+      }
+      taosFreeQitem(item);
+    }
+  }
+
+  return NULL;
+}

src/mnode/src/mnodeShow.c

@@ -122,7 +122,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
   SShowObj *pShow = (SShowObj *) calloc(1, showObjSize);
   pShow->type       = pShowMsg->type;
   pShow->payloadLen = htons(pShowMsg->payloadLen);
-  strcpy(pShow->db, pShowMsg->db);
+  tstrncpy(pShow->db, pShowMsg->db, TSDB_DB_NAME_LEN);
   memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
 
   pShow = mnodePutShowObj(pShow, showObjSize);

src/mnode/src/mnodeUser.c

@@ -102,11 +102,13 @@ static int32_t mnodeUserActionDecode(SSdbOper *pOper) {
 }
 
 static int32_t mnodeUserActionRestored() {
-  if (dnodeIsFirstDeploy()) {
+  int32_t numOfRows = sdbGetNumOfRows(tsUserSdb);
-    SAcctObj *pAcct = mnodeGetAcct("root");
+  if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
-    mnodeCreateUser(pAcct, "root", "taosdata");
+    mPrint("dnode first deploy, create root user");
-    mnodeCreateUser(pAcct, "monitor", tsInternalPass);
+    SAcctObj *pAcct = mnodeGetAcct(TSDB_DEFAULT_USER);
-    mnodeCreateUser(pAcct, "_root", tsInternalPass);
+    mnodeCreateUser(pAcct, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS, NULL);
+    mnodeCreateUser(pAcct, "monitor", tsInternalPass, NULL);
+    mnodeCreateUser(pAcct, "_"TSDB_DEFAULT_USER, tsInternalPass, NULL);
     mnodeDecAcctRef(pAcct);
   }
 
@@ -170,22 +172,24 @@ void mnodeDecUserRef(SUserObj *pUser) {
   return sdbDecRef(tsUserSdb, pUser); 
 }
 
-static int32_t mnodeUpdateUser(SUserObj *pUser) {
+static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) {
   SSdbOper oper = {
-    .type = SDB_OPER_GLOBAL,
+    .type  = SDB_OPER_GLOBAL,
     .table = tsUserSdb,
-    .pObj = pUser
+    .pObj  = pUser,
+    .pMsg  = pMsg
   };
 
   int32_t code = sdbUpdateRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_SDB_ERROR;
+    mLPrint("user:%s, is altered by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
+    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
 }
 
-int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass) {
+int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
   int32_t code = acctCheck(pAcct, ACCT_GRANT_USER);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
@@ -212,42 +216,47 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass) {
   }
 
   pUser = calloc(1, sizeof(SUserObj));
-  strcpy(pUser->user, name);
+  tstrncpy(pUser->user, name, TSDB_USER_LEN);
   taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass);
   strcpy(pUser->acct, pAcct->user);
   pUser->createdTime = taosGetTimestampMs();
   pUser->superAuth = 0;
   pUser->writeAuth = 1;
-  if (strcmp(pUser->user, "root") == 0 || strcmp(pUser->user, pUser->acct) == 0) {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0 || strcmp(pUser->user, pUser->acct) == 0) {
     pUser->superAuth = 1;
   }
 
   SSdbOper oper = {
-    .type = SDB_OPER_GLOBAL,
+    .type    = SDB_OPER_GLOBAL,
-    .table = tsUserSdb,
+    .table   = tsUserSdb,
-    .pObj = pUser,
+    .pObj    = pUser,
-    .rowSize = sizeof(SUserObj)
+    .rowSize = sizeof(SUserObj),
+    .pMsg    = pMsg
   };
 
   code = sdbInsertRow(&oper);
   if (code != TSDB_CODE_SUCCESS) {
     tfree(pUser);
-    code = TSDB_CODE_MND_SDB_ERROR;
+  } else {
+    mLPrint("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
+    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
 }
 
-static int32_t mnodeDropUser(SUserObj *pUser) {
+static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) {
   SSdbOper oper = {
-    .type = SDB_OPER_GLOBAL,
+    .type  = SDB_OPER_GLOBAL,
     .table = tsUserSdb,
-    .pObj = pUser
+    .pObj  = pUser,
+    .pMsg  = pMsg
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code == TSDB_CODE_SUCCESS) {
-    code = TSDB_CODE_MND_SDB_ERROR;
+    mLPrint("user:%s, is dropped by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
+    if (pMsg != NULL) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
   return code;
@@ -315,8 +324,7 @@ static int32_t mnodeRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, voi
     cols = 0;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    size_t size = sizeof(pUser->user);
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->user, pShow->bytes[cols]);
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->user, size);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -337,7 +345,7 @@ static int32_t mnodeRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, voi
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->acct, sizeof(pUser->user));
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->acct, pShow->bytes[cols]);
     cols++;
 
     numOfRows++;
@@ -358,22 +366,25 @@ SUserObj *mnodeGetUserFromConn(void *pConn) {
   }
 }
 
+char *mnodeGetUserFromMsg(void *pMsg) {
+  SMnodeMsg *pMnodeMsg = pMsg;
+  if (pMnodeMsg != NULL && pMnodeMsg->pUser != NULL) {
+    return pMnodeMsg->pUser->user;
+  } else {
+    return "system";
+  }
+}
+
 static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) {
-  int32_t code;
   SUserObj *pOperUser = pMsg->pUser;
   
   if (pOperUser->superAuth) {
     SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
-    code = mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass);
+    return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg);
-    if (code == TSDB_CODE_SUCCESS) {
-      mLPrint("user:%s, is created by %s", pCreate->user, pOperUser->user);
-    }
   } else {
     mError("user:%s, no rights to create user", pOperUser->user);
-    code = TSDB_CODE_MND_NO_RIGHTS;
+    return TSDB_CODE_MND_NO_RIGHTS;
   }
-
-  return code;
 }
 
 static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
@@ -393,12 +404,12 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
 
   if ((pAlter->flag & TSDB_ALTER_USER_PASSWD) != 0) {
     bool hasRight = false;
-    if (strcmp(pOperUser->user, "root") == 0) {
+    if (strcmp(pOperUser->user, TSDB_DEFAULT_USER) == 0) {
       hasRight = true;
     } else if (strcmp(pUser->user, pOperUser->user) == 0) {
       hasRight = true;
     } else if (pOperUser->superAuth) {
-      if (strcmp(pUser->user, "root") == 0) {
+      if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
         hasRight = false;
       } else if (strcmp(pOperUser->acct, pUser->acct) != 0) {
         hasRight = false;
@@ -410,8 +421,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
     if (hasRight) {
       memset(pUser->pass, 0, sizeof(pUser->pass));
       taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass);
-      code = mnodeUpdateUser(pUser);
+      code = mnodeUpdateUser(pUser, pMsg);
-      mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
     } else {
       mError("user:%s, no rights to alter user", pOperUser->user);
       code = TSDB_CODE_MND_NO_RIGHTS;
@@ -419,16 +429,16 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
   } else if ((pAlter->flag & TSDB_ALTER_USER_PRIVILEGES) != 0) {
     bool hasRight = false;
 
-    if (strcmp(pUser->user, "root") == 0) {
+    if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
       hasRight = false;
     } else if (strcmp(pUser->user, pUser->acct) == 0) {
       hasRight = false;
-    } else if (strcmp(pOperUser->user, "root") == 0) {
+    } else if (strcmp(pOperUser->user, TSDB_DEFAULT_USER) == 0) {
       hasRight = true;
     } else if (strcmp(pUser->user, pOperUser->user) == 0) {
       hasRight = false;
     } else if (pOperUser->superAuth) {
-      if (strcmp(pUser->user, "root") == 0) {
+      if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
         hasRight = false;
       } else if (strcmp(pOperUser->acct, pUser->acct) != 0) {
         hasRight = false;
@@ -451,8 +461,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
         pUser->writeAuth = 1;
       }
 
-      code = mnodeUpdateUser(pUser);
+      code = mnodeUpdateUser(pUser, pMsg);
-      mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
     } else {
       mError("user:%s, no rights to alter user", pOperUser->user);
       code = TSDB_CODE_MND_NO_RIGHTS;
@@ -483,9 +492,9 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
   }
 
   bool hasRight = false;
-  if (strcmp(pUser->user, "root") == 0) {
+  if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
     hasRight = false;
-  } else if (strcmp(pOperUser->user, "root") == 0) {
+  } else if (strcmp(pOperUser->user, TSDB_DEFAULT_USER) == 0) {
     hasRight = true;
   } else if (strcmp(pUser->user, pOperUser->user) == 0) {
     hasRight = false;
@@ -498,10 +507,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
   }
 
   if (hasRight) {
-    code = mnodeDropUser(pUser);
+    code = mnodeDropUser(pUser, pMsg);
-    if (code == TSDB_CODE_SUCCESS) {
-      mLPrint("user:%s, is dropped by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code));
-    }
   } else {
     code = TSDB_CODE_MND_NO_RIGHTS;
   }

src/mnode/src/mnodeVgroup.c

@@ -74,6 +74,7 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) {
   pVgroup->pDb = pDb;
   pVgroup->prev = NULL;
   pVgroup->next = NULL;
+  pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
 
   int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
   pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *));
@@ -132,7 +133,7 @@ static void mnodeVgroupUpdateIdPool(SVgObj *pVgroup) {
       taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
       int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
       pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
-      memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj **));
+      memset(pVgroup->tableList + oldTables, 0, (pDb->cfg.maxTables - oldTables) * sizeof(SChildTableObj *));
     }
   }
 }
@@ -149,8 +150,7 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) {
       }
     }
 
-    memcpy(pVgroup, pNew, pOper->rowSize);
+    memcpy(pVgroup, pNew, tsVgUpdateSize);
-    free(pNew);
 
     for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
       SDnodeObj *pDnode = mnodeGetDnode(pVgroup->vnodeGid[i].dnodeId);
@@ -252,10 +252,14 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
     .pObj = pVgroup
   };
 
-  sdbUpdateRow(&oper);
+  if (sdbUpdateRow(&oper) != TSDB_CODE_SUCCESS) {
+    mError("vgId:%d, failed to update vgroup", pVgroup->vgId);
+  }
   mnodeSendCreateVgroupMsg(pVgroup, NULL);
 }
 
+void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {}
+
 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
   bool dnodeExist = false;
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
@@ -299,11 +303,35 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
   return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); 
 }
 
+static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
+  if (code != TSDB_CODE_SUCCESS) {
+    pMsg->pVgroup = NULL;
+    return code;
+  }
+
+  SVgObj *pVgroup = pMsg->pVgroup;
+  SDbObj *pDb = pMsg->pDb;
+
+  mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
+  }
+
+  mnodeIncVgroupRef(pVgroup);
+  pMsg->expected = pVgroup->numOfVnodes;
+  mnodeSendCreateVgroupMsg(pVgroup, pMsg);
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
 int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) {
+  if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
+
   SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj));
-  strcpy(pVgroup->dbName, pDb->name);
+  tstrncpy(pVgroup->dbName, pDb->name, TSDB_DB_NAME_LEN);
   pVgroup->numOfVnodes = pDb->cfg.replications;
   pVgroup->createdTime = taosGetTimestampMs();
+  pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
   if (balanceAllocVnodes(pVgroup) != 0) {
     mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
     free(pVgroup);
@@ -314,26 +342,22 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) {
     .type = SDB_OPER_GLOBAL,
     .table = tsVgroupSdb,
     .pObj = pVgroup,
-    .rowSize = sizeof(SVgObj)
+    .rowSize = sizeof(SVgObj),
+    .pMsg = pMsg,
+    .cb = mnodeCreateVgroupCb
   };
 
+  pMsg->pVgroup = pVgroup;
+
   int32_t code = sdbInsertRow(&oper);
   if (code != TSDB_CODE_SUCCESS) {
+    pMsg->pVgroup = NULL;
     tfree(pVgroup);
-    return TSDB_CODE_MND_SDB_ERROR;
+  } else {
+    code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 
-  mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
+  return code;
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
-  }
-
-  mnodeIncVgroupRef(pVgroup);
-  pMsg->pVgroup = pVgroup;
-  pMsg->expected = pVgroup->numOfVnodes;
-  mnodeSendCreateVgroupMsg(pVgroup, pMsg);
-
-  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
 void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) {
@@ -479,12 +503,12 @@ int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pC
 
       if (pDnode != NULL) {
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-        STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+        STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols]);
         cols++;
 
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
         char *role = mnodeGetMnodeRoleStr(pVgroup->vnodeGid[i].role);
-        STR_TO_VARSTR(pWrite, role);
+        STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]);
         cols++;
       } else {
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -596,7 +620,6 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) {
 }
 
 void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
-  mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p db:%s", pVgroup->vgId, pVgroup->vgId, ahandle, pVgroup->dbName);
   SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup);
   SRpcMsg rpcMsg = {
     .handle  = ahandle,
@@ -609,9 +632,12 @@ void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
 }
 
 void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
-  mTrace("vgId:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
+  mTrace("vgId:%d, send create all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes,
+         pVgroup->dbName);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
+    mTrace("vgId:%d, index:%d, send create vnode msg to dnode %s, ahandle:%p", pVgroup->vgId,
+           i, pVgroup->vnodeGid[i].pDnode->dnodeEp, ahandle);
     mnodeSendCreateVnodeMsg(pVgroup, &ipSet, ahandle);
   }
 }
@@ -717,7 +743,7 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
 
   SDnodeObj *pDnode = mnodeGetDnode(pCfg->dnodeId);
   if (pDnode == NULL) {
-    mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
+    mTrace("dnode:%s, vgId:%d, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
     return TSDB_CODE_MND_VGROUP_NOT_EXIST;
   }
   mnodeDecDnodeRef(pDnode);
@@ -729,6 +755,7 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
   }
   mnodeDecVgroupRef(pVgroup);
 
+  mTrace("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp);
   SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp);
   mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
 

src/mnode/src/mnodeWrite.c

@@ -43,7 +43,7 @@ void mnodeAddWriteMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg))
 
 int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
   if (pMsg->rpcMsg.pCont == NULL) {
-    mError("%p, msg:%s  in mwrite queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("app:%p:%p, msg:%s content is null", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_INVALID_MSG_LEN;
   }
 
@@ -54,27 +54,31 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
     rpcRsp->rsp = ipSet;
     rpcRsp->len = sizeof(SRpcIpSet);
 
-    mTrace("%p, msg:%s in mwrite queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse);
+    mTrace("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType],
+           ipSet->inUse);
     for (int32_t i = 0; i < ipSet->numOfIps; ++i) {
-      mTrace("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i]));
+      mTrace("app:%p:%p, mnode index:%d ip:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, ipSet->fqdn[i],
+             htons(ipSet->port[i]));
     }
 
     return TSDB_CODE_RPC_REDIRECT;
   }
 
   if (tsMnodeProcessWriteMsgFp[pMsg->rpcMsg.msgType] == NULL) {
-    mError("%p, msg:%s in mwrite queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("app:%p:%p, msg:%s not processed", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_MSG_NOT_PROCESSED;
   }
 
   int32_t code = mnodeInitMsg(pMsg);
   if (code != TSDB_CODE_SUCCESS) {
-    mError("%p, msg:%s in mwrite queue, not processed reason:%s", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], tstrerror(code));
+    mError("app:%p:%p, msg:%s not processed, reason:%s", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType],
+           tstrerror(code));
     return code;
   }
 
   if (!pMsg->pUser->writeAuth) {
-    mError("%p, msg:%s  in mwrite queue, not processed, no write auth", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("app:%p:%p, msg:%s not processed, no write auth", pMsg->rpcMsg.ahandle, pMsg,
+           taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_NO_RIGHTS;
   }
 

src/plugins/http/inc/gcHandle.h

@@ -16,15 +16,11 @@
 #ifndef TDENGINE_GC_HANDLE_H
 #define TDENGINE_GC_HANDLE_H
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <string.h>
-#include <unistd.h>
-
 #include "http.h"
-#include "httpCode.h"
+#include "httpInt.h"
-#include "httpHandle.h"
+#include "httpUtil.h"
 #include "httpResp.h"
+#include "httpSql.h"
 
 #define GC_ROOT_URL_POS   0
 #define GC_ACTION_URL_POS 1

src/plugins/http/inc/httpAuth.h

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_TOKEN_H
+#define TDENGINE_HTTP_TOKEN_H
+
+bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len);
+bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len);
+bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen);
+
+#endif

src/plugins/http/inc/httpContext.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_CONTEXT_H
+#define TDENGINE_HTTP_CONTEXT_H
+
+#include "httpInt.h"
+
+bool        httpInitContexts();
+void        httpCleanupContexts();
+const char *httpContextStateStr(HttpContextState state);
+
+HttpContext *httpCreateContext(int32_t fd);
+bool         httpInitContext(HttpContext *pContext);
+HttpContext *httpGetContext(void * pContext);
+void         httpReleaseContext(HttpContext *pContext);
+void         httpCloseContextByServer(HttpContext *pContext);
+void         httpCloseContextByApp(HttpContext *pContext);
+void         httpNotifyContextClose(HttpContext *pContext);
+bool         httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState);
+
+#endif

src/plugins/http/inc/httpHandle.h

@@ -13,304 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_HTTP_SERVER_H
+#ifndef TDENGINE_HTTP_HANDLE_H
-#define TDENGINE_HTTP_SERVER_H
+#define TDENGINE_HTTP_HANDLE_H
-
-#include <stdbool.h>
-#include "pthread.h"
-#include "semaphore.h"
-#include "tmempool.h"
-#include "taosdef.h"
-#include "tutil.h"
-#include "zlib.h"
-#include "http.h"
-#include "httpJson.h"
-
-#define HTTP_MAX_CMD_SIZE           1024
-#define HTTP_MAX_BUFFER_SIZE        1024*1024
-
-#define HTTP_LABEL_SIZE             8
-#define HTTP_MAX_EVENTS             10
-#define HTTP_BUFFER_SIZE            1024*65 //65k
-#define HTTP_DECOMPRESS_BUF_SIZE    1024*64
-#define HTTP_STEP_SIZE              1024    //http message get process step by step
-#define HTTP_MAX_URL                5       //http url stack size
-#define HTTP_METHOD_SCANNER_SIZE    7       //http method fp size
-#define HTTP_GC_TARGET_SIZE         512
-
-#define HTTP_VERSION_10             0
-#define HTTP_VERSION_11             1
-//#define HTTP_VERSION_12           2
-
-#define HTTP_UNCUNKED               0
-#define HTTP_CHUNKED                1
-
-#define HTTP_KEEPALIVE_NO_INPUT     0
-#define HTTP_KEEPALIVE_ENABLE       1
-#define HTTP_KEEPALIVE_DISABLE      2
-
-#define HTTP_REQTYPE_OTHERS         0
-#define HTTP_REQTYPE_LOGIN          1
-#define HTTP_REQTYPE_HEARTBEAT      2
-#define HTTP_REQTYPE_SINGLE_SQL     3
-#define HTTP_REQTYPE_MULTI_SQL      4
-
-#define HTTP_CHECK_BODY_ERROR      -1
-#define HTTP_CHECK_BODY_CONTINUE    0
-#define HTTP_CHECK_BODY_SUCCESS     1
-
-#define HTTP_WRITE_RETRY_TIMES      500
-#define HTTP_WRITE_WAIT_TIME_MS     5
-#define HTTP_EXPIRED_TIME           60000
-#define HTTP_DELAY_CLOSE_TIME_MS    500
-
-#define HTTP_COMPRESS_IDENTITY      0
-#define HTTP_COMPRESS_GZIP          2
-
-#define HTTP_SESSION_ID_LEN         (TSDB_USER_LEN + TSDB_PASSWORD_LEN)
-
-typedef enum {
-    HTTP_CONTEXT_STATE_READY,
-    HTTP_CONTEXT_STATE_HANDLING,
-    HTTP_CONTEXT_STATE_DROPPING,
-    HTTP_CONTEXT_STATE_CLOSED
-} HttpContextState;
-
-struct HttpContext;
-struct HttpThread;
-
-typedef struct {
-  void *signature;
-  int   expire;
-  int   access;
-  void *taos;
-  char  id[HTTP_SESSION_ID_LEN];
-} HttpSession;
-
-typedef enum {
-  HTTP_CMD_TYPE_UN_SPECIFIED,
-  HTTP_CMD_TYPE_CREATE_DB,
-  HTTP_CMD_TYPE_CREATE_STBALE,
-  HTTP_CMD_TYPE_INSERT
-} HttpSqlCmdType;
-
-typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSqlCmdState;
-
-typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType;
-
-typedef struct {
-  // used by single cmd
-  char   *nativSql;
-  int32_t numOfRows;
-  int32_t code;
-
-  // these are the locations in the buffer
-  int32_t tagNames[TSDB_MAX_TAGS];
-  int32_t tagValues[TSDB_MAX_TAGS];
-  int32_t timestamp;
-  int32_t metric;
-  int32_t stable;
-  int32_t table;
-  int32_t values;
-  int32_t sql;
-
-  // used by multi-cmd
-  int8_t cmdType;
-  int8_t cmdReturnType;
-  int8_t cmdState;
-  int8_t tagNum;
-} HttpSqlCmd;
-
-typedef struct {
-  HttpSqlCmd *cmds;
-  int16_t     pos;
-  int16_t     size;
-  int16_t     maxSize;
-  int32_t     bufferPos;
-  int32_t     bufferSize;
-  char *      buffer;
-} HttpSqlCmds;
-
-typedef struct {
-  char *module;
-  bool (*decodeFp)(struct HttpContext *pContext);
-} HttpDecodeMethod;
-
-typedef struct {
-  void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result);
-  void (*stopJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd);
-  bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result, int numOfRows);
-  void (*buildAffectRowJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int affectRows);
-  void (*initJsonFp)(struct HttpContext *pContext);
-  void (*cleanJsonFp)(struct HttpContext *pContext);
-  bool (*checkFinishedFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
-  void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
-} HttpEncodeMethod;
-
-typedef struct {
-  char   *pos;
-  int32_t len;
-} HttpBuf;
-
-typedef struct {
-  char              buffer[HTTP_BUFFER_SIZE];
-  int               bufsize;
-  char             *pLast;
-  char             *pCur;
-  HttpBuf           method;
-  HttpBuf           path[HTTP_MAX_URL];  // url: dbname/meter/query
-  HttpBuf           data;                // body content
-  HttpBuf           token;               // auth token
-  HttpDecodeMethod *pMethod;
-} HttpParser;
-
-typedef struct HttpContext {
-  void *       signature;
-  int          fd;
-  uint32_t     accessTimes;
-  uint32_t     lastAccessTime;
-  uint8_t      httpVersion;
-  uint8_t      httpChunked;
-  uint8_t      httpKeepAlive;  // http1.0 and not keep-alive, close connection immediately
-  uint8_t      fromMemPool;
-  uint8_t      acceptEncoding;
-  uint8_t      contentEncoding;
-  uint8_t      reqType;
-  uint8_t      parsed;
-  int32_t      state;
-  char         ipstr[22];
-  char         user[TSDB_USER_LEN];  // parsed from auth token or login message
-  char         pass[TSDB_PASSWORD_LEN];
-  void        *taos;
-  HttpSession *session;
-  z_stream     gzipStream;
-  HttpEncodeMethod   *encodeMethod;
-  HttpSqlCmd          singleCmd;
-  HttpSqlCmds        *multiCmds;
-  JsonBuf            *jsonBuf;
-  HttpParser          parser;
-  void               *timer;
-  struct HttpThread  *pThread;
-  struct HttpContext *prev;
-  struct HttpContext *next;
-} HttpContext;
-
-typedef struct HttpThread {
-  pthread_t       thread;
-  HttpContext *   pHead;
-  pthread_mutex_t threadMutex;
-  bool            stop;
-  int             pollFd;
-  int             numOfFds;
-  int             threadId;
-  char            label[HTTP_LABEL_SIZE];
-  bool (*processData)(HttpContext *pContext);
-  struct HttpServer *pServer;  // handle passed by upper layer during pServer initialization
-} HttpThread;
-
-typedef struct HttpServer {
-  char              label[HTTP_LABEL_SIZE];
-  uint32_t          serverIp;
-  uint16_t          serverPort;
-  bool              online;
-  int               fd;
-  int               cacheContext;
-  int               sessionExpire;
-  int               numOfThreads;
-  HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE];
-  int               methodScannerLen;
-  pthread_mutex_t   serverMutex;
-  void             *pSessionHash;
-  void             *pContextPool;
-  void             *expireTimer;
-  HttpThread       *pThreads;
-  pthread_t         thread;
-  bool            (*processData)(HttpContext *pContext);
-  int               requestNum;
-  void             *timerHandle;
-} HttpServer;
-
-// http util method
-bool httpCheckUsedbSql(char *sql);
-void httpTimeToString(time_t t, char *buf, int buflen);
-
-// http init method
-void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
-void httpCleanUpServer(HttpServer *pServer);
-
-// http server connection
-void httpCleanUpConnect(HttpServer *pServer);
-bool httpInitConnect(HttpServer *pServer);
-
-// http context for each client connection
-HttpContext *httpCreateContext(HttpServer *pServer);
-bool httpInitContext(HttpContext *pContext);
-void httpCloseContextByApp(HttpContext *pContext);
-void httpCloseContextByServer(HttpThread *pThread, HttpContext *pContext);
-
-// http session method
-void httpCreateSession(HttpContext *pContext, void *taos);
-void httpAccessSession(HttpContext *pContext);
-void httpFetchSession(HttpContext *pContext);
-void httpRestoreSession(HttpContext *pContext);
-void httpRemoveExpireSessions(HttpServer *pServer);
-bool httpInitAllSessions(HttpServer *pServer);
-void httpRemoveAllSessions(HttpServer *pServer);
-void httpProcessSessionExpire(void *handle, void *tmrId);
-
-// http request parser
-void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod);
-
-// http token method
-bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len);
-bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len);
-bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen);
-
-// util
-bool httpUrlMatch(HttpContext *pContext, int pos, char *cmp);
-bool httpProcessData(HttpContext *pContext);
-bool httpReadDataImp(HttpContext *pContext);
-bool httpParseRequest(HttpContext* pContext);
-int  httpCheckReadCompleted(HttpContext* pContext);
-void httpReadDirtyData(HttpContext *pContext);
 
 // http request handler
 void httpProcessRequest(HttpContext *pContext);
-
+bool httpProcessData(HttpContext *pContext);
-// http json printer
-JsonBuf *httpMallocJsonBuf(HttpContext *pContext);
-void httpFreeJsonBuf(HttpContext *pContext);
-
-// http multicmds util
-
-int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...);
-int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...);
-int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize);
-int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext);
-
-bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize);
-bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize);
-bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize);
-void httpFreeMultiCmds(HttpContext *pContext);
-
-HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext);
-HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext);
-int httpCurSqlCmdPos(HttpContext *pContext);
-
-void httpTrimTableName(char *name);
-int httpShrinkTableName(HttpContext *pContext, int pos, char *name);
-char *httpGetCmdsString(HttpContext *pContext, int pos);
-
-int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData);
-int httpGzipCompressInit(HttpContext *pContext);
-int httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen,
-                     char *outDestData, int32_t *outDestDataLen, bool isTheLast);
-
-extern const char *httpKeepAliveStr[];
-extern const char *httpVersionStr[];
-const char* httpContextStateStr(HttpContextState state);
-
-bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState);
-void httpRemoveContextFromEpoll(HttpThread *pThread, HttpContext *pContext);
 
 #endif

src/plugins/http/inc/httpInt.h

@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_INT_H
+#define TDENGINE_HTTP_INT_H
+
+#include <stdbool.h>
+#include "pthread.h"
+#include "semaphore.h"
+#include "tmempool.h"
+#include "taosdef.h"
+#include "tutil.h"
+#include "zlib.h"
+#include "http.h"
+#include "httpCode.h"
+#include "httpLog.h"
+#include "httpJson.h"
+
+#define HTTP_MAX_CMD_SIZE           1024
+#define HTTP_MAX_BUFFER_SIZE        1024*1024
+
+#define HTTP_LABEL_SIZE             8
+#define HTTP_MAX_EVENTS             10
+#define HTTP_BUFFER_SIZE            1024*65 //65k
+#define HTTP_DECOMPRESS_BUF_SIZE    1024*64
+#define HTTP_STEP_SIZE              1024    //http message get process step by step
+#define HTTP_MAX_URL                5       //http url stack size
+#define HTTP_METHOD_SCANNER_SIZE    7       //http method fp size
+#define HTTP_GC_TARGET_SIZE         512
+
+#define HTTP_VERSION_10             0
+#define HTTP_VERSION_11             1
+//#define HTTP_VERSION_12           2
+
+#define HTTP_UNCUNKED               0
+#define HTTP_CHUNKED                1
+
+#define HTTP_KEEPALIVE_NO_INPUT     0
+#define HTTP_KEEPALIVE_ENABLE       1
+#define HTTP_KEEPALIVE_DISABLE      2
+
+#define HTTP_REQTYPE_OTHERS         0
+#define HTTP_REQTYPE_LOGIN          1
+#define HTTP_REQTYPE_HEARTBEAT      2
+#define HTTP_REQTYPE_SINGLE_SQL     3
+#define HTTP_REQTYPE_MULTI_SQL      4
+
+#define HTTP_CHECK_BODY_ERROR      -1
+#define HTTP_CHECK_BODY_CONTINUE    0
+#define HTTP_CHECK_BODY_SUCCESS     1
+
+#define HTTP_WRITE_RETRY_TIMES      500
+#define HTTP_WRITE_WAIT_TIME_MS     5
+#define HTTP_EXPIRED_TIME           60000
+#define HTTP_DELAY_CLOSE_TIME_MS    500
+
+#define HTTP_COMPRESS_IDENTITY      0
+#define HTTP_COMPRESS_GZIP          2
+
+#define HTTP_SESSION_ID_LEN         (TSDB_USER_LEN + TSDB_PASSWORD_LEN)
+
+typedef enum {
+  HTTP_SERVER_INIT,
+  HTTP_SERVER_RUNNING,
+  HTTP_SERVER_CLOSING,
+  HTTP_SERVER_CLOSED
+} HttpServerStatus;
+
+typedef enum {
+    HTTP_CONTEXT_STATE_READY,
+    HTTP_CONTEXT_STATE_HANDLING,
+    HTTP_CONTEXT_STATE_DROPPING,
+    HTTP_CONTEXT_STATE_CLOSED
+} HttpContextState;
+
+struct HttpContext;
+struct HttpThread;
+
+typedef struct {
+  char  id[HTTP_SESSION_ID_LEN];
+  int   refCount;
+  void *taos;
+} HttpSession;
+
+typedef enum {
+  HTTP_CMD_TYPE_UN_SPECIFIED,
+  HTTP_CMD_TYPE_CREATE_DB,
+  HTTP_CMD_TYPE_CREATE_STBALE,
+  HTTP_CMD_TYPE_INSERT
+} HttpSqlCmdType;
+
+typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSqlCmdState;
+
+typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType;
+
+typedef struct {
+  // used by single cmd
+  char   *nativSql;
+  int32_t numOfRows;
+  int32_t code;
+
+  // these are the locations in the buffer
+  int32_t tagNames[TSDB_MAX_TAGS];
+  int32_t tagValues[TSDB_MAX_TAGS];
+  int32_t timestamp;
+  int32_t metric;
+  int32_t stable;
+  int32_t table;
+  int32_t values;
+  int32_t sql;
+
+  // used by multi-cmd
+  int8_t cmdType;
+  int8_t cmdReturnType;
+  int8_t cmdState;
+  int8_t tagNum;
+} HttpSqlCmd;
+
+typedef struct {
+  HttpSqlCmd *cmds;
+  int16_t     pos;
+  int16_t     size;
+  int16_t     maxSize;
+  int32_t     bufferPos;
+  int32_t     bufferSize;
+  char *      buffer;
+} HttpSqlCmds;
+
+typedef struct {
+  char *module;
+  bool (*decodeFp)(struct HttpContext *pContext);
+} HttpDecodeMethod;
+
+typedef struct {
+  void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result);
+  void (*stopJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd);
+  bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result, int numOfRows);
+  void (*buildAffectRowJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int affectRows);
+  void (*initJsonFp)(struct HttpContext *pContext);
+  void (*cleanJsonFp)(struct HttpContext *pContext);
+  bool (*checkFinishedFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
+  void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
+} HttpEncodeMethod;
+
+typedef struct {
+  char   *pos;
+  int32_t len;
+} HttpBuf;
+
+typedef struct {
+  char              buffer[HTTP_BUFFER_SIZE];
+  int               bufsize;
+  char             *pLast;
+  char             *pCur;
+  HttpBuf           method;
+  HttpBuf           path[HTTP_MAX_URL];  // url: dbname/meter/query
+  HttpBuf           data;                // body content
+  HttpBuf           token;               // auth token
+  HttpDecodeMethod *pMethod;
+} HttpParser;
+
+typedef struct HttpContext {
+  int32_t      refCount;
+  int          fd;
+  uint32_t     accessTimes;
+  uint32_t     lastAccessTime;
+  int32_t      state;
+  uint8_t      httpVersion;
+  uint8_t      httpChunked;
+  uint8_t      httpKeepAlive;  // http1.0 and not keep-alive, close connection immediately
+  uint8_t      acceptEncoding;
+  uint8_t      contentEncoding;
+  uint8_t      reqType;
+  uint8_t      parsed;
+  char         ipstr[22];
+  char         user[TSDB_USER_LEN];  // parsed from auth token or login message
+  char         pass[TSDB_PASSWORD_LEN];
+  void *       taos;
+  void *       ppContext;
+  HttpSession *session;
+  z_stream     gzipStream;
+  HttpParser   parser;
+  HttpSqlCmd   singleCmd;
+  HttpSqlCmds *multiCmds;
+  JsonBuf *    jsonBuf;
+  void *       timer;
+  HttpEncodeMethod * encodeMethod;
+  struct HttpThread *pThread;
+} HttpContext;
+
+typedef struct HttpThread {
+  pthread_t       thread;
+  HttpContext *   pHead;
+  pthread_mutex_t threadMutex;
+  bool            stop;
+  int             pollFd;
+  int             numOfFds;
+  int             threadId;
+  char            label[HTTP_LABEL_SIZE];
+  bool (*processData)(HttpContext *pContext);
+} HttpThread;
+
+typedef struct HttpServer {
+  char              label[HTTP_LABEL_SIZE];
+  uint32_t          serverIp;
+  uint16_t          serverPort;
+  int               fd;
+  int               numOfThreads;
+  int               methodScannerLen;
+  int32_t           requestNum;
+  int32_t           status;
+  pthread_t         thread;
+  HttpThread *      pThreads;
+  void *            contextCache;
+  void *            sessionCache;
+  pthread_mutex_t   serverMutex;
+  HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE];
+  bool (*processData)(HttpContext *pContext);
+} HttpServer;
+
+extern const char *httpKeepAliveStr[];
+extern const char *httpVersionStr[];
+extern HttpServer  tsHttpServer;
+
+#endif

src/plugins/http/inc/httpJson.h

@@ -97,4 +97,8 @@ void httpJsonPrint(JsonBuf* buf, const char* json, int len);
 // quick
 void httpJsonPairStatus(JsonBuf* buf, int code);
 
+// http json printer
+JsonBuf* httpMallocJsonBuf(struct HttpContext* pContext);
+void     httpFreeJsonBuf(struct HttpContext* pContext);
+
 #endif

src/plugins/http/inc/httpResp.h

@@ -16,7 +16,7 @@
 #ifndef TDENGINE_HTTP_RESP_H
 #define TDENGINE_HTTP_RESP_H
 
-#include "httpHandle.h"
+#include "httpInt.h"
 
 enum _httpRespTempl {
   HTTP_RESPONSE_JSON_OK,

src/plugins/http/inc/httpServer.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_SERVER_H
+#define TDENGINE_HTTP_SERVER_H
+
+#include "httpInt.h"
+
+bool httpInitConnect();
+void httpCleanUpConnect();
+
+void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
+void httpCleanUpServer(HttpServer *pServer);
+bool httpReadDataImp(HttpContext *pContext);
+
+#endif

src/plugins/http/inc/httpSession.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_SESSION_H
+#define TDENGINE_HTTP_SESSION_H
+
+bool httpInitSessions();
+void httpCleanUpSessions();
+
+// http session method
+void httpCreateSession(HttpContext *pContext, void *taos);
+void httpGetSession(HttpContext *pContext);
+void httpReleaseSession(HttpContext *pContext);
+
+#endif

src/plugins/http/inc/httpSql.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_SQL_H
+#define TDENGINE_HTTP_SQL_H
+
+
+int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...);
+int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...);
+int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize);
+int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext);
+
+bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize);
+bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize);
+bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize);
+void httpFreeMultiCmds(HttpContext *pContext);
+
+HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext);
+HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext);
+int httpCurSqlCmdPos(HttpContext *pContext);
+
+void httpTrimTableName(char *name);
+int httpShrinkTableName(HttpContext *pContext, int pos, char *name);
+char *httpGetCmdsString(HttpContext *pContext, int pos);
+
+#endif

src/plugins/http/inc/httpUtil.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_UTIL_H
+#define TDENGINE_HTTP_UTIL_H
+
+bool httpCheckUsedbSql(char *sql);
+void httpTimeToString(time_t t, char *buf, int buflen);
+
+bool httpUrlMatch(HttpContext *pContext, int pos, char *cmp);
+bool httpParseRequest(HttpContext *pContext);
+int  httpCheckReadCompleted(HttpContext *pContext);
+void httpReadDirtyData(HttpContext *pContext);
+
+int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData);
+int httpGzipCompressInit(HttpContext *pContext);
+int httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen,
+                     char *outDestData, int32_t *outDestDataLen, bool isTheLast);
+
+// http request parser
+void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod);
+
+
+
+#endif

src/plugins/http/inc/restHandle.h

@@ -16,15 +16,11 @@
 #ifndef TDENGINE_REST_HANDLE_H
 #define TDENGINE_REST_HANDLE_H
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <string.h>
-#include <unistd.h>
-
 #include "http.h"
-#include "httpCode.h"
+#include "httpInt.h"
-#include "httpHandle.h"
+#include "httpUtil.h"
 #include "httpResp.h"
+#include "httpSql.h"
 
 #define REST_ROOT_URL_POS   0
 #define REST_ACTION_URL_POS 1

src/plugins/http/inc/tgHandle.h

@@ -16,16 +16,11 @@
 #ifndef TDENGINE_TG_HANDLE_H
 #define TDENGINE_TG_HANDLE_H
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "cJSON.h"
 #include "http.h"
-#include "httpCode.h"
+#include "httpInt.h"
-#include "httpHandle.h"
+#include "httpUtil.h"
 #include "httpResp.h"
+#include "httpSql.h"
 
 #define TG_ROOT_URL_POS   0
 #define TG_DB_URL_POS     1

src/plugins/http/src/gcHandle.c

@@ -52,7 +52,7 @@ bool gcGetUserFromUrl(HttpContext* pContext) {
     return false;
   }
 
-  strcpy(pContext->user, pParser->path[GC_USER_URL_POS].pos);
+  tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].pos, TSDB_USER_LEN);
   return true;
 }
 
@@ -62,7 +62,7 @@ bool gcGetPassFromUrl(HttpContext* pContext) {
     return false;
   }
 
-  strcpy(pContext->pass, pParser->path[GC_PASS_URL_POS].pos);
+  tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
   return true;
 }
 

src/plugins/http/src/httpAuth.c

@@ -18,8 +18,8 @@
 #include "tkey.h"
 #include "tutil.h"
 #include "http.h"
-#include "httpLog.h"
+#include "httpInt.h"
-#include "httpHandle.h"
+#include "httpAuth.h"
 
 #define KEY_DES_4 4971256377704625728L
 
@@ -29,6 +29,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) {
   char *base64 = (char *)base64_decode(token, len, &outlen);
   if (base64 == NULL || outlen == 0) {
     httpError("context:%p, fd:%d, ip:%s, basic token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token);
+    free(base64);
     return false;
   }
 
@@ -73,6 +74,7 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) {
   unsigned char *base64 = base64_decode(token, len, &outlen);
   if (base64 == NULL || outlen == 0) {
     httpError("context:%p, fd:%d, ip:%s, taosd token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token);
+    if (base64) free(base64);
     return false;
   }
   if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) {

src/plugins/http/src/httpContext.c

@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "tsocket.h"
+#include "tutil.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "tcache.h"
+#include "hash.h"
+#include "httpInt.h"
+#include "httpResp.h"
+#include "httpSql.h"
+#include "httpSession.h"
+
+static void httpRemoveContextFromEpoll(HttpContext *pContext) {
+  HttpThread *pThread = pContext->pThread;
+  if (pContext->fd >= 0) {
+    epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL);
+    taosCloseSocket(pContext->fd);
+    pContext->fd = -1;
+  }
+}
+
+static void httpDestroyContext(void *data) {
+  HttpContext *pContext = *(HttpContext **)data;
+  if (pContext->fd > 0) tclose(pContext->fd);
+
+  HttpThread *pThread = pContext->pThread;
+  httpRemoveContextFromEpoll(pContext);
+  httpReleaseSession(pContext);
+  atomic_sub_fetch_32(&pThread->numOfFds, 1);
+  
+  pContext->pThread = 0;
+  pContext->state = HTTP_CONTEXT_STATE_CLOSED;
+
+  // avoid double free
+  httpFreeJsonBuf(pContext);
+  httpFreeMultiCmds(pContext);
+  
+  httpTrace("context:%p, is destroyed, refCount:%d", pContext, pContext->refCount);
+  tfree(pContext);
+}
+
+bool httpInitContexts() {
+  tsHttpServer.contextCache = taosCacheInitWithCb(2, httpDestroyContext);
+  if (tsHttpServer.contextCache == NULL) {
+    httpError("failed to init context cache");
+    return false;
+  }
+
+  return true;
+}
+
+void httpCleanupContexts() {
+  if (tsHttpServer.contextCache != NULL) {
+    SCacheObj *cache = tsHttpServer.contextCache;
+    httpPrint("context cache is cleanuping, size:%zu", taosHashGetSize(cache->pHashTable));
+    taosCacheCleanup(tsHttpServer.contextCache);
+    tsHttpServer.contextCache = NULL;
+  }
+}
+
+const char *httpContextStateStr(HttpContextState state) {
+  switch (state) {
+    case HTTP_CONTEXT_STATE_READY:
+      return "ready";
+    case HTTP_CONTEXT_STATE_HANDLING:
+      return "handling";
+    case HTTP_CONTEXT_STATE_DROPPING:
+      return "dropping";
+    case HTTP_CONTEXT_STATE_CLOSED:
+      return "closed";
+    default:
+      return "unknown";
+  }
+}
+
+void httpNotifyContextClose(HttpContext *pContext) { 
+  shutdown(pContext->fd, SHUT_WR); 
+}
+
+bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) {
+  return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState);
+}
+
+HttpContext *httpCreateContext(int32_t fd) {
+  HttpContext *pContext = calloc(1, sizeof(HttpContext));
+  if (pContext == NULL) return NULL;
+
+  char contextStr[16] = {0};
+  snprintf(contextStr, sizeof(contextStr), "%p", pContext);
+  
+  pContext->fd = fd;
+  pContext->httpVersion = HTTP_VERSION_10;
+  pContext->lastAccessTime = taosGetTimestampSec();
+  pContext->state = HTTP_CONTEXT_STATE_READY;
+  
+  HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, contextStr, &pContext, sizeof(HttpContext *), 3);
+  pContext->ppContext = ppContext;
+  httpTrace("context:%p, fd:%d, is created, item:%p", pContext, fd, ppContext);
+
+  // set the ref to 0 
+  taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false);
+
+  return pContext;
+}
+
+HttpContext *httpGetContext(void *ptr) {
+  char contextStr[16] = {0};
+  snprintf(contextStr, sizeof(contextStr), "%p", ptr);
+  
+  HttpContext **ppContext = taosCacheAcquireByName(tsHttpServer.contextCache, contextStr);
+  
+  if (ppContext) {
+    HttpContext *pContext = *ppContext;
+    if (pContext) {
+      int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1);
+      httpTrace("context:%p, fd:%d, is accquired, refCount:%d", pContext, pContext->fd, refCount);
+      return pContext;
+    }
+  }
+  return NULL;
+}
+
+void httpReleaseContext(HttpContext *pContext) {
+  int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1);
+  assert(refCount >= 0);
+  httpTrace("context:%p, fd:%d, is releasd, refCount:%d", pContext, pContext->fd, refCount);
+
+  HttpContext **ppContext = pContext->ppContext;
+  taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false);
+}
+
+bool httpInitContext(HttpContext *pContext) {
+  pContext->accessTimes++;
+  pContext->lastAccessTime = taosGetTimestampSec();
+  pContext->httpVersion = HTTP_VERSION_10;
+  pContext->httpKeepAlive = HTTP_KEEPALIVE_NO_INPUT;
+  pContext->httpChunked = HTTP_UNCUNKED;
+  pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY;
+  pContext->contentEncoding = HTTP_COMPRESS_IDENTITY;
+  pContext->reqType = HTTP_REQTYPE_OTHERS;
+  pContext->encodeMethod = NULL;
+  pContext->timer = NULL;
+  memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
+
+  HttpParser *pParser = &pContext->parser;
+  memset(pParser, 0, sizeof(HttpParser));
+  pParser->pCur = pParser->pLast = pParser->buffer;
+
+  httpTrace("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d",
+          pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed);
+  return true;
+}
+
+void httpCloseContextByApp(HttpContext *pContext) {
+  pContext->parsed = false;
+
+  bool keepAlive = true;
+  if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) {
+    keepAlive = false;
+  } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) {
+    keepAlive = false;
+  } else {}
+
+  if (keepAlive) {
+    if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
+      httpTrace("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect",
+              pContext, pContext->fd, pContext->ipstr);
+    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
+      httpRemoveContextFromEpoll(pContext);
+      httpTrace("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect",
+              pContext, pContext->fd, pContext->ipstr);
+    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
+      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect",
+              pContext, pContext->fd, pContext->ipstr);
+    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
+      httpRemoveContextFromEpoll(pContext);
+      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect",
+                pContext, pContext->fd, pContext->ipstr);
+    } else {
+      httpRemoveContextFromEpoll(pContext);
+      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect",
+              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+    }
+  } else {
+    httpRemoveContextFromEpoll(pContext);
+    httpTrace("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect",
+              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+  }
+
+  httpReleaseContext(pContext);
+}
+
+void httpCloseContextByServer(HttpContext *pContext) {
+  if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr);
+  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr);
+  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr);
+  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr);
+  } else {
+    httpError("context:%p, fd:%d, ip:%s, unknown state:%d", pContext, pContext->fd, pContext->ipstr, pContext->state);
+  }
+
+  pContext->parsed = false;
+  httpRemoveContextFromEpoll(pContext);
+  httpReleaseContext(pContext);
+}

src/plugins/http/src/httpHandle.c

@@ -19,11 +19,12 @@
 #include "tglobal.h"
 #include "tsocket.h"
 #include "ttimer.h"
-#include "http.h"
+#include "httpInt.h"
-#include "httpLog.h"
-#include "httpCode.h"
-#include "httpHandle.h"
 #include "httpResp.h"
+#include "httpAuth.h"
+#include "httpServer.h"
+#include "httpContext.h"
+#include "httpHandle.h"
 
 void httpToLowerUrl(char* url) {
   /*ignore case */
@@ -58,6 +59,10 @@ bool httpParseURL(HttpContext* pContext) {
   HttpParser* pParser = &pContext->parser;
   char* pSeek;
   char* pEnd = strchr(pParser->pLast, ' ');
+  if (pEnd == NULL) {
+    return false;
+  }
+
   if (*pParser->pLast != '/') {
     httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL);
     return false;
@@ -159,7 +164,7 @@ bool httpGetHttpMethod(HttpContext* pContext) {
 bool httpGetDecodeMethod(HttpContext* pContext) {
   HttpParser* pParser = &pContext->parser;
 
-  HttpServer* pServer = pContext->pThread->pServer;
+  HttpServer* pServer = &tsHttpServer;
   int         methodLen = pServer->methodScannerLen;
   for (int i = 0; i < methodLen; i++) {
     HttpDecodeMethod* method = pServer->methodScanner[i];

src/plugins/http/src/httpJson.c

@@ -22,6 +22,7 @@
 #include "httpCode.h"
 #include "httpJson.h"
 #include "httpResp.h"
+#include "httpUtil.h"
 
 #define MAX_NUM_STR_SZ 25
 
@@ -138,7 +139,7 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
         return 0;  // there is no data to dump.
       }
     } else {
-      httpError("context:%p, fd:%d, ip:%s, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s",
+      httpError("context:%p, fd:%d, ip:%s, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s",
                 buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, isTheLast, ret, buf->buf);
       return 0;
     }
@@ -441,14 +442,13 @@ void httpJsonPairStatus(JsonBuf* buf, int code) {
     httpJsonPair(buf, "status", 6, "error", 5);
     httpJsonItemToken(buf);
     httpJsonPairIntVal(buf, "code", 4, code);
-    if (code >= 0) {
+    httpJsonItemToken(buf);
-      httpJsonItemToken(buf);
+    if (code == TSDB_CODE_MND_DB_NOT_SELECTED) {
-      if (code == TSDB_CODE_MND_DB_NOT_SELECTED) {
+      httpJsonPair(buf, "desc", 4, "failed to create database", 23);
-        httpJsonPair(buf, "desc", 4, "failed to create database", 23);
+    } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
-      } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
+      httpJsonPair(buf, "desc", 4, "failed to create table", 22);
-        httpJsonPair(buf, "desc", 4, "failed to create table", 22);
+    } else {
-      } else
+      httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int)strlen(tstrerror(code)));
-        httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int)strlen(tstrerror(code)));
     }
   }
 }

src/plugins/http/src/httpResp.c

@@ -21,6 +21,7 @@
 #include "httpResp.h"
 #include "httpCode.h"
 #include "httpJson.h"
+#include "httpContext.h"
 
 const char *httpKeepAliveStr[] = {"", "Connection: Keep-Alive\r\n", "Connection: Close\r\n"};
 

src/plugins/http/src/httpServer.c

@@ -21,244 +21,15 @@
 #include "ttime.h"
 #include "ttimer.h"
 #include "tglobal.h"
-#include "http.h"
+#include "httpInt.h"
-#include "httpLog.h"
+#include "httpContext.h"
-#include "httpCode.h"
-#include "httpHandle.h"
 #include "httpResp.h"
+#include "httpUtil.h"
 
 #ifndef EPOLLWAKEUP
  #define EPOLLWAKEUP (1u << 29)
 #endif
 
-const char* httpContextStateStr(HttpContextState state) {
-  switch (state) {
-    case HTTP_CONTEXT_STATE_READY:
-      return "ready";
-    case HTTP_CONTEXT_STATE_HANDLING:
-      return "handling";
-    case HTTP_CONTEXT_STATE_DROPPING:
-      return "dropping";
-    case HTTP_CONTEXT_STATE_CLOSED:
-      return "closed";
-    default:
-      return "unknown";
-  }
-}
-
-void httpRemoveContextFromEpoll(HttpThread *pThread, HttpContext *pContext) {
-  if (pContext->fd >= 0) {
-    epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL);
-    taosCloseSocket(pContext->fd);
-    pContext->fd = -1;
-  }
-}
-
-bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) {
-  return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState);
-}
-
-void httpFreeContext(HttpServer *pServer, HttpContext *pContext);
-
-/**
- * context will be reused while connection exist
- * multiCmds and jsonBuf will be malloc after taos_query_a called
- * and won't be freed until connection closed
- */
-HttpContext *httpCreateContext(HttpServer *pServer) {
-  HttpContext *pContext = (HttpContext *)taosMemPoolMalloc(pServer->pContextPool);
-  if (pContext != NULL) {
-    pContext->fromMemPool = 1;
-    httpTrace("context:%p, is malloced from mempool", pContext);
-  } else {
-    pContext = (HttpContext *)malloc(sizeof(HttpContext));
-    if (pContext == NULL) {
-      return NULL;
-    } else {
-      memset(pContext, 0, sizeof(HttpContext));
-    }
-    httpTrace("context:%p, is malloced from raw memory", pContext);
-  }
-
-  pContext->signature = pContext;
-  pContext->httpVersion = HTTP_VERSION_10;
-  pContext->lastAccessTime = taosGetTimestampSec();
-  pContext->state = HTTP_CONTEXT_STATE_READY;
-  return pContext;
-}
-
-void httpFreeContext(HttpServer *pServer, HttpContext *pContext) {
-  if (pContext->fromMemPool) {
-    httpTrace("context:%p, is freed from mempool", pContext);
-    taosMemPoolFree(pServer->pContextPool, (char *)pContext);
-  } else {
-    httpTrace("context:%p, is freed from raw memory", pContext);
-    tfree(pContext);
-  }
-}
-
-void httpCleanUpContextTimer(HttpContext *pContext) {
-  if (pContext->timer != NULL) {
-    taosTmrStopA(&pContext->timer);
-    //httpTrace("context:%p, ip:%s, close timer:%p", pContext, pContext->ipstr, pContext->timer);
-    pContext->timer = NULL;
-  }
-}
-
-void httpCleanUpContext(HttpContext *pContext, void *unused) {
-  httpTrace("context:%p, start the clean up operation, sig:%p", pContext, pContext->signature);
-  void *sig = atomic_val_compare_exchange_ptr(&pContext->signature, pContext, 0);
-  if (sig == NULL) {
-    httpTrace("context:%p is freed by another thread.", pContext);
-    return;
-  }
-
-  HttpThread *pThread = pContext->pThread;
-
-  httpCleanUpContextTimer(pContext);
-
-  httpRemoveContextFromEpoll(pThread, pContext);
-
-  httpRestoreSession(pContext);
-
-  pthread_mutex_lock(&pThread->threadMutex);
-
-  pThread->numOfFds--;
-  if (pThread->numOfFds < 0) {
-    httpError("context:%p, ip:%s, thread:%s, number of FDs:%d shall never be negative",
-              pContext, pContext->ipstr, pThread->label, pThread->numOfFds);
-    pThread->numOfFds = 0;
-  }
-
-  // remove from the link list
-  if (pContext->prev) {
-    (pContext->prev)->next = pContext->next;
-  } else {
-    pThread->pHead = pContext->next;
-  }
-
-  if (pContext->next) {
-    (pContext->next)->prev = pContext->prev;
-  }
-
-  pthread_mutex_unlock(&pThread->threadMutex);
-
-  httpTrace("context:%p, ip:%s, thread:%s, numOfFds:%d, context is cleaned up", pContext, pContext->ipstr,
-            pThread->label, pThread->numOfFds);
-
-  pContext->signature = 0;
-  pContext->fd = -1;
-  pContext->pThread = 0;
-  pContext->prev = 0;
-  pContext->next = 0;
-  pContext->state = HTTP_CONTEXT_STATE_READY;
-
-  // avoid double free
-  httpFreeJsonBuf(pContext);
-  httpFreeMultiCmds(pContext);
-  httpFreeContext(pThread->pServer, pContext);
-}
-
-bool httpInitContext(HttpContext *pContext) {
-  pContext->accessTimes++;
-  pContext->lastAccessTime = taosGetTimestampSec();
-  pContext->httpVersion = HTTP_VERSION_10;
-  pContext->httpKeepAlive = HTTP_KEEPALIVE_NO_INPUT;
-  pContext->httpChunked = HTTP_UNCUNKED;
-  pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY;
-  pContext->contentEncoding = HTTP_COMPRESS_IDENTITY;
-  pContext->reqType = HTTP_REQTYPE_OTHERS;
-  pContext->encodeMethod = NULL;
-  pContext->timer = NULL;
-  memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
-
-  HttpParser *pParser = &pContext->parser;
-  memset(pParser, 0, sizeof(HttpParser));
-  pParser->pCur = pParser->pLast = pParser->buffer;
-
-  httpTrace("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d",
-          pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed);
-  return true;
-}
-
-
-void httpCloseContext(HttpThread *pThread, HttpContext *pContext) {
-  taosTmrReset((TAOS_TMR_CALLBACK)httpCleanUpContext, HTTP_DELAY_CLOSE_TIME_MS, pContext, pThread->pServer->timerHandle, &pContext->timer);
-  httpTrace("context:%p, fd:%d, ip:%s, state:%s will be closed after:%d ms, timer:%p",
-          pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), HTTP_DELAY_CLOSE_TIME_MS, pContext->timer);
-}
-
-void httpCloseContextByApp(HttpContext *pContext) {
-  HttpThread *pThread = pContext->pThread;
-  pContext->parsed = false;
-
-  bool keepAlive = true;
-  if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) {
-    keepAlive = false;
-  } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) {
-    keepAlive = false;
-  } else {}
-
-  if (keepAlive) {
-    if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
-      httpTrace("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
-    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
-      httpRemoveContextFromEpoll(pThread, pContext);
-      httpTrace("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr);
-      httpCloseContext(pThread, pContext);
-    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
-      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
-    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
-      httpRemoveContextFromEpoll(pThread, pContext);
-      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect",
-                pContext, pContext->fd, pContext->ipstr);
-      httpCloseContext(pThread, pContext);
-    } else {
-      httpRemoveContextFromEpoll(pThread, pContext);
-      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
-      httpCloseContext(pThread, pContext);
-    }
-  } else {
-    httpRemoveContextFromEpoll(pThread, pContext);
-    httpTrace("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
-    httpCloseContext(pThread, pContext);
-  }
-}
-
-void httpCloseContextByServer(HttpThread *pThread, HttpContext *pContext) {
-  httpRemoveContextFromEpoll(pThread, pContext);
-  pContext->parsed = false;
-  
-  if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr);
-  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr);
-  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr);
-    httpCloseContext(pThread, pContext);
-  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr);
-    httpCloseContext(pThread, pContext);
-  } else {
-    httpError("context:%p, fd:%d, ip:%s, unknown state:%d", pContext, pContext->fd, pContext->ipstr, pContext->state);
-    httpCloseContext(pThread, pContext);
-  }
-}
-
-void httpCloseContextByServerForExpired(void *param, void *tmrId) {
-  HttpContext *pContext = (HttpContext *)param;
-  httpRemoveContextFromEpoll(pContext->pThread, pContext);
-  httpError("context:%p, fd:%d, ip:%s, read http body error, time expired, timer:%p", pContext, pContext->fd, pContext->ipstr, tmrId);
-  httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR);
-  httpCloseContextByServer(pContext->pThread, pContext);
-}
-
-
 static void httpStopThread(HttpThread* pThread) {
   pThread->stop = true;
 
@@ -281,19 +52,13 @@ static void httpStopThread(HttpThread* pThread) {
 
   close(pThread->pollFd);
   pthread_mutex_destroy(&(pThread->threadMutex));
-
-  //while (pThread->pHead) {
-  //  httpCleanUpContext(pThread->pHead, 0);
-  //}
 }
 
+void httpCleanUpConnect() {
+  HttpServer *pServer = &tsHttpServer;
+  if (pServer->pThreads == NULL) return;
 
-void httpCleanUpConnect(HttpServer *pServer) {
-  if (pServer == NULL) return;
-
-  shutdown(pServer->fd, SHUT_RD);
   pthread_join(pServer->thread, NULL);
-
   for (int i = 0; i < pServer->numOfThreads; ++i) {
     HttpThread* pThread = pServer->pThreads + i;
     if (pThread != NULL) {
@@ -302,19 +67,10 @@ void httpCleanUpConnect(HttpServer *pServer) {
   }
 
   tfree(pServer->pThreads);
+  pServer->pThreads = NULL;
   httpTrace("http server:%s is cleaned up", pServer->label);
 }
 
-// read all the data, then just discard it
-void httpReadDirtyData(HttpContext *pContext) {
-  int fd = pContext->fd;
-  char data[1024] = {0};
-  int  len = (int)taosReadSocket(fd, data, 1024);
-  while (len >= sizeof(data)) {
-    len = (int)taosReadSocket(fd, data, 1024);
-  }
-}
-
 bool httpReadDataImp(HttpContext *pContext) {
   HttpParser *pParser = &pContext->parser;
 
@@ -338,11 +94,10 @@ bool httpReadDataImp(HttpContext *pContext) {
     }
 
     if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) {
-      httpReadDirtyData(pContext);
       httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d",
                 pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE);
-      httpRemoveContextFromEpoll(pContext->pThread, pContext);
       httpSendErrorResp(pContext, HTTP_REQUSET_TOO_BIG);
+      httpNotifyContextClose(pContext);
       return false;
     }
   }
@@ -352,7 +107,7 @@ bool httpReadDataImp(HttpContext *pContext) {
   return true;
 }
 
-bool httpDecompressData(HttpContext *pContext) {
+static bool httpDecompressData(HttpContext *pContext) {
   if (pContext->contentEncoding != HTTP_COMPRESS_GZIP) {
     httpDump("context:%p, fd:%d, ip:%s, content:%s", pContext, pContext->fd, pContext->ipstr, pContext->parser.data.pos);
     return true;
@@ -382,45 +137,43 @@ bool httpDecompressData(HttpContext *pContext) {
   return ret == 0;
 }
 
-bool httpReadData(HttpThread *pThread, HttpContext *pContext) {
+static bool httpReadData(HttpContext *pContext) {
   if (!pContext->parsed) {
     httpInitContext(pContext);
   }
 
   if (!httpReadDataImp(pContext)) {
-    httpCloseContextByServer(pThread, pContext);
+    httpNotifyContextClose(pContext);
     return false;
   }
 
   if (!httpParseRequest(pContext)) {
-    httpCloseContextByServer(pThread, pContext);
+    httpNotifyContextClose(pContext);
     return false;
   }
 
   int ret = httpCheckReadCompleted(pContext);
   if (ret == HTTP_CHECK_BODY_CONTINUE) {
-    taosTmrReset(httpCloseContextByServerForExpired, HTTP_EXPIRED_TIME, pContext, pThread->pServer->timerHandle, &pContext->timer);
+    //httpTrace("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr);
-    //httpTrace("context:%p, fd:%d, ip:%s, not finished yet, try another times, timer:%p", pContext, pContext->fd, pContext->ipstr, pContext->timer);
     return false;
   } else if (ret == HTTP_CHECK_BODY_SUCCESS){
-    httpCleanUpContextTimer(pContext);
     httpTrace("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d",
               pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->parser.bufsize, pContext->parser.data.len);
     if (httpDecompressData(pContext)) {
       return true;
     } else {
-      httpCloseContextByServer(pThread, pContext);
+      httpNotifyContextClose(pContext);
       return false;
     }
   } else {
-    httpCleanUpContextTimer(pContext);
     httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr);
-    httpCloseContextByServer(pThread, pContext);
+    httpNotifyContextClose(pContext);
     return false;
   }
 }
 
-void httpProcessHttpData(void *param) {
+static void httpProcessHttpData(void *param) {
+  HttpServer  *pServer = &tsHttpServer;
   HttpThread  *pThread = (HttpThread *)param;
   HttpContext *pContext;
   int          fdNum;
@@ -441,77 +194,72 @@ void httpProcessHttpData(void *param) {
     if (fdNum <= 0) continue;
 
     for (int i = 0; i < fdNum; ++i) {
-      pContext = events[i].data.ptr;
+      pContext = httpGetContext(events[i].data.ptr);
-      if (pContext->signature != pContext || pContext->pThread != pThread || pContext->fd <= 0) {
+      if (pContext == NULL) {
+        httpError("context:%p, is already released, close connect", events[i].data.ptr);
+        //epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
+        //tclose(events[i].data.fd);
         continue;
       }
 
       if (events[i].events & EPOLLPRI) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLPRI events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
+        httpCloseContextByServer(pContext);
-        httpCloseContextByServer(pThread, pContext);
         continue;
       }
 
       if (events[i].events & EPOLLRDHUP) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
+        httpCloseContextByServer(pContext);
-        httpCloseContextByServer(pThread, pContext);
         continue;
       }
 
       if (events[i].events & EPOLLERR) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLERR events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
+        httpCloseContextByServer(pContext);
-        httpCloseContextByServer(pThread, pContext);
         continue;
       }
 
       if (events[i].events & EPOLLHUP) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLHUP events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
+        httpCloseContextByServer(pContext);
-        httpCloseContextByServer(pThread, pContext);
         continue;
       }
 
       if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, not in ready state, ignore read events",
                 pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state));
+        httpReleaseContext(pContext);
         continue;
       }
 
-      if (!pContext->pThread->pServer->online) {
+      if (pServer->status != HTTP_SERVER_RUNNING) {
-        httpTrace("context:%p, fd:%d, ip:%s, state:%s, server is not online, accessed:%d, close connect",
+        httpTrace("context:%p, fd:%d, ip:%s, state:%s, server is not running, accessed:%d, close connect", pContext,
-                  pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
+                  pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
-        httpReadDirtyData(pContext);
         httpSendErrorResp(pContext, HTTP_SERVER_OFFLINE);
-        httpCloseContextByServer(pThread, pContext);
+        httpNotifyContextClose(pContext);
-        continue;
       } else {
-        if (httpReadData(pThread, pContext)) {
+        if (httpReadData(pContext)) {
           (*(pThread->processData))(pContext);
-          atomic_fetch_add_32(&pThread->pServer->requestNum, 1);
+          atomic_fetch_add_32(&pServer->requestNum, 1);
         }
       }
     }
   }
 }
 
-void* httpAcceptHttpConnection(void *arg) {
+static void *httpAcceptHttpConnection(void *arg) {
   int                connFd = -1;
   struct sockaddr_in clientAddr;
   int                threadId = 0;
-  HttpThread *       pThread;
+  HttpServer *       pServer = &tsHttpServer;
-  HttpServer *       pServer;
+  HttpThread *       pThread = NULL;
-  HttpContext *      pContext;
+  HttpContext *      pContext = NULL;
-  int                totalFds;
+  int                totalFds = 0;
-
-  pServer = (HttpServer *)arg;
 
   sigset_t set;
   sigemptyset(&set);
@@ -521,12 +269,12 @@ void* httpAcceptHttpConnection(void *arg) {
   pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort);
 
   if (pServer->fd < 0) {
-    httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label, taosIpStr(pServer->serverIp),
+    httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label,
-              pServer->serverPort, strerror(errno));
+              taosIpStr(pServer->serverIp), pServer->serverPort, strerror(errno));
     return NULL;
   } else {
-    httpPrint("http service init success at %u", pServer->serverPort);
+    httpPrint("http server init success at %u", pServer->serverPort);
-    pServer->online = true;
+    pServer->status = HTTP_SERVER_RUNNING;
   }
 
   while (1) {
@@ -534,10 +282,10 @@ void* httpAcceptHttpConnection(void *arg) {
     connFd = (int)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen);
     if (connFd == -1) {
       if (errno == EINVAL) {
-        httpTrace("%s HTTP server socket was shutdown, exiting...", pServer->label);
+        httpTrace("http server:%s socket was shutdown, exiting...", pServer->label);
         break;
       }
-      httpError("http server:%s, accept connect failure, errno:%d, reason:%s", pServer->label, errno, strerror(errno));
+      httpError("http server:%s, accept connect failure, errno:%d reason:%s", pServer->label, errno, strerror(errno));
       continue;
     }
 
@@ -547,8 +295,8 @@ void* httpAcceptHttpConnection(void *arg) {
     }
 
     if (totalFds > tsHttpCacheSessions * 100) {
-      httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection",
+      httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection", connFd,
-              connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions);
+                inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions);
       taosCloseSocket(connFd);
       continue;
     }
@@ -559,7 +307,7 @@ void* httpAcceptHttpConnection(void *arg) {
     // pick up the thread to handle this connection
     pThread = pServer->pThreads + threadId;
 
-    pContext = httpCreateContext(pServer);
+    pContext = httpCreateContext(connFd);
     if (pContext == NULL) {
       httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, inet_ntoa(clientAddr.sin_addr),
                 htons(clientAddr.sin_port));
@@ -567,39 +315,24 @@ void* httpAcceptHttpConnection(void *arg) {
       continue;
     }
 
-    httpTrace("context:%p, fd:%d, ip:%s:%u, thread:%s, numOfFds:%d, totalFds:%d, accept a new connection",
-            pContext, connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), pThread->label,
-            pThread->numOfFds, totalFds);
-
-    pContext->fd = connFd;
-    sprintf(pContext->ipstr, "%s:%d", inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port));
     pContext->pThread = pThread;
-
+    sprintf(pContext->ipstr, "%s:%u", inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port));
+    
     struct epoll_event event;
     event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP;
-
     event.data.ptr = pContext;
     if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
-      httpError("context:%p, fd:%d, ip:%s:%u, thread:%s, failed to add http fd for epoll, error:%s",
+      httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd,
-                pContext, connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), pThread->label,
+                pContext->ipstr, pThread->label, strerror(errno));
-                strerror(errno));
+      tclose(pContext->fd);
-      httpFreeContext(pThread->pServer, pContext);
+      httpReleaseContext(pContext);
-      tclose(connFd);
       continue;
     }
 
     // notify the data process, add into the FdObj list
-    pthread_mutex_lock(&(pThread->threadMutex));
+    atomic_add_fetch_32(&pThread->numOfFds, 1);
-
+    httpTrace("context:%p, fd:%d, ip:%s, thread:%s numOfFds:%d totalFds:%d, accept a new connection", pContext, connFd,
-    pContext->next = pThread->pHead;
+              pContext->ipstr, pThread->label, pThread->numOfFds, totalFds);
-
-    if (pThread->pHead) (pThread->pHead)->prev = pContext;
-
-    pThread->pHead = pContext;
-
-    pThread->numOfFds++;
-
-    pthread_mutex_unlock(&(pThread->threadMutex));
 
     // pick up next thread for next connection
     threadId++;
@@ -610,21 +343,17 @@ void* httpAcceptHttpConnection(void *arg) {
   return NULL;
 }
 
-bool httpInitConnect(HttpServer *pServer) {
+bool httpInitConnect() {
-  int            i;
+  HttpServer *pServer = &tsHttpServer;
-  HttpThread *   pThread;
+  pServer->pThreads = calloc(pServer->numOfThreads, sizeof(HttpThread));
-
-  pServer->pThreads = (HttpThread *)malloc(sizeof(HttpThread) * (size_t)pServer->numOfThreads);
   if (pServer->pThreads == NULL) {
     httpError("init error no enough memory");
     return false;
   }
-  memset(pServer->pThreads, 0, sizeof(HttpThread) * (size_t)pServer->numOfThreads);
 
-  pThread = pServer->pThreads;
+  HttpThread *pThread = pServer->pThreads;
-  for (i = 0; i < pServer->numOfThreads; ++i) {
+  for (int i = 0; i < pServer->numOfThreads; ++i) {
     sprintf(pThread->label, "%s%d", pServer->label, i);
-    pThread->pServer = pServer;
     pThread->processData = pServer->processData;
     pThread->threadId = i;
 
@@ -643,8 +372,8 @@ bool httpInitConnect(HttpServer *pServer) {
     pthread_attr_init(&thattr);
     pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
     if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) {
-      httpError("http thread:%s, failed to create HTTP process data thread, reason:%s",
+      httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label,
-                pThread->label, strerror(errno));
+                strerror(errno));
       return false;
     }
     pthread_attr_destroy(&thattr);

src/plugins/http/src/httpSession.c

@@ -15,44 +15,29 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "hash.h"
 #include "taos.h"
 #include "ttime.h"
-#include "ttimer.h"
+#include "tglobal.h"
-#include "http.h"
+#include "tcache.h"
-#include "httpLog.h"
+#include "httpInt.h"
-#include "httpCode.h"
+#include "httpContext.h"
-#include "httpHandle.h"
+#include "httpSession.h"
-#include "httpResp.h"
-
-void httpAccessSession(HttpContext *pContext) {
-  HttpServer *server = pContext->pThread->pServer;
-  pthread_mutex_lock(&server->serverMutex);
-  if (pContext->session == pContext->session->signature) {
-    pContext->session->expire = (int) taosGetTimestampSec() + pContext->pThread->pServer->sessionExpire;
-  }
-  pthread_mutex_unlock(&server->serverMutex);
-}
 
 void httpCreateSession(HttpContext *pContext, void *taos) {
-  HttpServer *server = pContext->pThread->pServer;
+  HttpServer *server = &tsHttpServer;
+  httpReleaseSession(pContext);
+
   pthread_mutex_lock(&server->serverMutex);
 
-  if (pContext->session != NULL && pContext->session == pContext->session->signature) {
-    httpTrace("context:%p, fd:%d, ip:%s, user:%s, set exist session:%p:%p expired", pContext, pContext->fd,
-              pContext->ipstr, pContext->user, pContext->session, pContext->session->taos);
-    pContext->session->expire = 0;
-    pContext->session->access--;
-  }
-
   HttpSession session;
+  memset(&session, 0, sizeof(HttpSession));
   session.taos = taos;
-  session.expire = (int)taosGetTimestampSec() + server->sessionExpire;
+  session.refCount = 1;
-  session.access = 1;
+  snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
-  int sessionIdLen = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
 
-  taosHashPut(server->pSessionHash, session.id, sessionIdLen, (char *)(&session), sizeof(HttpSession));
+  pContext->session = taosCachePut(server->sessionCache, session.id, &session, sizeof(HttpSession), tsHttpSessionExpire);
-  pContext->session = taosHashGet(server->pSessionHash, session.id, sessionIdLen);
+  // void *temp = pContext->session;
+  // taosCacheRelease(server->sessionCache, (void **)&temp, false);
 
   if (pContext->session == NULL) {
     httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user,
@@ -62,26 +47,23 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
     return;
   }
 
-  pContext->session->signature = pContext->session;
+  httpTrace("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd,
-  httpTrace("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%p", pContext, pContext->fd, pContext->ipstr,
+            pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
-            pContext->user, pContext->session, pContext->session->taos);
   pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpFetchSessionImp(HttpContext *pContext) {
+static void httpFetchSessionImp(HttpContext *pContext) {
-  HttpServer *server = pContext->pThread->pServer;
+  HttpServer *server = &tsHttpServer;
   pthread_mutex_lock(&server->serverMutex);
 
   char sessionId[HTTP_SESSION_ID_LEN];
-  int sessonIdLen = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+  snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
 
-  pContext->session = taosHashGet(server->pSessionHash, sessionId, sessonIdLen);
+  pContext->session = taosCacheAcquireByName(server->sessionCache, sessionId);
-  if (pContext->session != NULL && pContext->session == pContext->session->signature) {
+  if (pContext->session != NULL) {
-    pContext->session->access++;
+    atomic_add_fetch_32(&pContext->session->refCount, 1);
-    httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, access:%d, expire:%d",
+    httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd,
-              pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session,
+              pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
-              pContext->session->taos, pContext->session->access, pContext->session->expire);
-    pContext->session->expire = (int)taosGetTimestampSec() + server->sessionExpire;
   } else {
     httpTrace("context:%p, fd:%d, ip:%s, user:%s, session not found", pContext, pContext->fd, pContext->ipstr,
               pContext->user);
@@ -90,113 +72,54 @@ void httpFetchSessionImp(HttpContext *pContext) {
   pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpFetchSession(HttpContext *pContext) {
+void httpGetSession(HttpContext *pContext) {
   if (pContext->session == NULL) {
     httpFetchSessionImp(pContext);
   } else {
     char sessionId[HTTP_SESSION_ID_LEN];
     snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
-    if (strcmp(pContext->session->id, sessionId) != 0) {
+    httpReleaseSession(pContext);
-      httpError("context:%p, fd:%d, ip:%s, user:%s, password may be changed", pContext, pContext->fd, pContext->ipstr, pContext->user);
+    httpFetchSessionImp(pContext);
-      httpRestoreSession(pContext);
-      httpFetchSessionImp(pContext);
-    }
   }
 }
 
-void httpRestoreSession(HttpContext *pContext) {
+void httpReleaseSession(HttpContext *pContext) {
-  HttpServer * server = pContext->pThread->pServer;
+  if (pContext == NULL || pContext->session == NULL) return;
 
-  // all access to the session is via serverMutex
+  int32_t refCount = atomic_sub_fetch_32(&pContext->session->refCount, 1);
-  pthread_mutex_lock(&server->serverMutex);
+  assert(refCount >= 0);
-  HttpSession *session = pContext->session;
+  httpTrace("context:%p, release session:%p:%p, sessionRef:%d", pContext, pContext->session, pContext->session->taos,
-  if (session == NULL || session != session->signature) {
+            pContext->session->refCount);
-    pthread_mutex_unlock(&server->serverMutex);
+
-    return;
+  taosCacheRelease(tsHttpServer.sessionCache, (void **)&pContext->session, false);
-  }
-  session->access--;
-  httpTrace("context:%p, ip:%s, user:%s, restore session:%p:%p, access:%d, expire:%d",
-            pContext, pContext->ipstr, pContext->user, session, session->taos,
-            session->access, pContext->session->expire);
   pContext->session = NULL;
-  pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpResetSession(HttpSession *pSession) {
+static void httpDestroySession(void *data) {
-  httpTrace("close session:%p:%p", pSession, pSession->taos);
+  HttpSession *session = data;
-  if (pSession->taos != NULL) {
+  httpTrace("session:%p:%p, is destroyed, sessionRef:%d", session, session->taos, session->refCount);
-    taos_close(pSession->taos);
+
-    pSession->taos = NULL;
+  if (session->taos != NULL) {
+    taos_close(session->taos);
+    session->taos = NULL;
   }
-  pSession->signature = NULL;
 }
 
-void httpRemoveAllSessions(HttpServer *pServer) {
+void httpCleanUpSessions() {
-  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
+  if (tsHttpServer.sessionCache != NULL) {
-
+    SCacheObj *cache = tsHttpServer.sessionCache;
-  while (taosHashIterNext(pIter)) {
+    httpPrint("session cache is cleanuping, size:%zu", taosHashGetSize(cache->pHashTable));
-    HttpSession *pSession = taosHashIterGet(pIter);
+    taosCacheCleanup(tsHttpServer.sessionCache);
-    if (pSession == NULL) continue;
+    tsHttpServer.sessionCache = NULL;
-    httpResetSession(pSession);
   }
-
-  taosHashDestroyIter(pIter);
 }
 
-bool httpInitAllSessions(HttpServer *pServer) {
+bool httpInitSessions() {
-  if (pServer->pSessionHash == NULL) {
+  tsHttpServer.sessionCache = taosCacheInitWithCb(5, httpDestroySession);
-    pServer->pSessionHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true);
+  if (tsHttpServer.sessionCache == NULL) {
-  }
+    httpError("failed to init session cache");
-  if (pServer->pSessionHash == NULL) {
-    httpError("http init session pool failed");
     return false;
   }
-  if (pServer->expireTimer == NULL) {
-    taosTmrReset(httpProcessSessionExpire, 50000, pServer, pServer->timerHandle, &pServer->expireTimer);
-  }
 
   return true;
 }
-
-bool httpSessionExpired(HttpSession *pSession) {
-  time_t cur = taosGetTimestampSec();
-
-  if (pSession->taos != NULL) {
-    if (pSession->expire > cur) {
-      return false;  // un-expired, so return false
-    }
-    if (pSession->access > 0) {
-      httpTrace("session:%p:%p is expired, but still access:%d", pSession, pSession->taos,
-                pSession->access);
-      return false;  // still used, so return false
-    }
-    httpTrace("need close session:%p:%p for it expired, cur:%d, expire:%d, invertal:%d",
-              pSession, pSession->taos, cur, pSession->expire, cur - pSession->expire);
-  }
-
-  return true;
-}
-
-void httpRemoveExpireSessions(HttpServer *pServer) {  
-  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
-
-  while (taosHashIterNext(pIter)) {
-    HttpSession *pSession = taosHashIterGet(pIter);
-    if (pSession == NULL) continue;
-
-    pthread_mutex_lock(&pServer->serverMutex);
-    if (httpSessionExpired(pSession)) {
-      httpResetSession(pSession);
-      taosHashRemove(pServer->pSessionHash, pSession->id, strlen(pSession->id));
-    }
-    pthread_mutex_unlock(&pServer->serverMutex);
-  }
-
-  taosHashDestroyIter(pIter);
-}
-
-void httpProcessSessionExpire(void *handle, void *tmrId) {
-  HttpServer *pServer = (HttpServer *)handle;
-  httpRemoveExpireSessions(pServer);
-  taosTmrReset(httpProcessSessionExpire, 60000, pServer, pServer->timerHandle, &pServer->expireTimer);
-}

src/plugins/http/src/httpSql.c

@@ -18,11 +18,12 @@
 #include "tnote.h"
 #include "taos.h"
 #include "tsclient.h"
-#include "http.h"
+#include "httpInt.h"
-#include "httpLog.h"
+#include "httpContext.h"
-#include "httpCode.h"
+#include "httpSql.h"
-#include "httpHandle.h"
 #include "httpResp.h"
+#include "httpAuth.h"
+#include "httpSession.h"
 
 void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, void **taos);
@@ -30,7 +31,7 @@ void httpProcessMultiSql(HttpContext *pContext);
 
 void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpSqlCmds *     multiCmds = pContext->multiCmds;
   HttpEncodeMethod *encode = pContext->encodeMethod;
@@ -72,7 +73,7 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO
 
 void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpSqlCmds *     multiCmds = pContext->multiCmds;
   HttpEncodeMethod *encode = pContext->encodeMethod;
@@ -172,7 +173,7 @@ void httpProcessMultiSql(HttpContext *pContext) {
 }
 
 void httpProcessMultiSqlCmd(HttpContext *pContext) {
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpSqlCmds *multiCmds = pContext->multiCmds;
   if (multiCmds == NULL || multiCmds->size <= 0 || multiCmds->pos >= multiCmds->size || multiCmds->pos < 0) {
@@ -192,7 +193,7 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) {
 
 void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpEncodeMethod *encode = pContext->encodeMethod;
 
@@ -230,7 +231,7 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num
 
 void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpEncodeMethod *encode = pContext->encodeMethod;
 
@@ -354,7 +355,7 @@ void httpExecCmd(HttpContext *pContext) {
 
 void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
   HttpContext *pContext = param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   if (code < 0) {
     httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%s", pContext, pContext->fd, pContext->ipstr,
@@ -383,16 +384,14 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
 }
 
 void httpProcessRequest(HttpContext *pContext) {
-  httpFetchSession(pContext);
+  httpGetSession(pContext);
 
-  if (pContext->session == NULL || pContext->session != pContext->session->signature ||
+  if (pContext->session == NULL || pContext->reqType == HTTP_REQTYPE_LOGIN) {
-      pContext->reqType == HTTP_REQTYPE_LOGIN) {
     taos_connect_a(NULL, pContext->user, pContext->pass, "", 0, httpProcessRequestCb, (void *)pContext,
                    &(pContext->taos));
     httpTrace("context:%p, fd:%d, ip:%s, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd,
               pContext->ipstr, pContext->user, pContext->taos);
   } else {
-    httpAccessSession(pContext);
     httpExecCmd(pContext);
   }
 }

src/plugins/http/src/httpSystem.c

@@ -20,84 +20,64 @@
 #include "tsocket.h"
 #include "ttimer.h"
 #include "tadmin.h"
-#include "http.h"
+#include "httpInt.h"
-#include "httpCode.h"
+#include "httpContext.h"
-#include "httpHandle.h"
+#include "httpSession.h"
+#include "httpServer.h"
 #include "httpResp.h"
-#include "httpLog.h"
-#include "gcHandle.h"
 #include "httpHandle.h"
+#include "gcHandle.h"
 #include "restHandle.h"
 #include "tgHandle.h"
 
 #ifndef _ADMIN
-
 void adminInitHandle(HttpServer* pServer) {}
 void opInitHandle(HttpServer* pServer) {}
-
 #endif
 
-static HttpServer *httpServer = NULL;
+HttpServer tsHttpServer;
 void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
 
 int httpInitSystem() {
-  // taos_init();
+  strcpy(tsHttpServer.label, "rest");
+  tsHttpServer.serverIp = 0;
+  tsHttpServer.serverPort = tsHttpPort;
+  tsHttpServer.numOfThreads = tsHttpMaxThreads;
+  tsHttpServer.processData = httpProcessData;
 
-  httpServer = (HttpServer *)malloc(sizeof(HttpServer));
+  pthread_mutex_init(&tsHttpServer.serverMutex, NULL);
-  memset(httpServer, 0, sizeof(HttpServer));
-
-  strcpy(httpServer->label, "rest");
-  httpServer->serverIp = 0;
-  httpServer->serverPort = tsHttpPort;
-  httpServer->cacheContext = tsHttpCacheSessions;
-  httpServer->sessionExpire = tsHttpSessionExpire;
-  httpServer->numOfThreads = tsHttpMaxThreads;
-  httpServer->processData = httpProcessData;
-
-  pthread_mutex_init(&httpServer->serverMutex, NULL);
 
   if (tsHttpEnableRecordSql != 0) {
     taosInitNote(tsNumOfLogLines / 10, 1, (char*)"http_note");
   }
-  restInitHandle(httpServer);
+  restInitHandle(&tsHttpServer);
-  adminInitHandle(httpServer);
+  adminInitHandle(&tsHttpServer);
-  gcInitHandle(httpServer);
+  gcInitHandle(&tsHttpServer);
-  tgInitHandle(httpServer);
+  tgInitHandle(&tsHttpServer);
-  opInitHandle(httpServer);
+  opInitHandle(&tsHttpServer);
 
   return 0;
 }
 
 int httpStartSystem() {
-  httpPrint("starting to initialize http service ...");
+  httpPrint("start http server ...");
 
-  if (httpServer == NULL) {
+  if (tsHttpServer.status != HTTP_SERVER_INIT) {
-    httpError("http server is null");
+    httpError("http server is already started");
-    httpInitSystem();
-  }
-
-  if (httpServer->pContextPool == NULL) {
-    httpServer->pContextPool = taosMemPoolInit(httpServer->cacheContext, sizeof(HttpContext));
-  }
-  if (httpServer->pContextPool == NULL) {
-    httpError("http init context pool failed");
     return -1;
   }
 
-  if (httpServer->timerHandle == NULL) {
+  if (!httpInitContexts()) {
-    httpServer->timerHandle = taosTmrInit(tsHttpCacheSessions * 100 + 100, 200, 60000, "http");
+    httpError("http init contexts failed");
-  }
-  if (httpServer->timerHandle == NULL) {
-    httpError("http init timer failed");
     return -1;
   }
 
-  if (!httpInitAllSessions(httpServer)) {
+  if (!httpInitSessions()) {
     httpError("http init session failed");
     return -1;
   }
 
-  if (!httpInitConnect(httpServer)) {
+  if (!httpInitConnect()) {
     httpError("http init server failed");
     return -1;
   }
@@ -106,53 +86,23 @@ int httpStartSystem() {
 }
 
 void httpStopSystem() {
-  if (httpServer != NULL) {
+  tsHttpServer.status = HTTP_SERVER_CLOSING;
-    httpServer->online = false;
+  shutdown(tsHttpServer.fd, SHUT_RD);
-  }
   tgCleanupHandle();
 }
 
 void httpCleanUpSystem() {
-  httpPrint("http service cleanup");
+  httpPrint("http server cleanup");
   httpStopSystem();
 
-//#if 0
+  httpCleanupContexts();
-  if (httpServer == NULL) {
+  httpCleanUpSessions();
-    return;
+  httpCleanUpConnect();
-  }
+  pthread_mutex_destroy(&tsHttpServer.serverMutex);
 
-  if (httpServer->expireTimer != NULL) {
+  tsHttpServer.status = HTTP_SERVER_CLOSED;
-    taosTmrStopA(&(httpServer->expireTimer));
-  }
-
-  if (httpServer->timerHandle != NULL) {
-    taosTmrCleanUp(httpServer->timerHandle);
-    httpServer->timerHandle = NULL;
-  }
-
-  if (httpServer->pThreads != NULL) {
-    httpCleanUpConnect(httpServer);
-    httpServer->pThreads = NULL;
-  }
-  
-
-#if 0
-  httpRemoveAllSessions(httpServer);
-
-  if (httpServer->pContextPool != NULL) {
-    taosMemPoolCleanUp(httpServer->pContextPool);
-    httpServer->pContextPool = NULL;
-  }
-
-  pthread_mutex_destroy(&httpServer->serverMutex);
-
-  tfree(httpServer);
-#endif
 }
 
 int32_t httpGetReqCount() {
-  if (httpServer != NULL) {
+  return atomic_exchange_32(&tsHttpServer.requestNum, 0);
-    return atomic_exchange_32(&httpServer->requestNum, 0);
-  }
-  return 0;
 }

src/plugins/http/src/httpUtil.c

@@ -17,11 +17,10 @@
 #include "os.h"
 #include "tmd5.h"
 #include "taos.h"
-#include "http.h"
+#include "httpInt.h"
-#include "httpLog.h"
-#include "httpCode.h"
-#include "httpHandle.h"
 #include "httpResp.h"
+#include "httpSql.h"
+#include "httpUtil.h"
 
 bool httpCheckUsedbSql(char *sql) {
   if (strstr(sql, "use ") != NULL) {
@@ -203,8 +202,7 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize) {
               pContext->user, cmdSize);
     return false;
   }
-  memset(multiCmds->cmds + multiCmds->maxSize * (int16_t)sizeof(HttpSqlCmd), 0,
+  memset(multiCmds->cmds + multiCmds->maxSize, 0, (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd));
-         (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd));
   multiCmds->maxSize = (int16_t)cmdSize;
 
   return true;

src/plugins/http/src/restHandle.c

@@ -65,7 +65,7 @@ bool restGetUserFromUrl(HttpContext* pContext) {
     return false;
   }
 
-  strcpy(pContext->user, pParser->path[REST_USER_URL_POS].pos);
+  tstrncpy(pContext->user, pParser->path[REST_USER_URL_POS].pos, TSDB_USER_LEN);
   return true;
 }
 
@@ -75,7 +75,7 @@ bool restGetPassFromUrl(HttpContext* pContext) {
     return false;
   }
 
-  strcpy(pContext->pass, pParser->path[REST_PASS_URL_POS].pos);
+  tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
   return true;
 }
 

src/plugins/http/src/tgHandle.c

@@ -18,9 +18,10 @@
 #include "tglobal.h"
 #include "taosdef.h"
 #include "taosmsg.h"
+#include "httpInt.h"
 #include "tgHandle.h"
 #include "tgJson.h"
-#include "httpLog.h"
+#include "cJSON.h"
 
 /*
  * taos.telegraf.cfg formats like
@@ -267,10 +268,10 @@ int tgReadSchema(char *fileName) {
 
   httpPrint("open telegraf schema file:%s success", fileName);
   fseek(fp, 0, SEEK_END);
-  size_t contentSize = (size_t)ftell(fp);
+  int32_t contentSize = (int32_t)ftell(fp);
   rewind(fp);
-  char *content = (char *)calloc(contentSize * sizeof(char) + 1, 1);
+  char *  content = (char *)calloc(contentSize + 1, 1);
-  size_t result = fread(content, 1, contentSize, fp);
+  int32_t result = fread(content, 1, contentSize, fp);
   if (result != contentSize) {
     httpError("failed to read telegraf schema file:%s", fileName);
     fclose(fp);
@@ -278,6 +279,7 @@ int tgReadSchema(char *fileName) {
     return -1;
   }
 
+  content[contentSize] = 0;
   int schemaNum = tgParseSchema(content, fileName);
 
   free(content);

src/plugins/monitor/src/monitorMain.c

@@ -61,7 +61,7 @@ typedef struct {
   char   ep[TSDB_EP_LEN];
   int8_t cmdIndex;
   int8_t state;
-  char   sql[SQL_LENGTH];
+  char   sql[SQL_LENGTH + 1];
   void * initTimer;
   void * diskTimer;
 } SMonitorConn;
@@ -177,8 +177,8 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
              ") tags (acctId binary(%d))",
              tsMonitorDbName, TSDB_USER_LEN);
   } else if (cmd == MONITOR_CMD_CREATE_TB_ACCT_ROOT) {
-    snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, "root",
+    snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, TSDB_DEFAULT_USER,
-             tsMonitorDbName, "root");
+             tsMonitorDbName, TSDB_DEFAULT_USER);
   } else if (cmd == MONITOR_CMD_CREATE_TB_SLOWQUERY) {
     snprintf(sql, SQL_LENGTH,
              "create table if not exists %s.slowquery(ts timestamp, username "
@@ -208,7 +208,7 @@ static void monitorInitDatabase() {
 
 static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) {
   if (-code == TSDB_CODE_MND_TABLE_ALREADY_EXIST || -code == TSDB_CODE_MND_DB_ALREADY_EXIST || code >= 0) {
-    monitorTrace("monitor:%p, sql success, reason:%d, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql);
+    monitorTrace("monitor:%p, sql success, reason:%s, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql);
     if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) {
       monitorPrint("dnode:%s is started", tsLocalEp);
     }

src/plugins/mqtt/src/mqttSystem.c

@@ -164,7 +164,7 @@ void mqtt_PublishCallback(void** unused, struct mqtt_response_publish* published
 void* mqttClientRefresher(void* client) {
   while (mttIsRuning) {
     mqtt_sync((struct mqtt_client*)client);
-    usleep(100000U);
+    taosMsleep(100);
   }
   mqttPrint("Exit  mqttClientRefresher");
   return NULL;

src/query/inc/qextbuffer.h

@@ -149,7 +149,7 @@ int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRow
  * @param pMemBuffer
  * @return
  */
-bool tExtMemBufferFlush(tExtMemBuffer *pMemBuffer);
+int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer);
 
 /**
  *

src/query/inc/qtsbuf.h

@@ -100,10 +100,10 @@ typedef struct STSBuf {
 typedef struct STSBufFileHeader {
   uint32_t magic;       // file magic number
   uint32_t numOfVnode;  // number of vnode stored in current file
-  uint32_t tsOrder;     // timestamp order in current file
+  int32_t  tsOrder;     // timestamp order in current file
 } STSBufFileHeader;
 
-STSBuf* tsBufCreate(bool autoDelete);
+STSBuf* tsBufCreate(bool autoDelete, int32_t order);
 STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
 STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder);
 

src/query/inc/sql.y

@@ -251,12 +251,20 @@ alter_db_optr(Y) ::= alter_db_optr(Z) comp(X).        { Y = Z; Y.compressionLeve
 alter_db_optr(Y) ::= alter_db_optr(Z) wal(X).         { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
 
 %type typename {TAOS_FIELD}
-typename(A) ::= ids(X).              { tSQLSetColumnType (&A, &X); }
+typename(A) ::= ids(X). { 
+  X.type = 0;
+  tSQLSetColumnType (&A, &X); 
+}
 
 //define binary type, e.g., binary(10), nchar(10)
 typename(A) ::= ids(X) LP signed(Y) RP.    {
-    X.type = -Y;          // negative value of name length
+    if (Y <= 0) {
-    tSQLSetColumnType(&A, &X);
+      X.type = 0;
+      tSQLSetColumnType(&A, &X);
+    } else {
+      X.type = -Y;          // negative value of name length
+      tSQLSetColumnType(&A, &X);
+    }
 }
 
 %type signed {int64_t}

src/query/src/qExecutor.c

@@ -12,6 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include <taosmsg.h>
 #include "os.h"
 #include "qfill.h"
 
@@ -272,7 +273,7 @@ static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) {
   if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) {
     pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total;
     
-    qTrace("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%d, total:%"PRId64,
+    qTrace("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64,
         pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
     assert(pQuery->rec.rows >= 0);
     setQueryStatus(pQuery, QUERY_COMPLETED);
@@ -847,7 +848,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
   }
 
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
-  if (isIntervalQuery(pQuery)) {
+  if (isIntervalQuery(pQuery) && tsCols != NULL) {
     int32_t offset = GET_COL_DATA_POS(pQuery, 0, step);
     TSKEY   ts = tsCols[offset];
 
@@ -1089,7 +1090,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
   // from top to bottom in desc
   // from bottom to top in asc order
   if (pRuntimeEnv->pTSBuf != NULL) {
-    SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery);
+    SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv);
     qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows,
            pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order);
   }
@@ -1215,7 +1216,6 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
 
   // interval query with limit applied
   int32_t numOfRes = 0;
-  
   if (isIntervalQuery(pQuery)) {
     numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo);
   } else {
@@ -1399,7 +1399,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
       pCtx->inputType = pQuery->colList[index].type;
     }
   
-    assert(isValidDataType(pCtx->inputType, pCtx->inputBytes));
+    assert(isValidDataType(pCtx->inputType));
     pCtx->ptsOutputBuf = NULL;
 
     pCtx->outputBytes = pQuery->pSelectExpr[i].bytes;
@@ -2060,7 +2060,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa
     pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data;
   }
 
-  qTrace("QInfo:%p realloc output buffer to inc output buffer from: %d rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv),
+  qTrace("QInfo:%p realloc output buffer to inc output buffer from: %" PRId64 " rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv),
          pQuery->rec.capacity, capacity);
 
   pQuery->rec.capacity = capacity;
@@ -2096,7 +2096,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
         }
       }
       
-      qTrace("QInfo:%p realloc output buffer, new size: %d rows, old:%d, remain:%d", GET_QINFO_ADDR(pRuntimeEnv),
+      qTrace("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv),
              newSize, pRec->capacity, newSize - pRec->rows);
       
       pRec->capacity = newSize;
@@ -2188,8 +2188,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
  * set tag value in SQLFunctionCtx
  * e.g.,tag information into input buffer
  */
-static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColId, tVariant *tag, int16_t type,
+static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) {
-    int16_t bytes) {
   tVariantDestroy(tag);
 
   if (tagColId == TSDB_TBNAME_COLUMN_INDEX) {
@@ -2205,8 +2204,18 @@ static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColI
     }
     
     if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+      if (isNull(varDataVal(val), type)) {
+        tag->nType = TSDB_DATA_TYPE_NULL;
+        return;
+      }
+
       tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
     } else {
+      if (isNull(val, type)) {
+        tag->nType = TSDB_DATA_TYPE_NULL;
+        return;
+      }
+
       tVariantCreateFromBinary(tag, val, bytes, type);
     }
   }
@@ -2214,35 +2223,55 @@ static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColI
 
 void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
+  SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
 
   SExprInfo *pExprInfo = &pQuery->pSelectExpr[0];
   if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) {
-  
     assert(pExprInfo->base.numOfParams == 1);
-    doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag,
+
-                         pExprInfo->type, pExprInfo->bytes);
+    // todo refactor extract function.
+    int16_t type = -1, bytes = -1;
+    for(int32_t i = 0; i < pQuery->numOfTags; ++i) {
+      if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) {
+        type = pQuery->tagColList[i].type;
+        bytes = pQuery->tagColList[i].bytes;
+      }
+    }
+
+    doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
   } else {
     // set tag value, by which the results are aggregated.
     for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) {
-      SExprInfo* pExprInfo = &pQuery->pSelectExpr[idx];
+      SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx];
   
       // ts_comp column required the tag value for join filter
-      if (!TSDB_COL_IS_TAG(pExprInfo->base.colInfo.flag)) {
+      if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) {
         continue;
       }
 
       // todo use tag column index to optimize performance
-      doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
+      doSetTagValueInParam(tsdb, pTableId, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag,
-          pExprInfo->type, pExprInfo->bytes);
+                           pLocalExprInfo->type, pLocalExprInfo->bytes);
     }
 
     // set the join tag for first column
     SSqlFuncMsg *pFuncMsg = &pExprInfo->base;
-    if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
+    if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
         pRuntimeEnv->pTSBuf != NULL) {
       assert(pFuncMsg->numOfParams == 1);
-      assert(0);  // to do fix me
+
-      //      doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag);
+      // todo refactor
+      int16_t type = -1, bytes = -1;
+      for(int32_t i = 0; i < pQuery->numOfTags; ++i) {
+        if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) {
+          type = pQuery->tagColList[i].type;
+          bytes = pQuery->tagColList[i].bytes;
+        }
+      }
+
+      doSetTagValueInParam(tsdb, pTableId, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes);
+      qTrace("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64,
+          pRuntimeEnv->pCtx[0].tag.i64Key)
     }
   }
 }
@@ -2465,7 +2494,7 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
     qTrace("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1);
   }
 
-  qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", pQInfo,
+  qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo,
          pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st);
 
   return TSDB_CODE_SUCCESS;
@@ -2923,7 +2952,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
   }
 
   if (pQuery->rec.rows <= pQuery->limit.offset) {
-    qTrace("QInfo:%p skip rows:%d, new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows,
+    qTrace("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows,
         pQuery->limit.offset - pQuery->rec.rows);
     
     pQuery->limit.offset -= pQuery->rec.rows;
@@ -3335,7 +3364,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
 
 int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-  assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL);
+  //assert(pTableQueryInfo->lastKey >= TSKEY_INITIAL_VAL);
 
   setTagVal(pRuntimeEnv, pTableId, pQInfo->tsdb);
 
@@ -3617,6 +3646,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
 
 static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
+
   for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
     int32_t bytes = pQuery->pSelectExpr[col].bytes;
 
@@ -3666,7 +3696,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
     }
 
     if (pQuery->limit.offset < ret) {
-      qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%d. Discard due to offset, remain:%d, new offset:%d",
+      qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d",
              pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0);
       
       ret -= pQuery->limit.offset;
@@ -3680,8 +3710,8 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
       pQuery->limit.offset = 0;
       return ret;
     } else {
-      qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%d. Discard due to offset, "
+      qTrace("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, "
-             "remain:%d, new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0,
+             "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0,
           pQuery->limit.offset - ret);
       
       pQuery->limit.offset -= ret;
@@ -4229,8 +4259,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     while (pQInfo->groupIndex < numOfGroups) {
       SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
 
-      qTrace("QInfo:%p last_row query on group:%d, total group:%d, current group:%d", pQInfo, pQInfo->groupIndex,
+      qTrace("QInfo:%p last_row query on group:%d, total group:%zu, current group:%p", pQInfo, pQInfo->groupIndex,
-             numOfGroups);
+             numOfGroups, group);
 
       STsdbQueryCond cond = {
           .twindow = pQuery->window,
@@ -4263,11 +4293,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       assert(taosArrayGetSize(s) >= 1);
       
       setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
-      
       if (isFirstLastRowQuery(pQuery)) {
         assert(taosArrayGetSize(s) == 1);
       }
-      
+
+      taosArrayDestroy(s);
+
       // here we simply set the first table as current table
       pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
       scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
@@ -4293,7 +4324,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     while (pQInfo->groupIndex < numOfGroups) {
       SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
 
-      qTrace("QInfo:%p group by normal columns group:%d, total group:%d", pQInfo, pQInfo->groupIndex, numOfGroups);
+      qTrace("QInfo:%p group by normal columns group:%d, total group:%zu", pQInfo, pQInfo->groupIndex, numOfGroups);
 
       STsdbQueryCond cond = {
           .twindow = pQuery->window,
@@ -4328,6 +4359,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
         // no results generated for current group, continue to try the next group
+      taosArrayDestroy(s); 
       if (pWindowResInfo->size <= 0) {
         continue;
       }
@@ -4452,6 +4484,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         }
       }
     }
+
+    if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
+      setQueryStatus(pQuery, QUERY_COMPLETED);
+    }
   }
 
   /*
@@ -4474,7 +4510,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
   }
 
   qTrace(
-      "QInfo %p numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64,
+      "QInfo %p numOfTables:%"PRIu64", index:%d, numOfGroups:%zu, %"PRId64" points returned, total:%"PRId64", offset:%" PRId64,
       pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
       pQuery->limit.offset);
 }
@@ -4561,7 +4597,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
       copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
     }
 
-    qTrace("QInfo:%p current:%lld, total:%lld", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+    qTrace("QInfo:%p current:%"PRId64", total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total);
     return;
   }
 
@@ -4570,7 +4606,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
 
   // do check all qualified data blocks
   int64_t el = scanMultiTableDataBlocks(pQInfo);
-  qTrace("QInfo:%p master scan completed, elapsed time: %lldms, reverse scan start", pQInfo, el);
+  qTrace("QInfo:%p master scan completed, elapsed time: %" PRId64 "ms, reverse scan start", pQInfo, el);
 
   // query error occurred or query is killed, abort current execution
   if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) {
@@ -4585,7 +4621,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
     doSaveContext(pQInfo);
 
     el = scanMultiTableDataBlocks(pQInfo);
-    qTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el);
+    qTrace("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el);
 
     doRestoreContext(pQInfo);
   } else {
@@ -4612,7 +4648,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
   }
 
   // handle the limitation of output buffer
-  qTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
+  qTrace("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
 }
 
 /*
@@ -4684,8 +4720,8 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
       break;
     }
 
-    qTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64,
+    qTrace("QInfo:%p skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64,
-           pQInfo, pQuery->limit.offset, pQuery->current->lastKey);
+           pQInfo, pQuery->limit.offset, pQuery->current->lastKey, pQuery->current->win.ekey);
 
     resetCtxOutputBuf(pRuntimeEnv);
   }
@@ -4813,7 +4849,7 @@ static void tableQueryImpl(SQInfo *pQInfo) {
       limitResults(pRuntimeEnv);
     }
     
-    qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+    qTrace("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total);
     return;
   }
 
@@ -4833,13 +4869,13 @@ static void tableQueryImpl(SQInfo *pQInfo) {
         clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex);
 
         if (pQuery->rec.rows > 0) {
-          qTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+          qTrace("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total);
           return;
         }
       }
     }
 
-    qTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total);
+    qTrace("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total);
     return;
   }
 
@@ -4895,7 +4931,7 @@ static void stableQueryImpl(SQInfo *pQInfo) {
   pQInfo->runtimeEnv.summary.elapsedTime += (taosGetTimestampUs() - st);
   
   if (pQuery->rec.rows == 0) {
-    qTrace("QInfo:%p over, %d tables queried, %d rows are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total);
+    qTrace("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total);
   }
 }
 
@@ -5019,7 +5055,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
 
   pQueryMsg->order = htons(pQueryMsg->order);
   pQueryMsg->orderColId = htons(pQueryMsg->orderColId);
-  pQueryMsg->queryType = htons(pQueryMsg->queryType);
+  pQueryMsg->queryType = htonl(pQueryMsg->queryType);
   pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType);
 
   pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols);
@@ -5038,7 +5074,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
   }
 
   char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols;
-
   for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) {
     SColumnInfo *pColInfo = &pQueryMsg->colList[col];
 
@@ -5188,17 +5223,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
     pMsg += len;
   }
 
-  qTrace("qmsg:%p query %d tables, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, "
+  qTrace("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, "
-         "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64,
+         "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64,
-         pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
+         pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
          pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime,
-         pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset);
+         pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset);
 
   return 0;
 }
 
 static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) {
-  qTrace("qmsg:%p create arithmetic expr from binary string", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz);
+  qTrace("qmsg:%p create arithmetic expr from binary string: %s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz);
 
   tExprNode* pExprNode = NULL;
   TRY(32) {
@@ -5278,7 +5313,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
     if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) {
       tagLen += pExprs[i].bytes;
     }
-    assert(isValidDataType(pExprs[i].type, pExprs[i].bytes));
+    assert(isValidDataType(pExprs[i].type));
   }
 
   // TODO refactor
@@ -5378,6 +5413,7 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
 
         if ((lower == TSDB_RELATION_GREATER_EQUAL || lower == TSDB_RELATION_GREATER) &&
             (upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) {
+          assert(rangeFilterArray != NULL);
           if (lower == TSDB_RELATION_GREATER_EQUAL) {
             if (upper == TSDB_RELATION_LESS_EQUAL) {
               pSingleColFilter->fp = rangeFilterArray[4];
@@ -5392,11 +5428,12 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
             }
           }
         } else {  // set callback filter function
+          assert(filterArray != NULL);
           if (lower != TSDB_RELATION_INVALID) {
             pSingleColFilter->fp = filterArray[lower];
 
             if (upper != TSDB_RELATION_INVALID) {
-              qError("pQInfo:%p failed to get filter function, invalid filter condition", pQInfo, type);
+              qError("pQInfo:%p failed to get filter function, invalid filter condition: %d", pQInfo, type);
               return TSDB_CODE_QRY_INVALID_MSG;
             }
           } else {
@@ -5638,7 +5675,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
 
   STSBuf *pTSBuf = NULL;
   if (pQueryMsg->tsLen > 0) {  // open new file to save the result
-    char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset;
+    char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset;
     pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder);
 
     tsBufResetPos(pTSBuf);
@@ -5816,7 +5853,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
   }
 
   pQuery->rec.total += pQuery->rec.rows;
-  qTrace("QInfo:%p current numOfRes rows:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+  qTrace("QInfo:%p current numOfRes rows:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total);
 
   if (pQuery->limit.limit > 0 && pQuery->limit.limit == pQuery->rec.total) {
     qTrace("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit);
@@ -5869,23 +5906,20 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
   bool isSTableQuery = false;
   STableGroupInfo groupInfo = {0};
   
-  //todo multitable_query??
+  if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) {
-  if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_TABLE_QUERY)) {
-    isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
-    
     STableIdInfo *id = taosArrayGet(pTableIdList, 0);
-    qTrace("qmsg:%p query table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
+
-    
+    qTrace("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
     if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }
-  } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_STABLE_QUERY)) {
+  } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) {
     isSTableQuery = true;
-    // TODO: need a macro from TSDB to check if table is super table,
+    // TODO: need a macro from TSDB to check if table is super table
-    // also note there's possiblity that only one table in the super table
+
-    if (taosArrayGetSize(pTableIdList) == 1) {
+    // also note there's possibility that only one table in the super table
+    if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) {
       STableIdInfo *id = taosArrayGet(pTableIdList, 0);
-      // if array size is 1 and assert super table
 
       // group by normal column, do not pass the group by condition to tsdb to group table into different group
       int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols;
@@ -5899,15 +5933,13 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
         goto _over;
       }
     } else {
-      SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
+      groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES);
+      groupInfo.numOfTables = taosArrayGetSize(pTableIdList);
 
-      SArray* sa = taosArrayInit(groupInfo.numOfTables, sizeof(STableId));
+      SArray* p = taosArrayClone(pTableIdList);
-      for(int32_t i = 0; i < groupInfo.numOfTables; ++i) {
+      taosArrayPush(groupInfo.pGroupList, &p);
-        STableIdInfo* tableId = taosArrayGet(pTableIdList, i);
+
-        taosArrayPush(sa, tableId);
+      qTrace("qmsg:%p query on %zu tables in one group from client", pQueryMsg, groupInfo.numOfTables);
-      }
-      taosArrayPush(pTableGroup, &sa);
-      groupInfo.pGroupList = pTableGroup;
     }
   } else {
     assert(0);
@@ -5948,7 +5980,7 @@ static void doDestoryQueryInfo(SQInfo* pQInfo) {
   freeQInfo(pQInfo);
 }
 
-void qDestroyQueryInfo(qinfo_t qHandle) {
+void qDestroyQueryInfo(qinfo_t qHandle, void (*fp)(void*), void* param) {
   SQInfo* pQInfo = (SQInfo*) qHandle;
   if (!isValidQInfo(pQInfo)) {
     return;
@@ -5959,10 +5991,14 @@ void qDestroyQueryInfo(qinfo_t qHandle) {
 
   if (ref == 0) {
     doDestoryQueryInfo(pQInfo);
+
+    if (fp != NULL) {
+      fp(param);
+    }
   }
 }
 
-void qTableQuery(qinfo_t qinfo) {
+void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || pQInfo->signature != pQInfo) {
@@ -5972,7 +6008,7 @@ void qTableQuery(qinfo_t qinfo) {
 
   if (isQueryKilled(pQInfo)) {
     qTrace("QInfo:%p it is already killed, abort", pQInfo);
-    qDestroyQueryInfo(pQInfo);
+    qDestroyQueryInfo(pQInfo, fp, param);
     return;
   }
 
@@ -5988,7 +6024,7 @@ void qTableQuery(qinfo_t qinfo) {
   }
 
   sem_post(&pQInfo->dataReady);
-  qDestroyQueryInfo(pQInfo);
+  qDestroyQueryInfo(pQInfo, fp, param);
 }
 
 int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
@@ -6005,7 +6041,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
   }
 
   sem_wait(&pQInfo->dataReady);
-  qTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
+  qTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
          pQInfo->code);
 
   return pQInfo->code;
@@ -6081,7 +6117,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   return code;
 }
 
-int32_t qKillQuery(qinfo_t qinfo) {
+int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6089,7 +6125,7 @@ int32_t qKillQuery(qinfo_t qinfo) {
   }
 
   setQueryKilled(pQInfo);
-  qDestroyQueryInfo(pQInfo);
+  qDestroyQueryInfo(pQInfo, fp, param);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -6119,6 +6155,17 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
     int32_t rsize = pExprInfo->bytes;
     count = 0;
 
+    int16_t bytes = pExprInfo->bytes;
+    int16_t type = pExprInfo->type;
+
+    for(int32_t i = 0; i < pQuery->numOfTags; ++i) {
+      if (pQuery->tagColList[i].colId == pExprInfo->base.colInfo.colId) {
+        bytes = pQuery->tagColList[i].bytes;
+        type = pQuery->tagColList[i].type;
+        break;
+      }
+    }
+
     while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) {
       int32_t i = pQInfo->tableIndex++;
       SGroupItem *item = taosArrayGet(pa, i);
@@ -6136,9 +6183,6 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
       *(int32_t *)output = pQInfo->vgId;
       output += sizeof(pQInfo->vgId);
 
-      int16_t bytes = pExprInfo->bytes;
-      int16_t type = pExprInfo->type;
-
       if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
         char *data = tsdbGetTableName(pQInfo->tsdb, &item->id);
         memcpy(output, data, varDataTLen(data));
@@ -6155,7 +6199,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
         } else {
           if (val == NULL) {
             setNull(output, type, bytes);
-          } else {
+          } else {  // todo here stop will cause client crash
             memcpy(output, val, bytes);
           }
         }

src/query/src/qast.c

@@ -496,7 +496,6 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
     printf("relation is like\n");
     assert(0);
   }
-  
   return TSDB_CODE_SUCCESS;
 }
 
@@ -511,7 +510,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
   if (cond.start != NULL) {
     iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
   } else {
-    iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.end->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+    iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC);
   }
   
   if (cond.start != NULL) {
@@ -578,8 +577,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
       assert(0);
     }
   } else {
-    int32_t optr = cond.end->optr;
+    int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID;
-    
     if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) {
       bool comp = true;
       int32_t ret = 0;
@@ -601,6 +599,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
       }
     }
   }
+  free(cond.start); 
+  free(cond.end);
+  tSkipListDestroyIter(iter);
 }
 
 int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
@@ -748,6 +749,7 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTravers
   }
   
   taosArrayCopy(pResult, array);
+  taosArrayDestroy(array);
 }
 
 static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) {
@@ -1182,4 +1184,4 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
 
   CLEANUP_EXECUTE_TO(anchor, false);
   return expr;
-}
+}

src/query/src/qextbuffer.c

@@ -245,30 +245,29 @@ static void tExtMemBufferClearFlushoutInfo(tExtMemBuffer *pMemBuffer) {
   memset(pFileMeta->flushoutData.pFlushoutInfo, 0, sizeof(tFlushoutInfo) * pFileMeta->flushoutData.nAllocSize);
 }
 
-bool tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
+int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
+  int32_t ret = 0;
   if (pMemBuffer->numOfTotalElems == 0) {
-    return true;
+    return ret;
   }
 
   if (pMemBuffer->file == NULL) {
     if ((pMemBuffer->file = fopen(pMemBuffer->path, "wb+")) == NULL) {
-      return false;
+      ret = TAOS_SYSTEM_ERROR(errno);
+      return ret;
     }
   }
 
   /* all data has been flushed to disk, ignore flush operation */
   if (pMemBuffer->numOfElemsInBuffer == 0) {
-    return true;
+    return ret;
   }
 
-  bool ret = true;
-  
   tFilePagesItem *first = pMemBuffer->pHead;
-
   while (first != NULL) {
     size_t retVal = fwrite((char *)&(first->item), pMemBuffer->pageSize, 1, pMemBuffer->file);
     if (retVal <= 0) {  // failed to write to buffer, may be not enough space
-      ret = false;
+      ret = TAOS_SYSTEM_ERROR(errno);
     }
 
     pMemBuffer->fileMeta.numOfElemsInFile += first->item.num;
@@ -896,6 +895,7 @@ void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int3
           char buf[4096] = {0};
           taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf);
           printf("%s\t", buf);
+          break;
         }
         case TSDB_DATA_TYPE_BINARY: {
           printBinaryData(val, pModel->pFields[j].field.bytes);
@@ -947,6 +947,7 @@ void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, in
           char buf[128] = {0};
           taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf);
           printf("%s\t", buf);
+          break;
         }
         case TSDB_DATA_TYPE_BINARY: {
           printBinaryDataEx(val, pModel->pFields[j].field.bytes, &param[j]);

src/query/src/qparserImpl.c

@@ -497,10 +497,18 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) {
          * number of bytes in UCS-4 format, which is 4 times larger than the
          * number of characters
          */
-        pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR;
+        if (type->type == 0) {
+          pField->bytes = 0;
+        } else {
+          pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR;
+        }
       } else if (i == TSDB_DATA_TYPE_BINARY) {
         /* for binary, the TOKENTYPE is the length of binary */
-        pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR;
+        if (type->type == 0) {
+          pField->bytes = 0;
+        } else {
+          pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR;
+        } 
       }
       break;
     }
@@ -775,19 +783,14 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) {
 
   while (nParam-- > 0) {
     SSQLToken *pToken = va_arg(va, SSQLToken *);
-    (void)tTokenListAppend(pInfo->pDCLInfo, pToken);
+    pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken);
   }
   va_end(va);
 }
 
 void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck) {
   pInfo->type = type;
-  
+  pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken);
-  if (pInfo->pDCLInfo == NULL) {
-    pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL));
-  }
-  
-  tTokenListAppend(pInfo->pDCLInfo, pToken);
   pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1);
 }
 

src/query/src/qpercentile.c

@@ -241,7 +241,6 @@ void tBucketDoubleHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_
 tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nElemSize, int16_t dataType,
                              tOrderDescriptor *pDesc) {
   tMemBucket *pBucket = (tMemBucket *)malloc(sizeof(tMemBucket));
-
   pBucket->nTotalSlots = totalSlots;
   pBucket->nSlotsOfSeg = 1 << 6;  // 64 Segments, 16 slots each seg.
   pBucket->dataType = dataType;
@@ -258,6 +257,7 @@ tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE
   pBucket->numOfTotalPages = pBucket->nTotalBufferSize / pBucket->pageSize;
   pBucket->numOfAvailPages = pBucket->numOfTotalPages;
 
+  pBucket->pSegs = NULL;
   pBucket->pOrderDesc = pDesc;
 
   switch (pBucket->dataType) {
@@ -283,7 +283,7 @@ tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE
       break;
     };
     default: {
-      uError("MemBucket:%p,not support data type %d,failed", *pBucket, pBucket->dataType);
+      uError("MemBucket:%p,not support data type %d,failed", pBucket, pBucket->dataType);
       tfree(pBucket);
       return NULL;
     }
@@ -315,7 +315,7 @@ tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE
     pBucket->pSegs[i].pBoundingEntries = NULL;
   }
 
-  uTrace("MemBucket:%p,created,buffer size:%d,elem size:%d", pBucket, pBucket->numOfTotalPages * DEFAULT_PAGE_SIZE,
+  uTrace("MemBucket:%p,created,buffer size:%ld,elem size:%d", pBucket, pBucket->numOfTotalPages * DEFAULT_PAGE_SIZE,
          pBucket->nElemSize);
 
   return pBucket;
@@ -751,7 +751,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
 
           char * thisVal = buffer->data + pMemBucket->nElemSize * currentIdx;
           char * nextVal = thisVal + pMemBucket->nElemSize;
-          double td, nd;
+          double td = 1.0, nd = 1.0;
           switch (pMemBucket->dataType) {
             case TSDB_DATA_TYPE_SMALLINT: {
               td = *(int16_t *)thisVal;

src/query/src/qresultBuf.c

@@ -6,7 +6,12 @@
 #include "queryLog.h"
 
 int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) {
-  SDiskbasedResultBuf* pResBuf = calloc(1, sizeof(SDiskbasedResultBuf));
+  *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
+  SDiskbasedResultBuf* pResBuf = *pResultBuf;
+  if (pResBuf == NULL) {
+    return TSDB_CODE_COM_OUT_OF_MEMORY;  
+  }
+  
   pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize;
   pResBuf->numOfPages = size;
 
@@ -46,7 +51,6 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t si
   qTrace("QInfo:%p create tmp file for output result, %s, %" PRId64 "bytes", handle, pResBuf->path,
       pResBuf->totalBufSize);
   
-  *pResultBuf = pResBuf;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -210,7 +214,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
 }
 
 int32_t getLastPageId(SIDList *pList) {
-  if (pList == NULL && pList->size <= 0) {
+  if (pList == NULL || pList->size <= 0) {
     return -1;
   }