From 32577d41966059b7a66ba3ce5a00e5c2a270319c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 14 Apr 2022 18:24:21 +0800
Subject: [PATCH 01/22] refactor: add json tag function for sql parser

---
 include/common/tdataformat.h           |   3 +-
 include/common/ttypes.h                |   2 +-
 include/util/taoserror.h               |   1 +
 include/util/tdef.h                    |   8 ++
 include/util/tjson.h                   |   1 +
 include/util/tutil.h                   |   2 -
 source/common/src/ttypes.c             |   3 +-
 source/common/src/tvariant.c           |   2 +-
 source/libs/parser/inc/parUtil.h       |   1 +
 source/libs/parser/src/parAstCreater.c |   1 +
 source/libs/parser/src/parInsert.c     |  54 ++++----
 source/libs/parser/src/parTranslater.c |  24 +++-
 source/libs/parser/src/parUtil.c       | 169 ++++++++++++++++++++++++-
 source/libs/scalar/src/filter.c        |   2 +-
 source/util/src/tjson.c                |   1 +
 source/util/src/tutil.c                |  59 ---------
 tools/shell/src/shellEngine.c          |  71 +++--------
 17 files changed, 252 insertions(+), 152 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 8a87a1bd2e..2dd887ae74 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -608,7 +608,7 @@ void    tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void    tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow  tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) {
+static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
@@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
 
   pBuilder->nCols++;
 
-  int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
   if (tlen > pBuilder->alloc - pBuilder->size) {
     while (tlen > pBuilder->alloc - pBuilder->size) {
       pBuilder->alloc *= 2;
diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index 032cb44122..1112021124 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor {
                      int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 
-extern tDataTypeDescriptor tDataTypes[15];
+extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
 
 bool isValidDataType(int32_t type);
 
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 60535e2f49..a11bb5162d 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -599,6 +599,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST    TAOS_DEF_ERROR_CODE(0, 0x2624)
 #define TSDB_CODE_PAR_INVALID_OPTION_UNIT       TAOS_DEF_ERROR_CODE(0, 0x2625)
 #define TSDB_CODE_PAR_INVALID_KEEP_UNIT         TAOS_DEF_ERROR_CODE(0, 0x2626)
+#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG         TAOS_DEF_ERROR_CODE(0, 0x2627)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR              TAOS_DEF_ERROR_CODE(0, 0x2700)
diff --git a/include/util/tdef.h b/include/util/tdef.h
index f276a1a812..0289bb0df7 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -254,6 +254,14 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAGS           128
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
+#define TSDB_MAX_JSON_TAG_LEN   16384
+#define TSDB_JSON_PLACEHOLDER      0x7F
+#define TSDB_JSON_null             0x00
+#define TSDB_JSON_KEY_NULL         0x00
+#define TSDB_JSON_NOT_NULL         0x01
+#define TSDB_JSON_NULL             0x00
+
+
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32
 #define TSDB_USET_PASSWORD_LEN 129
diff --git a/include/util/tjson.h b/include/util/tjson.h
index 6b2221f704..9f347e5896 100644
--- a/include/util/tjson.h
+++ b/include/util/tjson.h
@@ -76,6 +76,7 @@ char* tjsonToString(const SJson* pJson);
 char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
+bool   tjsonIsObject(const SJson* pJson);
 
 #ifdef __cplusplus
 }
diff --git a/include/util/tutil.h b/include/util/tutil.h
index e0f92be76a..444a893a88 100644
--- a/include/util/tutil.h
+++ b/include/util/tutil.h
@@ -26,8 +26,6 @@ extern "C" {
 #endif
 
 int32_t strdequote(char *src);
-int32_t strndequote(char *dst, const char *z, int32_t len);
-int32_t strRmquote(char *z, int32_t len);
 size_t  strtrim(char *src);
 char   *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
 char  **strsplit(char *src, const char *delim, int32_t *num);
diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c
index ef75adeb5d..6af4777537 100644
--- a/source/common/src/ttypes.c
+++ b/source/common/src/ttypes.c
@@ -365,7 +365,7 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
   *maxIndex = 0;
 }
 
-tDataTypeDescriptor tDataTypes[15] = {
+tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
     {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
     {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
     {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint,
@@ -388,6 +388,7 @@ tDataTypeDescriptor tDataTypes[15] = {
     {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
     {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint,
      getStatics_u64},
+    {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {
diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c
index a0503e43a1..a546d9bb6b 100644
--- a/source/common/src/tvariant.c
+++ b/source/common/src/tvariant.c
@@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
     }
     case TSDB_DATA_TYPE_BINARY: {
       pVar->pz = strndup(z, n);
-      pVar->nLen = strRmquote(pVar->pz, n);
+      //pVar->nLen = strRmquote(pVar->pz, n);
       break;
     }
     case TSDB_DATA_TYPE_TIMESTAMP: {
diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h
index 742ab303d3..460a089802 100644
--- a/source/libs/parser/inc/parUtil.h
+++ b/source/libs/parser/inc/parUtil.h
@@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta);
 int32_t  getNumOfColumns(const STableMeta* pTableMeta);
 int32_t  getNumOfTags(const STableMeta* pTableMeta);
 STableComInfo getTableInfo(const STableMeta* pTableMeta);
+int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId);
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
 
diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index a14053ada7..0dcf04ecc0 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -47,6 +47,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
 }
 
 static void trimEscape(SToken* pName) {
+  // todo need to deal with `ioo``ii` -> ioo`ii
   if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) {
     pName->z += 1;
     pName->n -= 2;
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 435ab317e6..c9a0f4114e 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -377,35 +377,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
     return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
   }
 
-  if (IS_NUMERIC_TYPE(type) && pToken->n == 0) {
-    return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
-  }
-
   // Remove quotation marks
   if (TK_NK_STRING == pToken->type) {
     if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
       return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
     }
 
-    // delete escape character: \\, \', \"
-    char delim = pToken->z[0];
-    int32_t cnt = 0;
-    int32_t j = 0;
-    for (uint32_t k = 1; k < pToken->n - 1; ++k) {
-      if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
-        tmpTokenBuf[j] = pToken->z[k + 1];
-        cnt++;
-        j++;
-        k++;
-        continue;
-      }
-      tmpTokenBuf[j] = pToken->z[k];
-      j++;
-    }
-
-    tmpTokenBuf[j] = 0;
+    int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
     pToken->z = tmpTokenBuf;
-    pToken->n -= 2 + cnt;
+    pToken->n = len;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -582,6 +562,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
       return func(pMsgBuf, pToken->z, pToken->n, param);
     }
 
+    case TSDB_DATA_TYPE_JSON: {
+      if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+        return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z);
+      }
+      return func(pMsgBuf, pToken->z, pToken->n, param);
+    }
+
     case TSDB_DATA_TYPE_TIMESTAMP: {
       int64_t tmpVal;
       if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
@@ -702,8 +689,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
     qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
   }
 
-  memset(&pColList->boundColumns[pColList->numOfBound], 0,
-         sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
+  if(pColList->numOfCols > pColList->numOfBound){
+    memset(&pColList->boundColumns[pColList->numOfBound], 0,
+           sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -720,22 +709,31 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi
   int8_t  type = pa->schema->type;
   int16_t colId = pa->schema->colId;
 
+  if(TSDB_DATA_TYPE_JSON == type){
+    return parseJsontoTagData(value, pa->builder, pMsgBuf, colId);
+  }
+
+  if (value == NULL) {  // it is a null data
+    // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx);
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (TSDB_DATA_TYPE_BINARY == type) {
     STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
-    tdAddColToKVRow(pa->builder, colId, type, pa->buf);
+    tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
   } else if (TSDB_DATA_TYPE_NCHAR == type) {
     // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
     int32_t output = 0;
     if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
       char buf[512] = {0};
       snprintf(buf, tListLen(buf), "%s", strerror(errno));
-      return buildSyntaxErrMsg(pMsgBuf, buf, value);;
+      return buildSyntaxErrMsg(pMsgBuf, buf, value);
     }
 
     varDataSetLen(pa->buf, output);
-    tdAddColToKVRow(pa->builder, colId, type, pa->buf);
+    tdAddColToKVRow(pa->builder, colId, varDataTLen(pa->buf));
   } else {
-    tdAddColToKVRow(pa->builder, colId, type, value);
+    tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]);
   }
 
   return TSDB_CODE_SUCCESS;
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index b3947b872e..bd461c7622 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1542,6 +1542,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+  SNode* pNode;
+  FOREACH(pNode, pStmt->pTags) {
+    SColumnDefNode* pCol = (SColumnDefNode*)pNode;
+    if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){
+      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
   if (NULL == pFuncs) {
     return TSDB_CODE_SUCCESS;
@@ -1577,6 +1588,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
   if (TSDB_CODE_SUCCESS == code) {
     code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = checkTableTags(pCxt, pStmt);
+  }
   return code;
 }
 
@@ -2893,11 +2907,19 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS
   }
   SVariant var;
   valueNodeToVariant(pVal, &var);
+  if(pSchema->type == TSDB_DATA_TYPE_JSON){
+    if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz);
+    }
+    return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId);
+  }
+
   char    tagVal[TSDB_MAX_TAGS_LEN] = {0};
   int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
   if (TSDB_CODE_SUCCESS == code) {
-    tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal);
+    tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]);
   }
+
   return code;
 }
 
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 85d574d9e7..4ace62a2f1 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -14,6 +14,7 @@
  */
 
 #include "parUtil.h"
+#include "cJSON.h"
 
 static char* getSyntaxErrFormat(int32_t errCode) {
   switch (errCode) {
@@ -95,6 +96,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "Invalid option %s unit: %c, only m, h, d allowed";
     case TSDB_CODE_PAR_INVALID_KEEP_UNIT:
       return "Invalid option keep unit: %c, %c, %c, only m, h, d allowed";
+    case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG:
+      return "Only one tag if there is a json tag";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:
@@ -195,24 +198,180 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) {
 }
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
-  // delete escape character: \\, \', \"
+  if (len <=0 || dlen <= 0) return 0;
+
   char delim = src[0];
-  int32_t cnt = 0;
   int32_t j = 0;
   for (uint32_t k = 1; k < len - 1; ++k) {
     if (j >= dlen) {
-      break;
+      dst[j - 1] = '\0';
+      return j;
     }
-    if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) {
+    if (src[k] == delim && src[k + 1] == delim) {   // deal with "", ''
       dst[j] = src[k + 1];
-      cnt++;
       j++;
       k++;
       continue;
     }
+
+    if (src[k] == '\\') {   // deal with escape character
+      if(src[k+1] == 'n'){
+        dst[j] = '\n';
+      }else if(src[k+1] == 'r'){
+        dst[j] = '\r';
+      }else if(src[k+1] == 't'){
+        dst[j] = '\t';
+      }else if(src[k+1] == '\\'){
+        dst[j] = '\\';
+      }else if(src[k+1] == '\''){
+        dst[j] = '\'';
+      }else if(src[k+1] == '"'){
+        dst[j] = '"';
+      }else if(src[k+1] == '%' || src[k+1] == '_'){
+        dst[j++] = src[k];
+        dst[j] = src[k+1];
+      }else{
+        dst[j] = src[k+1];
+      }
+      j++;
+      k++;
+      continue;
+    }
+
     dst[j] = src[k];
     j++;
   }
   dst[j] = '\0';
   return j;
 }
+
+static bool isValidateTag(char *input) {
+  if (!input) return false;
+  for (size_t i = 0; i < strlen(input); ++i) {
+    if (isprint(input[i]) == 0) return false;
+  }
+  return true;
+}
+
+int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
+  // set json NULL data
+  uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL;
+  uint8_t jsonNULL = TSDB_JSON_NULL;
+  int jsonIndex = startColId + 1;
+  tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES);   // add json null type
+  if (!json || strtrim(json) == 0 || strncasecmp(json, "null", 4) == 0){
+    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);   // add json null value
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // set json real data
+  cJSON *root = cJSON_Parse(json);
+  if (root == NULL){
+    return buildSyntaxErrMsg(pMsgBuf, "json parse error", json);
+  }
+
+  int size = cJSON_GetArraySize(root);
+  if(!cJSON_IsObject(root)){
+    return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json);
+  }
+
+  int retCode = 0;
+  SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
+  for(int i = 0; i < size; i++) {
+    cJSON* item = cJSON_GetArrayItem(root, i);
+    if (!item) {
+      qError("json inner error:%d", i);
+      return buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
+      goto end;
+    }
+
+    char *jsonKey = item->string;
+    if(!isValidateTag(jsonKey)){
+      retCode =  buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
+      goto end;
+    }
+//    if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){
+//      tscError("json key too long error");
+//      retCode =  tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
+//      goto end;
+//    }
+    size_t keyLen = strlen(jsonKey);
+    if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
+      continue;
+    }
+
+    if(taosHashGetSize(keyHash) == 0){
+      uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);   // add json type
+    }
+    // json key encode by binary
+    void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1);
+    if(!tagKey) {
+      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto end;
+    }
+    strncpy(varDataVal(tagKey), jsonKey, keyLen);
+    taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
+
+    varDataSetLen(tagKey, keyLen);
+    tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey));   // add json key
+    taosMemoryFree(tagKey);
+
+    if(item->type == cJSON_String){     // add json value  format: type|data
+      char *jsonValue = item->valuestring;
+      int32_t valLen = (int32_t)strlen(jsonValue);
+      char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1);
+      if(!tagVal) {
+        retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+        goto end;
+      } ;
+      tagVal[0] = TSDB_DATA_TYPE_NCHAR;
+      char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES);
+      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData),
+                                                  (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
+        qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
+        retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
+        taosMemoryFree(tagVal);
+        goto end;
+      }
+
+      varDataSetLen(tagData, valLen);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData));
+      taosMemoryFree(tagVal);
+    }else if(item->type == cJSON_Number){
+      if(!isfinite(item->valuedouble)){
+        qError("json value is invalidate");
+        retCode =  buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
+        goto end;
+      }
+      char tagVal[LONG_BYTES + CHAR_BYTES] = {0};
+      tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT
+                                                                        : TSDB_DATA_TYPE_DOUBLE;
+      char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
+      if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble;
+      else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES);
+    }else if(item->type == cJSON_True || item->type == cJSON_False){
+      char tagVal[CHAR_BYTES + CHAR_BYTES] = {0};
+      tagVal[0] = TSDB_DATA_TYPE_BOOL;
+      tagVal[1] = (char)(item->valueint);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES);
+    }else if(item->type == cJSON_NULL){
+      char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL};
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES);
+    }
+    else{
+      retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
+      goto end;
+    }
+  }
+
+  if(taosHashGetSize(keyHash) == 0){  // set json NULL true
+    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);
+  }
+
+end:
+  taosHashCleanup(keyHash);
+  cJSON_Delete(root);
+  return retCode;
+}
\ No newline at end of file
diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c
index 191143de12..711ece3b01 100644
--- a/source/libs/scalar/src/filter.c
+++ b/source/libs/scalar/src/filter.c
@@ -57,7 +57,7 @@ OptrStr gOptrStr[] = {
   {OP_TYPE_IS_NOT_UNKNOWN,                 "not unknown"},
 
   // json operator
-  {OP_TYPE_JSON_GET_VALUE,                 "json get"},
+  {OP_TYPE_JSON_GET_VALUE,                 "->"},
   {OP_TYPE_JSON_CONTAINS,                  "json contains"}
 };
 
diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c
index 93a843fee8..8df404f85f 100644
--- a/source/util/src/tjson.c
+++ b/source/util/src/tjson.c
@@ -276,3 +276,4 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void
 }
 
 SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); }
+bool tjsonIsObject(const SJson* pJson) { return cJSON_IsObject(pJson); }
\ No newline at end of file
diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c
index 8133e4d237..adb6a37ba7 100644
--- a/source/util/src/tutil.c
+++ b/source/util/src/tutil.c
@@ -47,65 +47,6 @@ int32_t strdequote(char *z) {
   return j + 1;  // only one quote, do nothing
 }
 
-int32_t strRmquote(char *z, int32_t len) {
-  // delete escape character: \\, \', \"
-  char delim = z[0];
-  if (delim != '\'' && delim != '\"') {
-    return len;
-  }
-
-  int32_t cnt = 0;
-  int32_t j = 0;
-  for (uint32_t k = 1; k < len - 1; ++k) {
-    if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) {
-      if (z[k] == '\\' && z[k + 1] == '_') {
-        // match '_' self
-      } else {
-        z[j] = z[k + 1];
-        cnt++;
-        j++;
-        k++;
-        continue;
-      }
-    }
-
-    z[j] = z[k];
-    j++;
-  }
-
-  z[j] = 0;
-
-  return len - 2 - cnt;
-}
-
-int32_t strndequote(char *dst, const char *z, int32_t len) {
-  assert(dst != NULL);
-  if (z == NULL || len == 0) {
-    return 0;
-  }
-
-  int32_t quote = z[0];
-  int32_t i = 1, j = 0;
-
-  while (z[i] != 0) {
-    if (z[i] == quote) {
-      if (z[i + 1] == quote) {
-        dst[j++] = (char)quote;
-        i++;
-      } else {
-        dst[j++] = 0;
-        return (j - 1);
-      }
-    } else {
-      dst[j++] = z[i];
-    }
-
-    i++;
-  }
-
-  return j + 1;  // only one quote, do nothing
-}
-
 size_t strtrim(char *z) {
   int32_t i = 0;
   int32_t j = 0;
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index b89d517ad3..5a96a2a87d 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -226,63 +226,27 @@ int32_t shellRunCommand(TAOS *con, char *command) {
     }
   }
 
-  bool esc = false;
-  char quote = 0, *cmd = command, *p = command;
+  char quote = 0, *cmd = command;
   for (char c = *command++; c != 0; c = *command++) {
-    if (esc) {
-      switch (c) {
-        case 'n':
-          c = '\n';
-          break;
-        case 'r':
-          c = '\r';
-          break;
-        case 't':
-          c = '\t';
-          break;
-        case 'G':
-          *p++ = '\\';
-          break;
-        case '\'':
-        case '"':
-          if (quote) {
-            *p++ = '\\';
-          }
-          break;
-      }
-      *p++ = c;
-      esc = false;
+    if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
+      command ++;
       continue;
     }
 
-    if (c == '\\') {
-      if (quote != 0 && (*command == '_' || *command == '\\')) {
-        // DO nothing
-      } else {
-        esc = true;
-        continue;
-      }
-    }
-
     if (quote == c) {
       quote = 0;
-    } else if (quote == 0 && (c == '\'' || c == '"')) {
+    } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
       quote = c;
-    }
-
-    *p++ = c;
-    if (c == ';' && quote == 0) {
-      c = *p;
-      *p = 0;
+    } else if (c == ';' && quote == 0) {
+      c = *command;
+      *command = 0;
       if (shellRunSingleCommand(con, cmd) < 0) {
         return -1;
       }
-      *p = c;
-      p = cmd;
+      *command = c;
+      cmd = command;
     }
   }
-
-  *p = 0;
   return shellRunSingleCommand(con, cmd);
 }
 
@@ -573,19 +537,23 @@ static void shellPrintNChar(const char *str, int length, int width) {
   while (pos < length) {
     TdWchar wc;
     int     bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
-    if (bytes == 0) {
+    if (bytes <= 0) {
       break;
     }
-    pos += bytes;
-    if (pos > length) {
+    if (pos + bytes > length) {
       break;
     }
-
+    int w = 0;
 #ifdef WINDOWS
-    int w = bytes;
+    w = bytes;
 #else
-    int w = taosWcharWidth(wc);
+    if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
+      w = bytes;
+    }else{
+      w = taosWcharWidth(wc);
+    }
 #endif
+    pos += bytes;
     if (w <= 0) {
       continue;
     }
@@ -679,6 +647,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
       shellPrintNChar(val, length, width);
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:

From 807252356147847cb4b3018e2aee04a651e128ac Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 14 Apr 2022 19:01:11 +0800
Subject: [PATCH 02/22] refactor: add json tag function for sql parser

---
 source/libs/parser/src/parInsert.c | 2 +-
 source/libs/parser/src/parUtil.c   | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 9bc81d8fba..7d2d928aa9 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -738,7 +738,7 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi
     }
 
     varDataSetLen(pa->buf, output);
-    tdAddColToKVRow(pa->builder, colId, varDataTLen(pa->buf));
+    tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
   } else {
     tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]);
   }
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 4ace62a2f1..a423643e42 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -242,6 +242,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
     j++;
   }
   dst[j] = '\0';
+  strtrim(dst);
   return j;
 }
 
@@ -259,7 +260,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
   uint8_t jsonNULL = TSDB_JSON_NULL;
   int jsonIndex = startColId + 1;
   tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES);   // add json null type
-  if (!json || strtrim(json) == 0 || strncasecmp(json, "null", 4) == 0){
+  if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
     tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);   // add json null value
     return TSDB_CODE_SUCCESS;
   }
@@ -281,13 +282,13 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
     cJSON* item = cJSON_GetArrayItem(root, i);
     if (!item) {
       qError("json inner error:%d", i);
-      return buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
+      retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
       goto end;
     }
 
     char *jsonKey = item->string;
     if(!isValidateTag(jsonKey)){
-      retCode =  buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
+      retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
       goto end;
     }
 //    if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){

From b7bd654c773306a781045ed7f5b21705d71cdb97 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Sat, 16 Apr 2022 21:24:07 +0800
Subject: [PATCH 03/22] refactor: add compare logic for json value

---
 include/common/ttypes.h                |   2 +-
 include/util/tcompare.h                |   2 +
 include/util/tdef.h                    |   1 -
 source/libs/nodes/src/nodesUtilFuncs.c |   2 +-
 source/libs/parser/src/parTranslater.c |  38 ++--
 source/libs/parser/src/parUtil.c       |  76 ++++---
 source/libs/scalar/inc/sclvector.h     |   4 +
 source/libs/scalar/src/filter.c        |   9 +-
 source/libs/scalar/src/scalar.c        |  77 ++++---
 source/libs/scalar/src/sclvector.c     | 283 +++++++++++++++++++++----
 source/util/src/tcompare.c             |   5 +
 11 files changed, 358 insertions(+), 141 deletions(-)

diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index 19455db258..7c87235c39 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -49,7 +49,7 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
+#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
diff --git a/include/util/tcompare.h b/include/util/tcompare.h
index cc9e8ae464..ed07ae1c9c 100644
--- a/include/util/tcompare.h
+++ b/include/util/tcompare.h
@@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
 
+int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
+
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char *a, const char *b, int32_t type, size_t size);
diff --git a/include/util/tdef.h b/include/util/tdef.h
index caf01f5a22..eacc3aa08e 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -274,7 +274,6 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_JSON_TAG_LEN   16384
 #define TSDB_JSON_PLACEHOLDER      0x7F
 #define TSDB_JSON_null             0x00
-#define TSDB_JSON_KEY_NULL         0x00
 #define TSDB_JSON_NOT_NULL         0x01
 #define TSDB_JSON_NULL             0x00
 
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 758f9d5d6f..1154894050 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -1006,6 +1006,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
     case OP_TYPE_NOT_LIKE:
     case OP_TYPE_MATCH:
     case OP_TYPE_NMATCH:
+    case OP_TYPE_JSON_CONTAINS:
     case OP_TYPE_IS_NULL:
     case OP_TYPE_IS_NOT_NULL:
     case OP_TYPE_IS_TRUE:
@@ -1024,7 +1025,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
 bool nodesIsJsonOp(const SOperatorNode* pOp) {
   switch (pOp->opType) {
     case OP_TYPE_JSON_GET_VALUE:
-    case OP_TYPE_JSON_CONTAINS:
       return true;
     default:
       break;
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index e0708e8cea..4a20d929b5 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -469,14 +469,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
     pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE;
     pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
   } else if (nodesIsComparisonOp(pOp)) {
-    if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
+    if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
         TSDB_DATA_TYPE_BLOB == rdt.type) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
     }
     pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
     pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
-  } else {
-    // todo json operator
+  } else if (nodesIsJsonOp(pOp)){
+    if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
+      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
+    }
+    pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
+    pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
   }
   return DEAL_RES_CONTINUE;
 }
@@ -2814,25 +2818,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
 
 static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
                              SKVRowBuilder* pBuilder) {
+  if(pSchema->type == TSDB_DATA_TYPE_JSON){
+    if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
+    }
+
+    return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
+  }
+
   if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
     return pCxt->errCode;
   }
-  SVariant var;
-  valueNodeToVariant(pVal, &var);
-  if(pSchema->type == TSDB_DATA_TYPE_JSON){
-    if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
-      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz);
-    }
-    return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId);
+
+  if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
+    // todo
+  }else{
+    tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
   }
 
-  char    tagVal[TSDB_MAX_TAGS_LEN] = {0};
-  int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
-  if (TSDB_CODE_SUCCESS == code) {
-    tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]);
-  }
-
-  return code;
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 968b799816..2bb867e8d4 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -252,12 +252,10 @@ static bool isValidateTag(char *input) {
 
 int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
   // set json NULL data
-  uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL;
   uint8_t jsonNULL = TSDB_JSON_NULL;
   int jsonIndex = startColId + 1;
-  tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES);   // add json null type
   if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
-    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);   // add json null value
+    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -273,6 +271,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
   }
 
   int retCode = 0;
+  char *tagKV = NULL;
   SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
   for(int i = 0; i < size; i++) {
     cJSON* item = cJSON_GetArrayItem(root, i);
@@ -296,66 +295,64 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
     if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
       continue;
     }
-
+    // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES
+    tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1);
+    if(!tagKV) {
+      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto end;
+    }
+    strncpy(varDataVal(tagKV), jsonKey, keyLen);
+    varDataSetLen(tagKV, keyLen);
     if(taosHashGetSize(keyHash) == 0){
       uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);   // add json type
     }
-    // json key encode by binary
-    void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1);
-    if(!tagKey) {
-      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto end;
-    }
-    strncpy(varDataVal(tagKey), jsonKey, keyLen);
     taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
 
-    varDataSetLen(tagKey, keyLen);
-    tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey));   // add json key
-    taosMemoryFree(tagKey);
-
     if(item->type == cJSON_String){     // add json value  format: type|data
       char *jsonValue = item->valuestring;
       int32_t valLen = (int32_t)strlen(jsonValue);
-      char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1);
-      if(!tagVal) {
+      int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
+      char *tmp = taosMemoryRealloc(tagKV, totalLen);
+      if(!tmp) {
         retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
         goto end;
-      } ;
-      tagVal[0] = TSDB_DATA_TYPE_NCHAR;
-      char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES);
-      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData),
+      }
+      tagKV = tmp;
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = TSDB_DATA_TYPE_NCHAR;
+      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
                                                   (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
         qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
         retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
-        taosMemoryFree(tagVal);
         goto end;
       }
 
-      varDataSetLen(tagData, valLen);
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData));
-      taosMemoryFree(tagVal);
+      varDataSetLen(valueData, valLen);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
     }else if(item->type == cJSON_Number){
       if(!isfinite(item->valuedouble)){
         qError("json value is invalidate");
         retCode =  buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
         goto end;
       }
-      char tagVal[LONG_BYTES + CHAR_BYTES] = {0};
-      tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT
-                                                                        : TSDB_DATA_TYPE_DOUBLE;
-      char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
-      if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble;
-      else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint;
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES);
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE;
+      if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble;
+      else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES);
     }else if(item->type == cJSON_True || item->type == cJSON_False){
-      char tagVal[CHAR_BYTES + CHAR_BYTES] = {0};
-      tagVal[0] = TSDB_DATA_TYPE_BOOL;
-      tagVal[1] = (char)(item->valueint);
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES);
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = TSDB_DATA_TYPE_BOOL;
+      *valueData = (char)(item->valueint);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
     }else if(item->type == cJSON_NULL){
-      char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL};
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES);
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      *valueType = TSDB_DATA_TYPE_NULL;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
     }
     else{
       retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
@@ -364,10 +361,11 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
   }
 
   if(taosHashGetSize(keyHash) == 0){  // set json NULL true
-    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);
+    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
   }
 
 end:
+  taosMemoryFree(tagKV);
   taosHashCleanup(keyHash);
   cJSON_Delete(root);
   return retCode;
diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h
index e51115e9c0..47205b7fba 100644
--- a/source/libs/scalar/inc/sclvector.h
+++ b/source/libs/scalar/inc/sclvector.h
@@ -53,6 +53,8 @@ static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index)
   return (double)*((double *)src + index);
 }
 
+double getVectorDoubleValue_JSON(void *src, int32_t index);
+
 static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
   _getDoubleValue_fn_t p = NULL;
   if (srcType == TSDB_DATA_TYPE_TINYINT) {
@@ -77,6 +79,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
     p = getVectorDoubleValue_DOUBLE;
   } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) {
     p = getVectorDoubleValue_BIGINT;
+  } else if (srcType == TSDB_DATA_TYPE_JSON) {
+    p = getVectorDoubleValue_JSON;
   } else {
     assert(0);
   }
diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c
index 711ece3b01..ea249005b4 100644
--- a/source/libs/scalar/src/filter.c
+++ b/source/libs/scalar/src/filter.c
@@ -170,7 +170,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
   compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
   setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, 
   compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
-  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch
+  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
 };
 
 int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@@ -221,7 +221,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         assert(0);
     }
   }
-  
+
+  if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
+    return 28;
+  }
+
+
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:   comparFn = 1;   break;
diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index 116dfdd5d5..371b011c40 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -600,44 +600,39 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
   return DEAL_RES_CONTINUE;
 }
 
-EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
-  STargetNode *target = (STargetNode *)pNode;
-  
-  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
-    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
-    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-    return DEAL_RES_ERROR;
-  }
-
-  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
-  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
-    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
-    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-    return DEAL_RES_ERROR;
-  }
-
-  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
-  
-  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-  if (NULL == res) {
-    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
-    ctx->code = TSDB_CODE_QRY_APP_ERROR;
-    return DEAL_RES_ERROR;
-  }
-  
-  for (int32_t i = 0; i < res->numOfRows; ++i) {
-    if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) {
-      colDataAppend(col, i, NULL, true);
-    } else {
-      char *p = colDataGetData(res->columnData, i);
-      colDataAppend(col, i, p, false);
-    }
-  }
-
-  sclFreeParam(res);
-  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-  return DEAL_RES_CONTINUE;
-}
+//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
+//  STargetNode *target = (STargetNode *)pNode;
+//
+//  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
+//    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
+//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+//    return DEAL_RES_ERROR;
+//  }
+//
+//  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
+//  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
+//    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
+//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+//    return DEAL_RES_ERROR;
+//  }
+//
+//  // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
+//  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
+//
+//  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+//  if (NULL == res) {
+//    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
+//    ctx->code = TSDB_CODE_QRY_APP_ERROR;
+//    return DEAL_RES_ERROR;
+//  }
+//
+//  colDataAssign(col, res->columnData, res->numOfRows);
+//  block->info.rows = res->numOfRows;
+//
+//  sclFreeParam(res);
+//  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+//  return DEAL_RES_CONTINUE;
+//}
 
 EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
   if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
@@ -657,9 +652,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
     return sclWalkOperator(pNode, ctx);
   }
 
-  if (QUERY_NODE_TARGET == nodeType(pNode)) {
-    return sclWalkTarget(pNode, ctx);
-  }
+//  if (QUERY_NODE_TARGET == nodeType(pNode)) {
+//    return sclWalkTarget(pNode, ctx);
+//  }
 
   sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
   ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 05456790a5..11dc60dab3 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -177,24 +177,24 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd
   colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v);
 }
 
-static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
-  int32_t len = 0;
-  int32_t inputLen = varDataLen(buf);
-
-  char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
-  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
-  varDataSetLen(t, len);
-
-  colDataAppend(pOut->columnData, rowIndex, t, false);
-  taosMemoryFree(t);
-}
+//static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
+//  int32_t len = 0;
+//  int32_t inputLen = varDataLen(buf);
+//
+//  char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
+//  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
+//  varDataSetLen(t, len);
+//
+//  colDataAppend(pOut->columnData, rowIndex, t, false);
+//  taosMemoryFree(t);
+//}
 
 //TODO opt performance, tmp is not needed.
 int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
   int32_t bufSize = pIn->columnData->info.bytes;
   char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE);
 
-  bool vton = false;
+//  bool vton = false;
 
   _bufConverteFunc func = NULL;
   if (TSDB_DATA_TYPE_BOOL == outType) {
@@ -205,9 +205,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     func = varToUnsigned;
   } else if (IS_FLOAT_TYPE(outType)) {
     func = varToFloat;
-  } else if (outType == TSDB_DATA_TYPE_NCHAR) {
-    func = varToNchar;
-    vton = true;
+//  } else if (outType == TSDB_DATA_TYPE_NCHAR) {   // can not be nchar or binary
+//    func = varToNchar;
+//    vton = true;
   } else {
     sclError("invalid convert outType:%d", outType);
     return TSDB_CODE_QRY_APP_ERROR;
@@ -221,9 +221,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     }
 
     char* data = colDataGetData(pIn->columnData, i);
-    if (vton) {
-      memcpy(tmp, data, varDataTLen(data));
-    } else {
+//    if (vton) {
+//      memcpy(tmp, data, varDataTLen(data));
+//    } else {
       if (TSDB_DATA_TYPE_VARCHAR == inType) {
         memcpy(tmp, varDataVal(data), varDataLen(data));
         tmp[varDataLen(data)] = 0;
@@ -239,7 +239,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
 
         tmp[len] = 0;
       }
-    }
+//    }
     
     (*func)(tmp, pOut, i);
   }
@@ -248,6 +248,135 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
   return TSDB_CODE_SUCCESS;
 }
 
+void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
+  switch (outType) {
+    case TSDB_DATA_TYPE_BOOL: {
+      GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_TINYINT: {
+      GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UTINYINT: {
+      GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT: {
+      GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT: {
+      GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UBIGINT: {
+      GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      GET_TYPED_DATA(*((float *)outData), float, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      GET_TYPED_DATA(*((double *)outData), double, inType, inData);
+      break;
+    }
+    default:{
+      ASSERT(0);
+    }
+  }
+}
+
+void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
+  char *tmp = taosMemoryMalloc(varDataTLen(inData));
+  if (inType == TSDB_DATA_TYPE_NCHAR) {
+    int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
+    if (len < 0) {
+      sclError("castConvert taosUcs4ToMbs error 1");
+    }
+
+    tmp[len] = 0;
+  } else {
+    memcpy(tmp, varDataVal(inData), varDataLen(inData));
+    tmp[varDataLen(inData)] = 0;
+  }
+
+  ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
+  double value = strtod(tmp, NULL);
+
+  *((double *)outData) = value;
+  taosMemoryFreeClear(tmp);
+}
+
+double getVectorDoubleValue_JSON(void *src, int32_t index){
+  ASSERT(!colDataIsNull_s(((SColumnInfoData*)src), index));
+  char *data = colDataGetData((SColumnInfoData*)src, index);
+  double out = 0;
+  if(*data == TSDB_DATA_TYPE_NCHAR) {
+    convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+  } {
+    convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+  }
+  return out;
+}
+
+void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut){
+  if(optr == OP_TYPE_JSON_CONTAINS) {
+    return;
+  }
+
+  if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
+    return;
+  }
+
+  if(typeLeft == TSDB_DATA_TYPE_JSON){
+    typeLeft = **pLeftData;
+    (*pLeftData) ++;
+  }
+  if(typeRight == TSDB_DATA_TYPE_JSON){
+    typeRight = **pRightData;
+    (*pRightData) ++;
+  }
+  int8_t type = vectorGetConvertType(typeLeft, typeRight);
+
+  if(type == 0) {
+    *fp = filterGetCompFunc(typeLeft, optr);
+    return;
+  }
+
+  *fp = filterGetCompFunc(type, optr);
+
+  if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
+    convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type);
+    *pLeftData = pLeftOut;
+  } else if(typeLeft != type) {
+    convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
+    *pLeftData = pLeftOut;
+  }
+
+  if(IS_VAR_DATA_TYPE(typeRight)) {
+    convertStringToDouble(*pRightData, pRightOut, typeRight, type);
+    *pRightData = pRightOut;
+  } else if(typeRight != type) {
+    convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
+    *pRightData = pRightOut;
+  }
+}
+
 // TODO opt performance
 int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
   SColumnInfoData* pInputCol  = pIn->columnData;
@@ -522,7 +651,7 @@ enum {
 static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) {
   SColumnInfoData* pCol = pParam->columnData;
 
-  if (IS_VAR_DATA_TYPE(pCol->info.type)) {
+  if (IS_VAR_DATA_TYPE(pCol->info.type) || pCol->info.type != TSDB_DATA_TYPE_JSON) {
     pDest->numOfRows = pParam->numOfRows;
 
     SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
@@ -556,7 +685,8 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0);
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -587,6 +717,36 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
   }
 }
 
+char *getJsonValue(char *json, char *key){    //todo
+  return NULL;
+}
+
+void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  SColumnInfoData *pOutputCol = pOut->columnData;
+
+  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
+
+  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
+
+  char *pRightData = colDataGetData(pRight->columnData, 0);
+  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
+    if (pLeft->columnData->varmeta.offset[i] == -1) {
+      pOutputCol->varmeta.offset[i] = -1;
+      pOutputCol->hasNull = true;
+      continue;
+    }
+    char *pLeftData = colDataGetData(pLeft->columnData, i);
+    char *value = getJsonValue(pLeftData, pRightData);
+    if (!value || *value == TSDB_DATA_TYPE_NULL) {
+      pOutputCol->varmeta.offset[i] = -1;
+      pOutputCol->hasNull = true;
+      continue;
+    }
+    colDataAppend(pOutputCol, i, pLeftData, false);
+  }
+}
+
 void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   SColumnInfoData *pOutputCol = pOut->columnData;
 
@@ -605,7 +765,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -637,7 +802,8 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor;
+      *output = (getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                 - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0)) * factor;
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -664,7 +830,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -696,7 +867,8 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0);
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -722,7 +894,12 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -760,7 +937,12 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {  // check for the 0 value
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                 /getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -776,7 +958,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
       for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i);
+        *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0)
+                  / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
       }
       pOutputCol->hasNull = pRightCol->hasNull;
       if (pOutputCol->hasNull) {
@@ -788,7 +971,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
       for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0);
+        *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                  / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
       }
       pOutputCol->hasNull = pLeftCol->hasNull;
       if (pOutputCol->hasNull) {
@@ -825,8 +1009,8 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
         continue;
       }
 
-      double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
-      double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
+      double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
+      double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
       if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
         colDataAppendNULL(pOutputCol, i);
         continue;
@@ -835,7 +1019,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
       *output = lx - ((int64_t)(lx / rx)) * rx;
     }
   } else if (pLeft->numOfRows == 1) {
-    double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0);
+    double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0);
     if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
@@ -845,7 +1029,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
           continue;
         }
 
-        double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
+        double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
         if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
@@ -855,7 +1039,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
       }
     }
   } else if (pRight->numOfRows == 1) {
-    double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0);
+    double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
     if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
@@ -865,7 +1049,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
           continue;
         }
 
-        double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
+        double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
         if (isnan(lx) || isinf(lx)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
@@ -895,7 +1079,11 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
 
   double *output = (double *)pOutputCol->pData;
   for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
-    *output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i);
+    if (colDataIsNull_s(pLeft->columnData, i)) {
+      colDataAppendNULL(pOutputCol, i);
+      continue;  // TODO set null or ignore
+    }
+    *output = - getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
   }
 
   pOutputCol->hasNull = pLeftCol->hasNull;
@@ -1099,30 +1287,39 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
 
       char *pLeftData = colDataGetData(pLeft->columnData, i);
       char *pRightData = colDataGetData(pRight->columnData, i);
+
+      int64_t leftOut = 0;
+      int64_t rightOut = 0;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
   } else if (pRight->numOfRows == 1) {
-    char *pRightData = colDataGetData(pRight->columnData, 0);
     ASSERT(pLeft->pHashFilter == NULL);
-
     for (; i >= 0 && i < pLeft->numOfRows; i += step) {
       if (colDataIsNull_s(pLeft->columnData, i)) {
         continue;
       }
 
       char *pLeftData = colDataGetData(pLeft->columnData, i);
+      char *pRightData = colDataGetData(pRight->columnData, 0);
+      int64_t leftOut = 0;
+      int64_t rightOut = 0;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
   } else if (pLeft->numOfRows == 1) {
-    char *pLeftData = colDataGetData(pLeft->columnData, 0);
     for (; i >= 0 && i < pRight->numOfRows; i += step) {
       if (colDataIsNull_s(pRight->columnData, i)) {
         continue;
       }
 
+      char *pLeftData = colDataGetData(pLeft->columnData, 0);
       char *pRightData = colDataGetData(pLeft->columnData, i);
+      int64_t leftOut = 0;
+      int64_t rightOut = 0;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
@@ -1203,6 +1400,10 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu
   vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH);
 }
 
+void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS);
+}
+
 void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
@@ -1271,6 +1472,10 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
       return vectorBitOr;
     case OP_TYPE_IS_TRUE:
       return vectorIsTrue;
+    case OP_TYPE_JSON_GET_VALUE:
+      return vectorJsonArrow;
+    case OP_TYPE_JSON_CONTAINS:
+      return vectorJsonContains;
     default:
       assert(0);
       return NULL;
diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c
index 93022de021..0ddc76e415 100644
--- a/source/util/src/tcompare.c
+++ b/source/util/src/tcompare.c
@@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
   return compareLenPrefixedWStr(pRight, pLeft);
 }
 
+int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
+  if(pLeft) return 0;
+  return 1;
+}
+
 /*
  * Compare two strings
  *    TSDB_MATCH:            Match

From e2c6bf41df1d33ba3698fe0aef0f32b3e52bec0a Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 18 Apr 2022 13:36:01 +0800
Subject: [PATCH 04/22] refactor: add compare logic for json value

---
 include/common/tdatablock.h        |  20 +-
 source/libs/scalar/src/sclvector.c | 289 +++++++++++++++++------------
 2 files changed, 186 insertions(+), 123 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 15f3246013..8b9cf74d7a 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -54,12 +54,15 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
+#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
+#define colDataSetNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] = -1)
+
 static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
   if (!pColumnInfoData->hasNull) {
     return false;
   }
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    return pColumnInfoData->varmeta.offset[row] == -1;
+    return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
       return false;
@@ -86,7 +89,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    return pColumnInfoData->varmeta.offset[row] == -1;
+    return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
       return false;
@@ -98,15 +101,18 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
 
 #define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
 
+#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
+
+#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
 // SColumnInfoData, rowNumber
-#define colDataGetData(p1_, r_)                                                        \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
-                                        : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
+#define colDataGetData(p1_, r_)                                        \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
+                                        :  colDataGetNumData(p1_, r_)
 
 static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
   // There is a placehold for each NULL value of binary or nchar type.
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    pColumnInfoData->varmeta.offset[currentRow] = -1;  // it is a null value of VAR type.
+    colDataSetNull_var(pColumnInfoData, currentRow);  // it is a null value of VAR type.
   } else {
     colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
   }
@@ -117,7 +123,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
 static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     for (int32_t i = start; i < start + nRows; ++i) {
-      pColumnInfoData->varmeta.offset[i] = -1;  // it is a null value of VAR type.
+      colDataSetNull_var(pColumnInfoData,i);  // it is a null value of VAR type.
     }
   } else {
     for (int32_t i = start; i < start + nRows; ++i) {
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 11dc60dab3..aa51c9b631 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -25,6 +25,9 @@
 #include "tdatablock.h"
 #include "ttypes.h"
 
+#define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
+#define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
+
 typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
 
 int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
@@ -189,65 +192,6 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd
 //  taosMemoryFree(t);
 //}
 
-//TODO opt performance, tmp is not needed.
-int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
-  int32_t bufSize = pIn->columnData->info.bytes;
-  char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE);
-
-//  bool vton = false;
-
-  _bufConverteFunc func = NULL;
-  if (TSDB_DATA_TYPE_BOOL == outType) {
-    func = varToBool;
-  } else if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) {
-    func = varToSigned;
-  } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) {
-    func = varToUnsigned;
-  } else if (IS_FLOAT_TYPE(outType)) {
-    func = varToFloat;
-//  } else if (outType == TSDB_DATA_TYPE_NCHAR) {   // can not be nchar or binary
-//    func = varToNchar;
-//    vton = true;
-  } else {
-    sclError("invalid convert outType:%d", outType);
-    return TSDB_CODE_QRY_APP_ERROR;
-  }
-
-  pOut->numOfRows = pIn->numOfRows;
-  for (int32_t i = 0; i < pIn->numOfRows; ++i) {
-    if (colDataIsNull_s(pIn->columnData, i)) {
-      colDataAppendNULL(pOut->columnData, i);
-      continue;
-    }
-
-    char* data = colDataGetData(pIn->columnData, i);
-//    if (vton) {
-//      memcpy(tmp, data, varDataTLen(data));
-//    } else {
-      if (TSDB_DATA_TYPE_VARCHAR == inType) {
-        memcpy(tmp, varDataVal(data), varDataLen(data));
-        tmp[varDataLen(data)] = 0;
-      } else {
-        ASSERT(varDataLen(data) <= bufSize);
-
-        int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
-        if (len < 0) {
-          sclError("castConvert taosUcs4ToMbs error 1");
-          taosMemoryFreeClear(tmp);
-          return TSDB_CODE_QRY_APP_ERROR;
-        }
-
-        tmp[len] = 0;
-      }
-//    }
-    
-    (*func)(tmp, pOut, i);
-  }
-  
-  taosMemoryFreeClear(tmp);
-  return TSDB_CODE_SUCCESS;
-}
-
 void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
   switch (outType) {
     case TSDB_DATA_TYPE_BOOL: {
@@ -303,18 +247,13 @@ void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int
 
 void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
   char *tmp = taosMemoryMalloc(varDataTLen(inData));
-  if (inType == TSDB_DATA_TYPE_NCHAR) {
-    int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
-    if (len < 0) {
-      sclError("castConvert taosUcs4ToMbs error 1");
-    }
-
-    tmp[len] = 0;
-  } else {
-    memcpy(tmp, varDataVal(inData), varDataLen(inData));
-    tmp[varDataLen(inData)] = 0;
+  int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
+  if (len < 0) {
+    sclError("castConvert taosUcs4ToMbs error 1");
   }
 
+  tmp[len] = 0;
+
   ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
   double value = strtod(tmp, NULL);
 
@@ -322,11 +261,85 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int
   taosMemoryFreeClear(tmp);
 }
 
+//TODO opt performance, tmp is not needed.
+int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
+  ASSERT(!IS_VAR_DATA_TYPE(outType));
+  int32_t bufSize = pIn->columnData->info.bytes;
+  char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE);
+
+//  bool vton = false;
+
+  _bufConverteFunc func = NULL;
+  if (TSDB_DATA_TYPE_BOOL == outType) {
+    func = varToBool;
+  } else if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) {
+    func = varToSigned;
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) {
+    func = varToUnsigned;
+  } else if (IS_FLOAT_TYPE(outType)) {
+    func = varToFloat;
+//  } else if (outType == TSDB_DATA_TYPE_NCHAR) {   // can not be nchar or binary
+//    func = varToNchar;
+//    vton = true;
+  } else {
+    sclError("invalid convert outType:%d", outType);
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  pOut->numOfRows = pIn->numOfRows;
+  for (int32_t i = 0; i < pIn->numOfRows; ++i) {
+    if (colDataIsNull_s(pIn->columnData, i)) {
+      colDataAppendNULL(pOut->columnData, i);
+      continue;
+    }
+
+    char* data = colDataGetVarData(pIn->columnData, i);
+    int32_t convertType = inType;
+    if(inType == TSDB_DATA_TYPE_JSON){
+      if(*data == TSDB_DATA_TYPE_NULL) {
+        colDataAppendNULL(pOut->columnData, i);
+        continue;
+      }
+      else if(*data == TSDB_DATA_TYPE_NCHAR) {
+        data += CHAR_BYTES;
+        convertType = TSDB_DATA_TYPE_NCHAR;
+      } else {
+        convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType);
+        continue;
+      }
+    }
+//    if (vton) {
+//      memcpy(tmp, data, varDataTLen(data));
+//    } else {
+      if (TSDB_DATA_TYPE_VARCHAR == convertType) {
+        memcpy(tmp, varDataVal(data), varDataLen(data));
+        tmp[varDataLen(data)] = 0;
+      } else if (TSDB_DATA_TYPE_NCHAR == convertType){
+        ASSERT(varDataLen(data) <= bufSize);
+
+        int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
+        if (len < 0) {
+          sclError("castConvert taosUcs4ToMbs error 1");
+          taosMemoryFreeClear(tmp);
+          return TSDB_CODE_QRY_APP_ERROR;
+        }
+
+        tmp[len] = 0;
+      }
+//    }
+    
+    (*func)(tmp, pOut, i);
+  }
+  
+  taosMemoryFreeClear(tmp);
+  return TSDB_CODE_SUCCESS;
+}
+
 double getVectorDoubleValue_JSON(void *src, int32_t index){
-  ASSERT(!colDataIsNull_s(((SColumnInfoData*)src), index));
-  char *data = colDataGetData((SColumnInfoData*)src, index);
+  ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index));
+  char *data = colDataGetVarData((SColumnInfoData*)src, index);
   double out = 0;
-  if(*data == TSDB_DATA_TYPE_NCHAR) {
+  if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
     convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   } {
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
@@ -685,8 +698,12 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
+      if (colDataIsNull_s(pLeftCol, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+                + getVectorDoubleValueFnRight(RIGHT_COL, 0);
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -729,17 +746,17 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
 
   pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
 
-  char *pRightData = colDataGetData(pRight->columnData, 0);
+  char *pRightData = colDataGetVarData(pRight->columnData, 0);
   for (; i >= 0 && i < pLeft->numOfRows; i += step) {
-    if (pLeft->columnData->varmeta.offset[i] == -1) {
-      pOutputCol->varmeta.offset[i] = -1;
+    if (colDataIsNull_var(pLeft->columnData, i)) {
+      colDataSetNull_var(pOutputCol, i);
       pOutputCol->hasNull = true;
       continue;
     }
-    char *pLeftData = colDataGetData(pLeft->columnData, i);
+    char *pLeftData = colDataGetVarData(pLeft->columnData, i);
     char *value = getJsonValue(pLeftData, pRightData);
     if (!value || *value == TSDB_DATA_TYPE_NULL) {
-      pOutputCol->varmeta.offset[i] = -1;
+      colDataSetNull_var(pOutputCol, i);
       pOutputCol->hasNull = true;
       continue;
     }
@@ -769,8 +786,8 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
-      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+                + getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -802,8 +819,12 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = (getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                 - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0)) * factor;
+      if (colDataIsNull_s(pLeftCol, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = (getVectorDoubleValueFnLeft(LEFT_COL, i)
+                 - getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor;
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -834,8 +855,8 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
-      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+                - getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -867,8 +888,12 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
+      if (colDataIsNull_s(pLeftCol, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+                * getVectorDoubleValueFnRight(RIGHT_COL, 0);
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -898,8 +923,8 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
-      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+                * getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -941,8 +966,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
-      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                 /getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+                 /getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -954,12 +979,16 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
     }
 
   } else if (pLeft->numOfRows == 1) {
-    if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+    if (colDataIsNull_s(pLeftCol, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
       for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0)
-                  / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
+        if (colDataIsNull_s(pRightCol, i)) {
+          colDataAppendNULL(pOutputCol, i);
+          continue;  // TODO set null or ignore
+        }
+        *output = getVectorDoubleValueFnLeft(LEFT_COL, 0)
+                  / getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
       pOutputCol->hasNull = pRightCol->hasNull;
       if (pOutputCol->hasNull) {
@@ -967,12 +996,16 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
       }
     }
   } else if (pRight->numOfRows == 1) {
-    if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+    if (colDataIsNull_s(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
       for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
-                  / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
+        if (colDataIsNull_s(pLeftCol, i)) {
+          colDataAppendNULL(pOutputCol, i);
+          continue;  // TODO set null or ignore
+        }
+        *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+                  / getVectorDoubleValueFnRight(RIGHT_COL, 0);
       }
       pOutputCol->hasNull = pLeftCol->hasNull;
       if (pOutputCol->hasNull) {
@@ -1004,13 +1037,13 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
 
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      if (colDataIsNull_f(pLeftCol->nullbitmap, i) || colDataIsNull_f(pRightCol->nullbitmap, i)) {
+      if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;
       }
 
-      double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
-      double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
+      double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
+      double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
       if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
         colDataAppendNULL(pOutputCol, i);
         continue;
@@ -1019,17 +1052,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
       *output = lx - ((int64_t)(lx / rx)) * rx;
     }
   } else if (pLeft->numOfRows == 1) {
-    double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0);
-    if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) {  // Set pLeft->numOfRows NULL value
+    double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0);
+    if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
       for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
-        if (colDataIsNull_f(pRightCol->nullbitmap, i)) {
+        if (colDataIsNull_s(pRightCol, i)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
 
-        double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
+        double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
         if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
@@ -1039,17 +1072,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
       }
     }
   } else if (pRight->numOfRows == 1) {
-    double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
-    if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) {  // Set pLeft->numOfRows NULL value
+    double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0);
+    if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
       for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
-        if (colDataIsNull_f(pLeftCol->nullbitmap, i)) {
+        if (colDataIsNull_s(pLeftCol, i)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
 
-        double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
+        double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
         if (isnan(lx) || isinf(lx)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
@@ -1083,7 +1116,7 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
       colDataAppendNULL(pOutputCol, i);
       continue;  // TODO set null or ignore
     }
-    *output = - getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
+    *output = - getVectorDoubleValueFnLeft(LEFT_COL, i);
   }
 
   pOutputCol->hasNull = pLeftCol->hasNull;
@@ -1151,11 +1184,15 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh
 
   double *output = (double *)pOutputCol->pData;
 
-  if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+  if (colDataIsNull_s(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, 0);
+      if (colDataIsNull_s(pLeftCol, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0);
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -1181,7 +1218,11 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   int64_t *output = (int64_t *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -1208,12 +1249,16 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight
 
   int64_t *output = (int64_t *)pOutputCol->pData;
 
-  if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+  if (colDataIsNull_s(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
-    int64_t rx = getVectorBigintValueFnRight(pRightCol->pData, 0);
+    int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0);
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | rx;
+      if (colDataIsNull_s(pLeftCol, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx;
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -1239,7 +1284,11 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   int64_t *output = (int64_t *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | getVectorBigintValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -1407,6 +1456,10 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
 void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
+    if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){
+      char *data = colDataGetVarData(pLeft->columnData, i);
+      v = (*data == TSDB_DATA_TYPE_NULL)? 1:0;
+    }
     colDataAppendInt8(pOut->columnData, i, &v);
   }
   pOut->numOfRows = pLeft->numOfRows;
@@ -1415,6 +1468,10 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
 void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1;
+    if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){
+      char *data = colDataGetVarData(pLeft->columnData, i);
+      v = (*data == TSDB_DATA_TYPE_NULL)? 0:1;
+    }
     colDataAppendInt8(pOut->columnData, i, &v);
   }
   pOut->numOfRows = pLeft->numOfRows;

From d2d29d992ff8c155e09e389191c51ba2713a9209 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 18 Apr 2022 14:25:19 +0800
Subject: [PATCH 05/22] refactor: add compare logic for json value

---
 include/common/tdatablock.h        |  2 +-
 source/libs/scalar/src/sclvector.c | 26 +++++++++++++++++++++-----
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 8b9cf74d7a..e40efd815d 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -107,7 +107,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
 // SColumnInfoData, rowNumber
 #define colDataGetData(p1_, r_)                                        \
   ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
-                                        :  colDataGetNumData(p1_, r_)
+                                        :  colDataGetNumData(p1_, r_))
 
 static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
   // There is a placehold for each NULL value of binary or nchar type.
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index aa51c9b631..56787b229b 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -347,7 +347,7 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
   return out;
 }
 
-void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut){
+void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
   if(optr == OP_TYPE_JSON_CONTAINS) {
     return;
   }
@@ -364,6 +364,10 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
     typeRight = **pRightData;
     (*pRightData) ++;
   }
+  if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
+    *isNull = true;
+    return;
+  }
   int8_t type = vectorGetConvertType(typeLeft, typeRight);
 
   if(type == 0) {
@@ -381,7 +385,7 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
     *pLeftData = pLeftOut;
   }
 
-  if(IS_VAR_DATA_TYPE(typeRight)) {
+  if(typeRight == TSDB_DATA_TYPE_NCHAR) {
     convertStringToDouble(*pRightData, pRightOut, typeRight, type);
     *pRightData = pRightOut;
   } else if(typeRight != type) {
@@ -1339,7 +1343,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
 
       int64_t leftOut = 0;
       int64_t rightOut = 0;
-      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
+      bool isJsonnull = false;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
+      if(isJsonnull){
+        continue;  // TODO set null or ignore
+      }
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
@@ -1354,7 +1362,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       char *pRightData = colDataGetData(pRight->columnData, 0);
       int64_t leftOut = 0;
       int64_t rightOut = 0;
-      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
+      bool isJsonnull = false;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
+      if(isJsonnull){
+        continue;  // TODO set null or ignore
+      }
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
@@ -1368,7 +1380,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       char *pRightData = colDataGetData(pLeft->columnData, i);
       int64_t leftOut = 0;
       int64_t rightOut = 0;
-      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
+      bool isJsonnull = false;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
+      if(isJsonnull){
+        continue;  // TODO set null or ignore
+      }
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }

From 24f805c5c2ebfbb1ed8fe5dc8c967b210eca84d2 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 18 Apr 2022 15:54:40 +0800
Subject: [PATCH 06/22] refactor: add to_json function

---
 include/libs/scalar/scalar.h        |  1 +
 include/util/tjson.h                |  2 +-
 source/libs/function/src/builtins.c |  2 +-
 source/libs/scalar/src/sclfunc.c    | 54 +++++++++++++++++++++++++++++
 source/util/src/tjson.c             | 35 ++++++++++++++++++-
 5 files changed, 91 insertions(+), 3 deletions(-)

diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h
index 49af10cd2c..6f4a87b02b 100644
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 /* Time related functions */
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
diff --git a/include/util/tjson.h b/include/util/tjson.h
index 9f347e5896..28bcb3cfc6 100644
--- a/include/util/tjson.h
+++ b/include/util/tjson.h
@@ -76,7 +76,7 @@ char* tjsonToString(const SJson* pJson);
 char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
-bool   tjsonIsObject(const SJson* pJson);
+bool   tjsonValidateJson(const char* pJson);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index bba5145812..342eb5990b 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -893,7 +893,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .translateFunc = translateToJson,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,
-    .sprocessFunc = NULL,
+    .sprocessFunc = toJsonFunction,
     .finalizeFunc = NULL
   }
 };
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 97cc80b946..05269989cf 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -4,6 +4,7 @@
 #include "ttime.h"
 #include "sclInt.h"
 #include "sclvector.h"
+#include "tjson.h"
 
 typedef float (*_float_fn)(float);
 typedef double (*_double_fn)(double);
@@ -920,6 +921,59 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  int32_t type = GET_PARAM_TYPE(pInput);
+  if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (inputNum != 1) {
+    return TSDB_CODE_FAILED;
+  }
+
+  char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0];
+  char *tmp = taosMemoryCalloc(pInput[0].columnData->info.bytes + 1, 1);
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    if (colDataIsNull_s(pInput[0].columnData, i)) {
+      colDataAppendNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    if(type == TSDB_DATA_TYPE_NCHAR){
+      if (varDataTLen(input) > TSDB_MAX_JSON_TAG_LEN){
+        colDataAppendNULL(pOutput->columnData, i);
+        continue;
+      }
+      int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), tmp);
+      if (len < 0) {
+        colDataAppendNULL(pOutput->columnData, i);
+        continue;
+      }
+      tmp[len] = 0;
+    }else{
+      if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+        colDataAppendNULL(pOutput->columnData, i);
+        continue;
+      }
+      memcpy(tmp, varDataVal(input), varDataLen(input));
+      tmp[varDataTLen(input)] = 0;
+    }
+
+    if(!tjsonValidateJson(tmp)){
+      colDataAppendNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    colDataAppend(pOutput->columnData, i, input, false);
+    input += varDataTLen(input);
+  }
+  taosMemoryFree(tmp);
+
+  pOutput->numOfRows = pInput->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   int32_t type = GET_PARAM_TYPE(&pInput[0]);
   int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);
diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c
index 8df404f85f..59dab76989 100644
--- a/source/util/src/tjson.c
+++ b/source/util/src/tjson.c
@@ -276,4 +276,37 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void
 }
 
 SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); }
-bool tjsonIsObject(const SJson* pJson) { return cJSON_IsObject(pJson); }
\ No newline at end of file
+
+bool tjsonValidateJson(const char *jIn) {
+  if (!jIn){
+    return false;
+  }
+
+  // set json real data
+  cJSON *root = cJSON_Parse(jIn);
+  if (root == NULL){
+    return false;
+  }
+
+  if(!cJSON_IsObject(root)){
+    return false;
+  }
+  int size = cJSON_GetArraySize(root);
+  for(int i = 0; i < size; i++) {
+    cJSON* item = cJSON_GetArrayItem(root, i);
+    if (!item) {
+      return false;
+    }
+
+    char* jsonKey = item->string;
+    if (!jsonKey) return false;
+    for (size_t j = 0; j < strlen(jsonKey); ++i) {
+      if (isprint(jsonKey[i]) == 0) return false;
+    }
+
+    if (item->type == cJSON_Object || item->type == cJSON_Array) {
+      return false;
+    }
+  }
+  return true;
+}
\ No newline at end of file

From ad299634232e4c07dec38eab35ed6461f250b2bc Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 18 Apr 2022 21:00:35 +0800
Subject: [PATCH 07/22] refactor: modify json compare in where condition

---
 source/client/src/clientImpl.c     | 72 ++++++++++++++++++++++++++-
 source/libs/scalar/inc/filterInt.h |  2 -
 source/libs/scalar/src/filter.c    | 80 +++++++++---------------------
 3 files changed, 94 insertions(+), 60 deletions(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 74b8e711dc..c1351d0fc8 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -619,7 +619,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
     int32_t bytes = pResultInfo->fields[i].bytes;
 
     if (IS_VAR_DATA_TYPE(type)) {
-      if (pCol->offset[pResultInfo->current] != -1) {
+      if (!colDataIsNull_var(pCol,pResultInfo->current)) {
         char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
 
         pResultInfo->length[i] = varDataLen(pStart);
@@ -725,6 +725,76 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
       pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
       pResultInfo->row[i] = pResultInfo->pCol[i].pData;
     }
+
+    if (type == TSDB_DATA_TYPE_JSON) {
+      char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
+      if (p == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+
+      pResultInfo->convertBuf[i] = p;
+      int32_t len = 0;
+      SResultColumn* pCol = &pResultInfo->pCol[i];
+      for (int32_t j = 0; j < numOfRows; ++j) {
+        if (pCol->offset[j] != -1) {
+          char* pStart = pCol->offset[j] + pCol->pData;
+
+          int32_t jsonInnerType = *pStart;
+          char *jsonInnerData = pStart + CHAR_BYTES;
+          char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
+          if(jsonInnerType == TSDB_DATA_TYPE_NULL){
+            sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else if(jsonInnerType == TSDB_DATA_TYPE_JSON){
+            int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
+
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length);
+          }else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {   // value -> "value"
+            *(char*)varDataVal(dst) = '\"';
+            int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst) + CHAR_BYTES);
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length + CHAR_BYTES*2);
+            *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
+          }else if(jsonInnerType == TSDB_DATA_TYPE_DOUBLE){
+            double jsonVd = *(double*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%.9lf", jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else if(jsonInnerType == TSDB_DATA_TYPE_BIGINT){
+            int64_t jsonVd = *(int64_t*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%" PRId64, jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else if(jsonInnerType == TSDB_DATA_TYPE_BOOL){
+            sprintf(varDataVal(dst), "%s", (*((char *)jsonInnerData) == 1) ? "true" : "false");
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          }else {
+            ASSERT(0);
+          }
+
+          if(len + varDataTLen(dst) > colLength[i]){
+            p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
+            if (p == NULL) {
+              return TSDB_CODE_OUT_OF_MEMORY;
+            }
+
+            pResultInfo->convertBuf[i] = p;
+          }
+          p = pResultInfo->convertBuf[i] + len;
+          memcpy(p, dst, varDataTLen(dst));
+          pCol->offset[j] = len;
+          len += varDataTLen(dst);
+        }
+      }
+
+      pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
+      pResultInfo->row[i] = pResultInfo->pCol[i].pData;
+    }
   }
 
   return TSDB_CODE_SUCCESS;
diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h
index 977fab2e85..2b9e3c7f39 100644
--- a/source/libs/scalar/inc/filterInt.h
+++ b/source/libs/scalar/inc/filterInt.h
@@ -308,7 +308,6 @@ struct SFilterInfo {
 #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri)))
 #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
 #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
-#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
 #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
 
 #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
@@ -321,7 +320,6 @@ struct SFilterInfo {
 #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
-#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
 #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
 #define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
 #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c
index ea249005b4..ee85b99ad5 100644
--- a/source/libs/scalar/src/filter.c
+++ b/source/libs/scalar/src/filter.c
@@ -63,22 +63,18 @@ OptrStr gOptrStr[] = {
 
 bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(maxv, minr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result >= 0;
 }
 bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(maxv, minr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result > 0;
 }
 bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(minv, maxr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result <= 0;
 }
 bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(minv, maxr);
-  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
   return result < 0;
 }
 bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
@@ -1783,9 +1779,9 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
       bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
 
       fi->data = taosMemoryCalloc(1, bytes);
-    } else if (type != TSDB_DATA_TYPE_JSON){
+    } else{
       if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) {  //TIME RANGE
-/*      
+/*
         fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes);
         for (int32_t a = 0; a < dType->bytes; ++a) {
           int64_t *v = taosArrayGet(var->arr, a);
@@ -1796,31 +1792,27 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
       } else {
         fi->data = taosMemoryCalloc(1, sizeof(int64_t));
       }
-    } else{     // type == TSDB_DATA_TYPE_JSON
-      // fi->data = null;  use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
     }
 
-    if(type != TSDB_DATA_TYPE_JSON) {
-      if (dType->type == type) {
-        assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
+    if (dType->type == type) {
+      assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
+    } else {
+      SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
+      out.columnData->info.type = type;
+      if (IS_VAR_DATA_TYPE(type)) {
+        out.columnData->info.bytes = bytes;
       } else {
-        SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
-        out.columnData->info.type = type;
-        if (IS_VAR_DATA_TYPE(type)) {
-          out.columnData->info.bytes = bytes;
-        } else {
-          out.columnData->info.bytes = tDataTypes[type].bytes;
-        }
-
-        // todo refactor the convert
-        int32_t code = doConvertDataType(var, &out);
-        if (code != TSDB_CODE_SUCCESS) {
-          qError("convert value to type[%d] failed", type);
-          return TSDB_CODE_TSC_INVALID_OPERATION;
-        }
-
-        memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
+        out.columnData->info.bytes = tDataTypes[type].bytes;
       }
+
+      // todo refactor the convert
+      int32_t code = doConvertDataType(var, &out);
+      if (code != TSDB_CODE_SUCCESS) {
+        qError("convert value to type[%d] failed", type);
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+
+      memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
     }
 
     // match/nmatch for nchar type need convert from ucs4 to mbs
@@ -2561,11 +2553,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
     
     if (unit->right.type == FLD_TYPE_VALUE) {
-      if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){   // json value is tVariant
-        info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
-      }else{
-        info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
-      }
+      info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
     } else {
       info->cunits[i].valData = NULL;
     }
@@ -2891,18 +2879,8 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
   for (int32_t i = 0; i < numOfRows; ++i) {
     uint32_t uidx = info->groups[0].unitIdxs[0];
     void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
-    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
-      if (!colData){  // for json->'key' is null
-        (*p)[i] = 1;
-      }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){  // for json is null
-        colData = POINTER_SHIFT(colData, CHAR_BYTES);
-        (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
-      }else{
-        (*p)[i] = 0;
-      }
-    }else{
-      (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
-    }
+    (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
+
     if ((*p)[i] == 0) {
       all = false;
     }    
@@ -2926,19 +2904,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
     uint32_t uidx = info->groups[0].unitIdxs[0];
     void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i);
 
-    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
-      if (!colData) {   // for json->'key' is not null
-        (*p)[i] = 0;
-      }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){   // for json is not null
-        colData = POINTER_SHIFT(colData, CHAR_BYTES);
-        (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL);
-      }else{    // for json->'key' is not null
-        (*p)[i] = 1;
-      }
-    }else {
-      (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
-    }
-
+    (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL));
     if ((*p)[i] == 0) {
       all = false;
     }

From 134a161c7205157157395b3adc23a48be8525d9e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 18 Apr 2022 21:05:34 +0800
Subject: [PATCH 08/22] refactor: modify json compare in where condition

---
 source/client/src/clientImpl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index c1351d0fc8..49f663772d 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -619,7 +619,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
     int32_t bytes = pResultInfo->fields[i].bytes;
 
     if (IS_VAR_DATA_TYPE(type)) {
-      if (!colDataIsNull_var(pCol,pResultInfo->current)) {
+      if (pCol->offset[pResultInfo->current] != -1) {
         char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
 
         pResultInfo->length[i] = varDataLen(pStart);

From b96082fe1b36468466e78e76c6ff9017da8f280a Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 19 Apr 2022 09:46:12 +0800
Subject: [PATCH 09/22] refactor: fix compare logic

---
 source/libs/scalar/src/filter.c    |  9 +++++++
 source/libs/scalar/src/scalar.c    |  1 +
 source/libs/scalar/src/sclvector.c | 39 +++++++++++++++---------------
 3 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c
index ee85b99ad5..23047c2930 100644
--- a/source/libs/scalar/src/filter.c
+++ b/source/libs/scalar/src/filter.c
@@ -1050,6 +1050,8 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
 
       cell = cell->pNext;
     }
+    colDataDestroy(out.columnData);
+    taosMemoryFree(out.columnData);
   } else {
     filterAddFieldFromNode(info, node->pRight, &right);
     
@@ -1813,6 +1815,8 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
       }
 
       memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes);
+      colDataDestroy(out.columnData);
+      taosMemoryFree(out.columnData);
     }
 
     // match/nmatch for nchar type need convert from ucs4 to mbs
@@ -3537,6 +3541,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
         return DEAL_RES_CONTINUE;
       }
 
+      if (OP_TYPE_JSON_CONTAINS == node->opType) {
+        stat->scalarMode = true;
+        return DEAL_RES_CONTINUE;
+      }
+
       if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
         SNode *t = node->pLeft;
         node->pLeft = node->pRight;
diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index 5f1e08dbcf..9c4d2d94dd 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -132,6 +132,7 @@ void sclFreeRes(SHashObj *res) {
 void sclFreeParam(SScalarParam *param) {
   if (param->columnData != NULL) {
     colDataDestroy(param->columnData);
+    taosMemoryFree(param->columnData);
   }
 
   if (param->pHashFilter != NULL) {
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index e6deef5824..e97831b1f6 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -188,17 +188,17 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd
   colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v);
 }
 
-//static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
-//  int32_t len = 0;
-//  int32_t inputLen = varDataLen(buf);
-//
-//  char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
-//  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
-//  varDataSetLen(t, len);
-//
-//  colDataAppend(pOut->columnData, rowIndex, t, false);
-//  taosMemoryFree(t);
-//}
+static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
+  int32_t len = 0;
+  int32_t inputLen = varDataLen(buf);
+
+  char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
+  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
+  varDataSetLen(t, len);
+
+  colDataAppend(pOut->columnData, rowIndex, t, false);
+  taosMemoryFree(t);
+}
 
 void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
   switch (outType) {
@@ -275,7 +275,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
   int32_t bufSize = pIn->columnData->info.bytes;
   char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE);
 
-//  bool vton = false;
+  bool vton = false;
 
   _bufConverteFunc func = NULL;
   if (TSDB_DATA_TYPE_BOOL == outType) {
@@ -286,9 +286,10 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     func = varToUnsigned;
   } else if (IS_FLOAT_TYPE(outType)) {
     func = varToFloat;
-//  } else if (outType == TSDB_DATA_TYPE_NCHAR) {   // can not be nchar or binary
-//    func = varToNchar;
-//    vton = true;
+  } else if (outType == TSDB_DATA_TYPE_NCHAR) {   // binary -> nchar
+    ASSERT(inType == TSDB_DATA_TYPE_VARCHAR);
+    func = varToNchar;
+    vton = true;
   } else {
     sclError("invalid convert outType:%d", outType);
     return TSDB_CODE_QRY_APP_ERROR;
@@ -316,9 +317,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
         continue;
       }
     }
-//    if (vton) {
-//      memcpy(tmp, data, varDataTLen(data));
-//    } else {
+    if (vton) {
+      memcpy(tmp, data, varDataTLen(data));
+    } else {
       if (TSDB_DATA_TYPE_VARCHAR == convertType) {
         memcpy(tmp, varDataVal(data), varDataLen(data));
         tmp[varDataLen(data)] = 0;
@@ -334,7 +335,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
 
         tmp[len] = 0;
       }
-//    }
+    }
     
     (*func)(tmp, pOut, i);
   }

From b7bf0fe10bc85243fdfe3e623b0260c3da7bfb4d Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 19 Apr 2022 17:02:49 +0800
Subject: [PATCH 10/22] refactor(tmq): subscribe and rebalance process

---
 include/common/tmsg.h                      |  62 ++-
 include/common/tmsgdef.h                   |   2 +
 include/dnode/mnode/sdb/sdb.h              |   8 +-
 include/util/talgo.h                       |   5 +
 include/util/taoserror.h                   |   1 +
 include/util/tarray.h                      |  26 +-
 source/client/src/tmq.c                    |   9 +-
 source/common/src/tmsg.c                   |   8 +-
 source/dnode/mnode/impl/inc/mndConsumer.h  |  17 +-
 source/dnode/mnode/impl/inc/mndDef.h       | 134 +++++-
 source/dnode/mnode/impl/inc/mndSubscribe.h |   4 +-
 source/dnode/mnode/impl/src/mndConsumer.c  | 536 ++++++++++++++++++++-
 source/dnode/mnode/impl/src/mndDef.c       | 380 +++++++++++++++
 source/dnode/mnode/impl/src/mndScheduler.c |  20 +-
 source/dnode/mnode/impl/src/mndSubscribe.c | 492 ++++++++++++++++++-
 source/util/src/tarray.c                   |  41 +-
 16 files changed, 1660 insertions(+), 85 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 050a414254..2e0882097c 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -329,7 +329,7 @@ typedef struct SEpSet {
 int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp);
 int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
 int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
-void*   taosDecodeSEpSet(void* buf, SEpSet* pEp);
+void*   taosDecodeSEpSet(const void* buf, SEpSet* pEp);
 
 typedef struct {
   int8_t  connType;
@@ -1290,10 +1290,14 @@ typedef struct {
 int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
 int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
 
+typedef struct {
+  int64_t consumerId;
+} SMqConsumerLostMsg;
+
 typedef struct {
   int32_t topicNum;
   int64_t consumerId;
-  char*   consumerGroup;
+  char    cgroup[TSDB_CGROUP_LEN];
   SArray* topicNames;  // SArray<char*>
 } SCMSubscribeReq;
 
@@ -1301,7 +1305,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pReq->topicNum);
   tlen += taosEncodeFixedI64(buf, pReq->consumerId);
-  tlen += taosEncodeString(buf, pReq->consumerGroup);
+  tlen += taosEncodeString(buf, pReq->cgroup);
 
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i));
@@ -1312,7 +1316,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
 static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
   buf = taosDecodeFixedI32(buf, &pReq->topicNum);
   buf = taosDecodeFixedI64(buf, &pReq->consumerId);
-  buf = taosDecodeString(buf, &pReq->consumerGroup);
+  buf = taosDecodeStringTo(buf, pReq->cgroup);
   pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*));
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     char* name;
@@ -1388,10 +1392,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq
 }
 
 typedef struct {
-  const char* key;
-  SArray*     lostConsumers;     // SArray<int64_t>
-  SArray*     removedConsumers;  // SArray<int64_t>
-  SArray*     newConsumers;      // SArray<int64_t>
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
+  SArray* lostConsumers;     // SArray<int64_t>
+  SArray* removedConsumers;  // SArray<int64_t>
+  SArray* newConsumers;      // SArray<int64_t>
 } SMqRebSubscribe;
 
 static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) {
@@ -1399,7 +1403,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) {
   if (pRebSub == NULL) {
     goto _err;
   }
-  pRebSub->key = strdup(key);
+  strcpy(pRebSub->key, key);
   pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t));
   if (pRebSub->lostConsumers == NULL) {
     goto _err;
@@ -1424,6 +1428,7 @@ _err:
 // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or
 // deserialization
 typedef struct {
+  int8_t*   mqInReb;
   SHashObj* rebSubHash;  // SHashObj<key, SMqRebSubscribe>
 } SMqDoRebalanceMsg;
 
@@ -1876,6 +1881,40 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
   return buf;
 }
 
+typedef struct {
+  int64_t leftForVer;
+  int32_t vgId;
+  int64_t oldConsumerId;
+  int64_t newConsumerId;
+  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  char*   qmsg;
+} SMqRebVgReq;
+
+static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
+  tlen += taosEncodeFixedI32(buf, pReq->vgId);
+  tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
+  tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
+  tlen += taosEncodeString(buf, pReq->subKey);
+  tlen += taosEncodeString(buf, pReq->qmsg);
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) {
+  buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
+  buf = taosDecodeFixedI32(buf, &pReq->vgId);
+  buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
+  buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
+  buf = taosDecodeStringTo(buf, pReq->subKey);
+  buf = taosDecodeString(buf, &pReq->qmsg);
+  return (void*)buf;
+}
+
+typedef struct {
+  int8_t reserved;
+} SMqRebVgRsp;
+
 typedef struct {
   int64_t leftForVer;
   int32_t vgId;
@@ -2441,7 +2480,10 @@ typedef struct {
   SArray*    topics;  // SArray<SMqSubTopicEp>
 } SMqCMGetSubEpRsp;
 
-static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
+static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
+  taosMemoryFree(pSubTopicEp->schema.pSchema);
+  taosArrayDestroy(pSubTopicEp->vgs);
+}
 
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
   int32_t tlen = 0;
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index e553dff270..0bc1ea6924 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -149,6 +149,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
@@ -180,6 +181,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h
index a28c093e85..d7d53ad1d0 100644
--- a/include/dnode/mnode/sdb/sdb.h
+++ b/include/dnode/mnode/sdb/sdb.h
@@ -92,7 +92,13 @@ extern "C" {
 typedef struct SMnode  SMnode;
 typedef struct SSdbRaw SSdbRaw;
 typedef struct SSdbRow SSdbRow;
-typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType;
+
+typedef enum {
+  SDB_KEY_BINARY = 1,
+  SDB_KEY_INT32 = 2,
+  SDB_KEY_INT64 = 3,
+} EKeyType;
+
 typedef enum {
   SDB_STATUS_INIT = 0,
   SDB_STATUS_CREATING = 1,
diff --git a/include/util/talgo.h b/include/util/talgo.h
index 0fd44a6e91..ebddce62a8 100644
--- a/include/util/talgo.h
+++ b/include/util/talgo.h
@@ -27,6 +27,11 @@ extern "C" {
 typedef int32_t (*__compar_fn_t)(const void *, const void *);
 #endif
 
+typedef void *(*FCopy)(void *);
+typedef void (*FDelete)(void *);
+typedef int32_t (*FEncode)(void **buf, const void *dst);
+typedef void *(*FDecode)(const void *buf, void *dst);
+
 #define TD_EQ 0x1
 #define TD_GT 0x2
 #define TD_LT 0x4
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 60535e2f49..fe434fb7f5 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -279,6 +279,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
diff --git a/include/util/tarray.h b/include/util/tarray.h
index 521e54040d..3a97c51c1c 100644
--- a/include/util/tarray.h
+++ b/include/util/tarray.h
@@ -41,10 +41,10 @@ extern "C" {
 #define TARRAY_GET_START(array)       ((array)->pData)
 
 typedef struct SArray {
-  size_t size;
+  size_t   size;
   uint32_t capacity;
   uint32_t elemSize;
-  void*  pData;
+  void*    pData;
 } SArray;
 
 /**
@@ -199,24 +199,21 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize);
  */
 SArray* taosArrayDup(const SArray* pSrc);
 
+/**
+ * deep copy a new array
+ * @param pSrc
+ */
+SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
+
 /**
  * clear the array (remove all element)
  * @param pArray
  */
 void taosArrayClear(SArray* pArray);
 
-/**
- * destroy array list
- * @param pArray
- */
 void* taosArrayDestroy(SArray* pArray);
-
-/**
- *
- * @param pArray
- * @param fp
- */
-void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
+void  taosArrayDestroyP(SArray* pArray, FDelete fp);
+void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
 
 /**
  * sort the array
@@ -264,6 +261,9 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t
 
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
 
+int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
+void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index 478e328a16..9ea7a85d9b 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -255,7 +255,12 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
 void tmq_list_destroy(tmq_list_t* list) {
   SArray* container = &list->container;
   /*taosArrayDestroy(container);*/
-  taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree);
+  int32_t sz = taosArrayGetSize(container);
+  for (int32_t i = 0; i < sz; i++) {
+    char* str = taosArrayGetP(container, i);
+    taosMemoryFree(str);
+  }
+  taosArrayDestroy(container);
 }
 
 static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
@@ -496,7 +501,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
   SCMSubscribeReq req;
   req.topicNum = sz;
   req.consumerId = tmq->consumerId;
-  req.consumerGroup = strdup(tmq->groupId);
+  strcpy(req.cgroup, tmq->groupId);
   req.topicNames = taosArrayInit(sz, sizeof(void*));
 
   for (int i = 0; i < sz; i++) {
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 821f24eedc..e4b1ecaa7e 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -121,14 +121,14 @@ int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   return tlen;
 }
 
-void *taosDecodeSEpSet(void *buf, SEpSet *pEp) {
+void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) {
   buf = taosDecodeFixedI8(buf, &pEp->inUse);
   buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
     buf = taosDecodeFixedU16(buf, &pEp->eps[i].port);
     buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn);
   }
-  return buf;
+  return (void *)buf;
 }
 
 static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) {
@@ -2184,7 +2184,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->showId) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->type) < 0) return -1;
-//  if (tEncodeI8(&encoder, pReq->free) < 0) return -1;
+  //  if (tEncodeI8(&encoder, pReq->free) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
   tEndEncode(&encoder);
@@ -2201,7 +2201,7 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
-//  if (tDecodeI8(&decoder, &pReq->free) < 0) return -1;
+  //  if (tDecodeI8(&decoder, &pReq->free) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
   tEndDecode(&decoder);
diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h
index cc345379cf..a818f5d7c3 100644
--- a/source/dnode/mnode/impl/inc/mndConsumer.h
+++ b/source/dnode/mnode/impl/inc/mndConsumer.h
@@ -22,27 +22,30 @@
 extern "C" {
 #endif
 
-
 enum {
-  MQ_CONSUMER_STATUS__INIT = 1,
-  MQ_CONSUMER_STATUS__IDLE,
-  MQ_CONSUMER_STATUS__ACTIVE,
+  // MQ_CONSUMER_STATUS__INIT = 1,
+  MQ_CONSUMER_STATUS__MODIFY = 1,
+  MQ_CONSUMER_STATUS__MODIFY_IN_REB,
+  // MQ_CONSUMER_STATUS__IDLE,
+  MQ_CONSUMER_STATUS__READY,
   MQ_CONSUMER_STATUS__LOST,
-  MQ_CONSUMER_STATUS__MODIFY
+  MQ_CONSUMER_STATUS__LOST_IN_REB,
+  MQ_CONSUMER_STATUS__LOST_REBD,
 };
 
-
 int32_t mndInitConsumer(SMnode *pMnode);
 void    mndCleanupConsumer(SMnode *pMnode);
 
 SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId);
 void            mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
 
-SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup);
+SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup);
 
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
 SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
 
+int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index eaec99c186..a96be41308 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -88,6 +88,7 @@ typedef enum {
   TRN_TYPE_CREATE_STREAM = 1019,
   TRN_TYPE_DROP_STREAM = 1020,
   TRN_TYPE_ALTER_STREAM = 1021,
+  TRN_TYPE_CONSUMER_LOST = 1022,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -506,6 +507,7 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset)
   return buf;
 }
 
+#if 0
 typedef struct {
   char    key[TSDB_SUBSCRIBE_KEY_LEN];
   int32_t status;
@@ -636,6 +638,7 @@ static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
     pSub->unassignedVg = NULL;
   }
 }
+#endif
 
 typedef struct {
   char           name[TSDB_TOPIC_FNAME_LEN];
@@ -654,6 +657,7 @@ typedef struct {
   SSchemaWrapper schema;
 } SMqTopicObj;
 
+#if 0
 typedef struct {
   int64_t  consumerId;
   int64_t  connId;
@@ -708,7 +712,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
   buf = taosDecodeStringTo(buf, pConsumer->cgroup);
 
   buf = taosDecodeFixedI32(buf, &sz);
-  pConsumer->currentTopics = taosArrayInit(sz, sizeof(SMqConsumerObj));
+  pConsumer->currentTopics = taosArrayInit(sz, sizeof(void*));
   for (int32_t i = 0; i < sz; i++) {
     char* topic;
     buf = taosDecodeString(buf, &topic);
@@ -716,7 +720,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
   }
 
   buf = taosDecodeFixedI32(buf, &sz);
-  pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(SMqConsumerObj));
+  pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(void*));
   for (int32_t i = 0; i < sz; i++) {
     char* topic;
     buf = taosDecodeString(buf, &topic);
@@ -724,6 +728,132 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
   }
   return buf;
 }
+#endif
+
+enum {
+  CONSUMER_UPDATE__TOUCH = 1,
+  CONSUMER_UPDATE__ADD,
+  CONSUMER_UPDATE__REMOVE,
+  CONSUMER_UPDATE__LOST,
+  CONSUMER_UPDATE__MODIFY,
+};
+
+typedef struct {
+  int64_t consumerId;
+  char    cgroup[TSDB_CGROUP_LEN];
+  int8_t  updateType;  // used only for update
+  int32_t epoch;
+  int32_t status;
+  // hbStatus is not applicable to serialization
+  int32_t hbStatus;
+  // lock is used for topics update
+  SRWLatch lock;
+  SArray*  currentTopics;  // SArray<char*>
+#if 0
+  SArray*  waitingRebTopics;  // SArray<char*>
+#endif
+  SArray* rebNewTopics;      // SArray<char*>
+  SArray* rebRemovedTopics;  // SArray<char*>
+} SMqConsumerObj;
+
+SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);
+void            tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer);
+int32_t         tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer);
+void*           tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer);
+
+typedef struct {
+  int32_t vgId;
+  char*   qmsg;
+  // char    topic[TSDB_TOPIC_FNAME_LEN];
+  SEpSet epSet;
+} SMqVgEp;
+
+SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp);
+void     tDeleteSMqVgEp(SMqVgEp* pVgEp);
+int32_t  tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp);
+void*    tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp);
+
+typedef struct {
+  int64_t consumerId;  // -1 for unassigned
+  SArray* vgs;         // SArray<SMqVgEp*>
+} SMqConsumerEpInSub;
+
+SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub);
+void                tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub);
+int32_t             tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub);
+void*               tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub);
+
+typedef struct {
+  char      key[TSDB_SUBSCRIBE_KEY_LEN];
+  SRWLatch  lock;
+  int32_t   vgNum;
+  SHashObj* consumerHash;  // consumerId -> SMqConsumerEpInSub
+} SMqSubscribeObj;
+
+SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
+SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub);
+void             tDeleteSubscribeObj(SMqSubscribeObj* pSub);
+int32_t          tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub);
+void*            tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub);
+
+typedef struct {
+  int32_t epoch;
+  SArray* consumers;  // SArray<SMqConsumerEpInSub*>
+} SMqSubActionLogEntry;
+
+SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+void                  tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+int32_t               tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry);
+void*                 tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry);
+
+typedef struct {
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
+  SArray* logs;  // SArray<SMqSubActionLogEntry*>
+} SMqSubActionLogObj;
+
+SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+void                tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+int32_t             tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog);
+void*               tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog);
+
+typedef struct {
+  int64_t  consumerId;
+  char     cgroup[TSDB_CGROUP_LEN];
+  SRWLatch lock;
+  SArray*  vgs;  // SArray<SMqVgEp*>
+} SMqConsumerEpObj;
+
+SMqConsumerEpObj* tCloneSMqConsumerEpObj(const SMqConsumerEpObj* pConsumerEp);
+void              tDeleteSMqConsumerEpObj(SMqConsumerEpObj* pConsumerEp);
+int32_t           tEncodeSMqConsumerEpObj(void** buf, const SMqConsumerEpObj* pConsumerEp);
+void*             tDecodeSMqConsumerEpObj(const void* buf, SMqConsumerEpObj* pConsumerEp);
+
+typedef struct {
+  const SMqSubscribeObj* pOldSub;
+  const SMqTopicObj*     pTopic;
+  const SMqRebSubscribe* pRebInfo;
+} SMqRebInputObj;
+
+typedef struct {
+  int64_t  oldConsumerId;
+  int64_t  newConsumerId;
+  SMqVgEp* pVgEp;
+} SMqRebOutputVg;
+
+#if 0
+typedef struct {
+  int64_t consumerId;
+} SMqRebOutputConsumer;
+#endif
+
+typedef struct {
+  SArray*               rebVgs;            // SArray<SMqRebOutputVg>
+  SArray*               newConsumers;      // SArray<int64_t>
+  SArray*               removedConsumers;  // SArray<int64_t>
+  SArray*               touchedConsumers;  // SArray<int64_t>
+  SMqSubscribeObj*      pSub;
+  SMqSubActionLogEntry* pLogEntry;
+} SMqRebOutputObj;
 
 typedef struct {
   char     name[TSDB_TOPIC_FNAME_LEN];
diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h
index 3f897067a2..7915006f27 100644
--- a/source/dnode/mnode/impl/inc/mndSubscribe.h
+++ b/source/dnode/mnode/impl/inc/mndSubscribe.h
@@ -26,9 +26,11 @@ int32_t mndInitSubscribe(SMnode *pMnode);
 void    mndCleanupSubscribe(SMnode *pMnode);
 
 SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName);
-SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char* key);
+SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key);
 void             mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
 
+int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index d2a3c38135..8a66e5cb69 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -19,8 +19,10 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndShow.h"
 #include "mndStb.h"
+#include "mndSubscribe.h"
 #include "mndTopic.h"
 #include "mndTrans.h"
 #include "mndUser.h"
@@ -28,9 +30,13 @@
 #include "tcompare.h"
 #include "tname.h"
 
-#define MND_CONSUMER_VER_NUMBER 1
+#define MND_CONSUMER_VER_NUMBER   1
 #define MND_CONSUMER_RESERVE_SIZE 64
 
+#define MND_CONSUMER_LOST_HB_CNT 3
+
+static int8_t mqInRebFlag = 0;
+
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
@@ -38,6 +44,11 @@ static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg);
 static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConsumer(SMnode *pMnode, void *pIter);
 
+static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
+static int32_t mndProcessAskEpReq(SNodeMsg *pMsg);
+static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
+static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg);
+
 int32_t mndInitConsumer(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_CONSUMER,
                      .keyType = SDB_KEY_INT64,
@@ -47,25 +58,392 @@ int32_t mndInitConsumer(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndConsumerActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
 
+  mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg);
   return sdbSetTable(pMnode->pSdb, table);
 }
 
 void mndCleanupConsumer(SMnode *pMnode) {}
 
-SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) {
-  SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
-  if (pConsumer == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
+static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) {
+  SMnode             *pMnode = pMsg->pNode;
+  SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont;
+  SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
+  ASSERT(pConsumer);
+
+  SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
+  pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
+
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_LOST, &pMsg->rpcMsg);
+  if (pTrans == NULL) goto FAIL;
+  if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+FAIL:
+  // TODO delete consumer
+  mndTransDrop(pTrans);
+  return -1;
+}
+
+static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
+  SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1);
+  if (pRebSub == NULL) {
+    pRebSub = tNewSMqRebSubscribe(key);
+    if (pRebSub == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+    taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe));
+  }
+  return pRebSub;
+}
+
+static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
+  SMnode         *pMnode = pMsg->pNode;
+  SSdb           *pSdb = pMnode->pSdb;
+  SMqConsumerObj *pConsumer;
+  void           *pIter = NULL;
+
+  // rebalance cannot be parallel
+  int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1);
+  if (old != 0) {
+    mInfo("mq rebalance already in progress, do nothing");
+    return 0;
   }
 
-  pConsumer->recentRemovedTopics = taosArrayInit(1, sizeof(char *));
-  pConsumer->epoch = 1;
-  pConsumer->consumerId = consumerId;
-  atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT);
-  strcpy(pConsumer->cgroup, cgroup);
-  taosInitRWLatch(&pConsumer->lock);
-  return pConsumer;
+  SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
+  pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
+  // TODO set cleanfp
+  pRebMsg->mqInReb = &mqInRebFlag;
+
+  // iterate all consumers, find all modification
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
+    if (pIter == NULL) break;
+
+    taosRLockLatch(&pConsumer->lock);
+    int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
+    int32_t status = atomic_load_32(&pConsumer->status);
+    if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) {
+      SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg));
+
+      pLostMsg->consumerId = pConsumer->consumerId;
+      SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
+      pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_LOST;
+      pRpcMsg->pCont = pLostMsg;
+      pRpcMsg->contLen = sizeof(SMqConsumerLostMsg);
+      tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
+    }
+    if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) {
+      // do nothing
+    } else if (status == MQ_CONSUMER_STATUS__LOST) {
+      int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics);
+      for (int32_t i = 0; i < topicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
+      }
+    } else if (status == MQ_CONSUMER_STATUS__MODIFY) {
+      int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
+      for (int32_t i = 0; i < newTopicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId);
+      }
+
+      int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics);
+      for (int32_t i = 0; i < removedTopicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
+      }
+    } else {
+      // do nothing
+    }
+
+    taosRUnLockLatch(&pConsumer->lock);
+    mndReleaseConsumer(pMnode, pConsumer);
+  }
+
+  if (taosHashGetSize(pRebMsg->rebSubHash) != 0) {
+    mInfo("mq rebalance will be triggered");
+    SRpcMsg rpcMsg = {
+        .msgType = TDMT_MND_MQ_DO_REBALANCE,
+        .pCont = pRebMsg,
+        .contLen = sizeof(SMqDoRebalanceMsg),
+    };
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+  } else {
+    taosHashCleanup(pRebMsg->rebSubHash);
+    rpcFreeCont(pRebMsg);
+    mInfo("mq rebalance finished, no modification");
+    atomic_store_8(&mqInRebFlag, 0);
+  }
+  return 0;
+}
+
+static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
+  SMnode           *pMnode = pMsg->pNode;
+  SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
+  SMqCMGetSubEpRsp  rsp = {0};
+  int64_t           consumerId = be64toh(pReq->consumerId);
+  int32_t           epoch = ntohl(pReq->epoch);
+
+  SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId);
+  if (pConsumer == NULL) {
+    terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
+    return -1;
+  }
+
+  ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
+  /*int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);*/
+  atomic_store_32(&pConsumer->hbStatus, 0);
+
+  // 1. check consumer status
+  int32_t status = atomic_load_32(&pConsumer->status);
+  if (status == MQ_CONSUMER_STATUS__LOST) {
+    // recover consumer
+  }
+
+  if (status != MQ_CONSUMER_STATUS__READY) {
+    mndReleaseConsumer(pMnode, pConsumer);
+    terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
+    return -1;
+  }
+
+  int32_t serverEpoch = atomic_load_32(&pConsumer->epoch);
+
+  // 2. check epoch, only send ep info when epoches do not match
+  if (epoch != serverEpoch) {
+    taosRLockLatch(&pConsumer->lock);
+    int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics);
+
+    rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp));
+    if (rsp.topics == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto FAIL;
+    }
+
+    // handle all topic subscribed by the consumer
+    for (int32_t i = 0; i < numOfTopics; i++) {
+      char            *topic = taosArrayGetP(pConsumer->currentTopics, i);
+      SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
+
+      // txn guarantees pSub is created
+      ASSERT(pSub);
+      taosRLockLatch(&pSub->lock);
+
+      SMqSubTopicEp topicEp = {0};
+      strcpy(topicEp.topic, topic);
+
+      // 2.1 fetch topic schema
+      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+      ASSERT(pTopic);
+      taosRLockLatch(&pTopic->lock);
+      topicEp.schema.nCols = pTopic->schema.nCols;
+      topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema));
+      memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema));
+      taosRUnLockLatch(&pTopic->lock);
+      mndReleaseTopic(pMnode, pTopic);
+
+      // 2.2 iterate all vg assigned to the consumer of that topic
+      SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
+      int32_t             vgNum = taosArrayGetSize(pEpInSub->vgs);
+
+      topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
+      if (topicEp.vgs == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        goto FAIL;
+      }
+
+      for (int32_t j = 0; j < vgNum; j++) {
+        SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        char     offsetKey[TSDB_PARTITION_KEY_LEN];
+        mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
+        // 2.2.1 build vg ep
+        SMqSubVgEp vgEp = {
+            .epSet = pVgEp->epSet,
+            .vgId = pVgEp->vgId,
+            .offset = -1,
+        };
+
+        // 2.2.2 fetch vg offset
+        SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
+        if (pOffsetObj != NULL) {
+          vgEp.offset = atomic_load_64(&pOffsetObj->offset);
+          mndReleaseOffset(pMnode, pOffsetObj);
+        }
+        taosArrayPush(topicEp.vgs, &vgEp);
+      }
+      taosArrayPush(rsp.topics, &topicEp);
+
+      taosRUnLockLatch(&pSub->lock);
+      mndReleaseSubscribe(pMnode, pSub);
+    }
+    taosRUnLockLatch(&pConsumer->lock);
+  }
+  // encode rsp
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
+  void   *buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
+  ((SMqRspHead *)buf)->epoch = serverEpoch;
+  ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
+
+  void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+  tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
+
+  // release consumer and free memory
+  tDeleteSMqCMGetSubEpRsp(&rsp);
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  // send rsp
+  pMsg->pRsp = buf;
+  pMsg->rspLen = tlen;
+  return 0;
+FAIL:
+  tDeleteSMqCMGetSubEpRsp(&rsp);
+  mndReleaseConsumer(pMnode, pConsumer);
+  return -1;
+}
+
+int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer) {
+  SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
+  if (pCommitRaw == NULL) return -1;
+  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
+  if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
+
+static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
+  SMnode         *pMnode = pMsg->pNode;
+  char           *msgStr = pMsg->rpcMsg.pCont;
+  SCMSubscribeReq subscribe = {0};
+  tDeserializeSCMSubscribeReq(msgStr, &subscribe);
+  int64_t         consumerId = subscribe.consumerId;
+  char           *cgroup = subscribe.cgroup;
+  SMqConsumerObj *pConsumerOld = NULL;
+  SMqConsumerObj *pConsumerNew = NULL;
+
+  int32_t code = -1;
+  SArray *newSub = subscribe.topicNames;
+  taosArraySortString(newSub, taosArrayCompareString);
+
+  int32_t newTopicNum = taosArrayGetSize(newSub);
+  // check topic existance
+  for (int32_t i = 0; i < newTopicNum; i++) {
+    char        *topic = taosArrayGetP(newSub, i);
+    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+    if (pTopic == NULL) {
+      terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
+      goto SUBSCRIBE_OVER;
+    }
+    // TODO lock topic to prevent drop
+    mndReleaseTopic(pMnode, pTopic);
+  }
+
+  pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+  if (pConsumerOld == NULL) {
+    pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
+    /*pConsumerNew->waitingRebTopics = newSub;*/
+    pConsumerNew->rebNewTopics = newSub;
+    subscribe.topicNames = NULL;
+
+    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
+    if (pTrans == NULL) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+
+  } else {
+    taosRLockLatch(&pConsumerOld->lock);
+    int32_t status = atomic_load_32(&pConsumerOld->status);
+    if (status != MQ_CONSUMER_STATUS__READY) {
+      terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
+      goto SUBSCRIBE_OVER;
+    }
+
+    pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
+    if (pConsumerNew == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto SUBSCRIBE_OVER;
+    }
+    pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
+    /*pConsumerOld->waitingRebTopics = newSub;*/
+
+    int32_t oldTopicNum = 0;
+    if (pConsumerOld->currentTopics) {
+      oldTopicNum = taosArrayGetSize(pConsumerOld->currentTopics);
+    }
+
+    int32_t i = 0, j = 0;
+    while (i < oldTopicNum || j < newTopicNum) {
+      if (i >= oldTopicNum) {
+        char *newTopicCopy = strdup(taosArrayGetP(newSub, j));
+        taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
+        j++;
+        continue;
+      } else if (j >= newTopicNum) {
+        char *oldTopicCopy = strdup(taosArrayGetP(pConsumerOld->currentTopics, i));
+        taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
+        i++;
+        continue;
+      } else {
+        char *oldTopic = taosArrayGetP(pConsumerOld->currentTopics, i);
+        char *newTopic = taosArrayGetP(newSub, j);
+        int   comp = compareLenPrefixedStr(oldTopic, newTopic);
+        if (comp == 0) {
+          i++;
+          j++;
+          continue;
+        } else if (comp < 0) {
+          char *oldTopicCopy = strdup(oldTopic);
+          taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
+          i++;
+          continue;
+        } else {
+          char *newTopicCopy = strdup(newTopic);
+          taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
+          j++;
+          continue;
+        }
+      }
+    }
+
+    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
+    if (pTrans == NULL) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+  }
+
+  code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+
+SUBSCRIBE_OVER:
+  if (pConsumerOld) {
+    taosRUnLockLatch(&pConsumerOld->lock);
+    mndReleaseConsumer(pMnode, pConsumerOld);
+  }
+  if (pConsumerNew) {
+    tDeleteSMqConsumerObj(pConsumerNew);
+  }
+  // TODO: replace with destroy subscribe msg
+  if (subscribe.topicNames) taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
+  return code;
 }
 
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
@@ -154,15 +532,141 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
   mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId);
+  tDeleteSMqConsumerObj(pConsumer);
   return 0;
 }
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
-  /*taosWLockLatch(&pOldConsumer->lock);*/
-  atomic_add_fetch_32(&pOldConsumer->epoch, 1);
-  /*taosWUnLockLatch(&pOldConsumer->lock);*/
 
+  taosWLockLatch(&pOldConsumer->lock);
+
+  if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) {
+    ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
+    SArray *tmp = pOldConsumer->rebNewTopics;
+    pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics;
+    pNewConsumer->rebNewTopics = tmp;
+
+    tmp = pOldConsumer->rebRemovedTopics;
+    pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics;
+    pNewConsumer->rebRemovedTopics = tmp;
+
+    pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
+    int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
+    pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i));
+      taosArrayPush(pNewConsumer->rebRemovedTopics, &topic);
+    }
+    pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) {
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
+    ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);
+    ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
+
+    char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
+    // not exist in current topic
+#if 1
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
+      ASSERT(strcmp(topic, addedTopic) != 0);
+    }
+#endif
+
+    // remove from new topic
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      if (strcmp(addedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->rebNewTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+
+    // add to current topic
+    taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
+    taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString);
+    // set status
+    if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
+      }
+    } else {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
+      }
+    }
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
+    ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);
+    char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
+
+    // not exist in new topic
+#if 1
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      ASSERT(strcmp(topic, removedTopic) != 0);
+    }
+#endif
+
+    // remove from removed topic
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      if (strcmp(removedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->rebNewTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+
+    // remove from current topic
+    int32_t i = 0;
+    int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
+    for (i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
+      if (strcmp(removedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->currentTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+    // must find the topic
+    ASSERT(i < sz);
+
+    // set status
+    if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
+      }
+    } else {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
+      }
+    }
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  }
+
+  taosWUnLockLatch(&pOldConsumer->lock);
   return 0;
 }
 
diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index 6374b4cad2..dd2d6801e1 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -14,6 +14,386 @@
  */
 
 #include "mndDef.h"
+#include "mndConsumer.h"
+
+SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) {
+  SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
+  if (pConsumer == NULL) {
+    return NULL;
+  }
+
+  pConsumer->consumerId = consumerId;
+  memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN);
+
+  pConsumer->epoch = 0;
+  pConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
+  pConsumer->hbStatus = 0;
+
+  taosInitRWLatch(&pConsumer->lock);
+
+  pConsumer->currentTopics = taosArrayInit(0, sizeof(void *));
+#if 0
+  pConsumer->waitingRebTopics = NULL;
+#endif
+  pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
+  pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
+
+  if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) {
+    taosArrayDestroy(pConsumer->currentTopics);
+    taosArrayDestroy(pConsumer->rebNewTopics);
+    taosArrayDestroy(pConsumer->rebRemovedTopics);
+    taosMemoryFree(pConsumer);
+    return NULL;
+  }
+
+  return pConsumer;
+}
+
+void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
+  if (pConsumer->currentTopics) {
+    taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree);
+  }
+#if 0
+  if (pConsumer->waitingRebTopics) {
+    taosArrayDestroyP(pConsumer->waitingRebTopics, taosMemoryFree);
+  }
+#endif
+  if (pConsumer->rebNewTopics) {
+    taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree);
+  }
+  if (pConsumer->rebRemovedTopics) {
+    taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
+  }
+}
+
+int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
+  int32_t tlen = 0;
+  int32_t sz;
+  tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
+  tlen += taosEncodeString(buf, pConsumer->cgroup);
+  tlen += taosEncodeFixedI8(buf, pConsumer->updateType);
+  tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
+  tlen += taosEncodeFixedI32(buf, pConsumer->status);
+
+  // current topics
+  if (pConsumer->currentTopics) {
+    sz = taosArrayGetSize(pConsumer->currentTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->currentTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+#if 0
+  // waiting reb topics
+  if (pConsumer->waitingRebTopics) {
+    sz = taosArrayGetSize(pConsumer->waitingRebTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->waitingRebTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+#endif
+
+  // reb new topics
+  if (pConsumer->rebNewTopics) {
+    sz = taosArrayGetSize(pConsumer->rebNewTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->rebNewTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+  // reb removed topics
+  if (pConsumer->rebRemovedTopics) {
+    sz = taosArrayGetSize(pConsumer->rebRemovedTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->rebRemovedTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+  return tlen;
+}
+
+void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
+  int32_t sz;
+  buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
+  buf = taosDecodeStringTo(buf, pConsumer->cgroup);
+  buf = taosDecodeFixedI8(buf, &pConsumer->updateType);
+  buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
+  buf = taosDecodeFixedI32(buf, &pConsumer->status);
+
+  // current topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->currentTopics, &topic);
+  }
+
+#if 0
+  // waiting reb topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->waitingRebTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->waitingRebTopics, &topic);
+  }
+#endif
+
+  // reb new topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->rebNewTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->rebNewTopics, &topic);
+  }
+
+  // reb removed topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->rebRemovedTopics, &topic);
+  }
+
+  return (void *)buf;
+}
+
+SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
+  SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
+  if (pVgEpNew == NULL) return NULL;
+  pVgEpNew->vgId = pVgEp->vgId;
+  pVgEpNew->qmsg = strdup(pVgEp->qmsg);
+  /*memcpy(pVgEpNew->topic, pVgEp->topic, TSDB_TOPIC_FNAME_LEN);*/
+  pVgEpNew->epSet = pVgEp->epSet;
+  return pVgEpNew;
+}
+
+void tDeleteSMqVgEp(SMqVgEp *pVgEp) { taosMemoryFree(pVgEp->qmsg); }
+
+int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
+  tlen += taosEncodeString(buf, pVgEp->qmsg);
+  /*tlen += taosEncodeString(buf, pVgEp->topic);*/
+  tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
+  return tlen;
+}
+
+void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
+  buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
+  buf = taosDecodeString(buf, &pVgEp->qmsg);
+  /*buf = taosDecodeStringTo(buf, pVgEp->topic);*/
+  buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
+  return (void *)buf;
+}
+
+SMqConsumerEpObj *tCloneSMqConsumerEpObj(const SMqConsumerEpObj *pConsumerEp) {
+  SMqConsumerEpObj *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEpObj));
+  if (pConsumerEpNew == NULL) return NULL;
+  pConsumerEpNew->consumerId = pConsumerEp->consumerId;
+  memcpy(pConsumerEpNew->cgroup, pConsumerEp->cgroup, TSDB_CGROUP_LEN);
+  taosInitRWLatch(&pConsumerEpNew->lock);
+  pConsumerEpNew->vgs = taosArrayDeepCopy(pConsumerEpNew->vgs, (FCopy)tCloneSMqVgEp);
+  return pConsumerEpNew;
+}
+
+void tDeleteSMqConsumerEpObj(SMqConsumerEpObj *pConsumerEp) {
+  taosArrayDestroyEx(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp);
+}
+
+int32_t tEncodeSMqConsumerEpObj(void **buf, const SMqConsumerEpObj *pConsumerEp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
+  tlen += taosEncodeString(buf, pConsumerEp->cgroup);
+  tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
+  return tlen;
+}
+
+void *tDecodeSMqConsumerEpObj(const void *buf, SMqConsumerEpObj *pConsumerEp) {
+  buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
+  buf = taosDecodeStringTo(buf, pConsumerEp->cgroup);
+  buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));
+  return (void *)buf;
+}
+
+SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) {
+  SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));
+  if (pEpInSubNew == NULL) return NULL;
+  pEpInSubNew->consumerId = pEpInSub->consumerId;
+  pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp);
+  return pEpInSubNew;
+}
+
+void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) {
+  taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp);
+}
+
+int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId);
+  tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);
+  return tlen;
+}
+
+void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) {
+  buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId);
+  buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));
+  return (void *)buf;
+}
+
+SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
+  SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj));
+  if (pSubNew == NULL) return NULL;
+  memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN);
+  taosInitRWLatch(&pSubNew->lock);
+  pSubNew->vgNum = -1;
+  pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  // TODO set free fp
+  SMqConsumerEpInSub *pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));
+  pEpInSub->vgs = taosArrayInit(0, sizeof(SMqVgEp));
+  int64_t unexistKey = -1;
+  taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), pEpInSub, sizeof(SMqConsumerEpInSub));
+  return pSubNew;
+}
+
+SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
+  SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj));
+  if (pSubNew == NULL) return NULL;
+  memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
+  taosInitRWLatch(&pSubNew->lock);
+  pSubNew->vgNum = pSub->vgNum;
+  /*pSubNew->consumerEps = taosArrayDeepCopy(pSub->consumerEps, (FCopy)tCloneSMqConsumerEpInSub);*/
+  pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  /*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/
+  void               *pIter = NULL;
+  SMqConsumerEpInSub *pEpInSub = NULL;
+  while (1) {
+    pIter = taosHashIterate(pSubNew->consumerHash, pIter);
+    if (pIter == NULL) break;
+    pEpInSub = (SMqConsumerEpInSub *)pIter;
+    SMqConsumerEpInSub newEp = {
+        .consumerId = pEpInSub->consumerId,
+        .vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp),
+    };
+    taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub));
+  }
+  return pSubNew;
+}
+
+void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
+  /*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/
+  taosHashCleanup(pSub->consumerHash);
+}
+
+int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pSub->key);
+  tlen += taosEncodeFixedI32(buf, pSub->vgNum);
+
+  void   *pIter = NULL;
+  int32_t sz = taosHashGetSize(pSub->consumerHash);
+  tlen += taosEncodeFixedI32(buf, sz);
+
+  int32_t cnt = 0;
+  while (1) {
+    pIter = taosHashIterate(pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub);
+    cnt++;
+  }
+  ASSERT(cnt == sz);
+  /*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/
+  return tlen;
+}
+
+void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
+  //
+  buf = taosDecodeStringTo(buf, pSub->key);
+  buf = taosDecodeFixedI32(buf, &pSub->vgNum);
+
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+
+  pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  for (int32_t i = 0; i < sz; i++) {
+    /*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/
+    SMqConsumerEpInSub epInSub = {0};
+    buf = tDecodeSMqConsumerEpInSub(buf, &epInSub);
+    taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
+  }
+
+  /*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/
+  return (void *)buf;
+}
+
+SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+  SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
+  if (pEntryNew == NULL) return NULL;
+  pEntryNew->epoch = pEntry->epoch;
+  pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub);
+  return pEntryNew;
+}
+
+void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub);
+}
+
+int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pEntry->epoch);
+  tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
+  return tlen;
+}
+void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
+  buf = taosDecodeFixedI32(buf, &pEntry->epoch);
+  buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+  return (void *)buf;
+}
+
+SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+  SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
+  if (pLogNew == NULL) return pLogNew;
+  memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
+  pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub);
+  return pLogNew;
+}
+
+void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub);
+}
+
+int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pLog->key);
+  tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry);
+  return tlen;
+}
+
+void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
+  buf = taosDecodeStringTo(buf, pLog->key);
+  buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+  return (void *)buf;
+}
 
 int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   int32_t sz = 0;
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index df7946a0c1..5629421a84 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -453,6 +453,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
   }
   SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
 
+  int64_t             unexistKey = -1;
+  SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
+
   void* pIter = NULL;
   while (1) {
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@@ -466,24 +469,33 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     plan->execNode.nodeId = pVgroup->vgId;
     plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
+    SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
+    pVgEp->epSet = plan->execNode.epSet;
+    pVgEp->vgId = plan->execNode.nodeId;
+
+#if 0
     SMqConsumerEp consumerEp = {0};
     consumerEp.status = 0;
     consumerEp.consumerId = -1;
     consumerEp.epSet = plan->execNode.epSet;
     consumerEp.vgId = plan->execNode.nodeId;
-    
-    mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId);
+#endif
+
+    mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId);
 
     int32_t msgLen;
-    if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
+    if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) {
       sdbRelease(pSdb, pVgroup);
       qDestroyQueryPlan(pPlan);
       terrno = TSDB_CODE_QRY_INVALID_INPUT;
       return -1;
     }
-    taosArrayPush(pSub->unassignedVg, &consumerEp);
+    taosArrayPush(pEpInSub->vgs, &pVgEp);
+
+    /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
   }
 
+  taosHashRelease(pSub->consumerHash, pEpInSub);
   qDestroyQueryPlan(pPlan);
 
   return 0;
diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index 3a31b0abb9..48c9804619 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -40,30 +40,47 @@ enum {
   MQ_SUBSCRIBE_STATUS__DELETED,
 };
 
-static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
-
 static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
 static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
 static int32_t  mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub);
 
-static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
-static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg);
+/*static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);*/
+/*static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg);*/
 static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg);
 static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg);
-static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
-static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg);
-static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg);
-static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);
+/*static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);*/
+/*static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg);*/
+/*static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg);*/
+/*static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);*/
 
-static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
-                                      const SMqConsumerEp *pConsumerEp);
+static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg);
 
-static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
-                                      const char *topicName);
-static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
-                                       const char *oldTopicName);
+static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
+  SSdbRaw *pRedoRaw = mndSubActionEncode(pSub);
+  if (pRedoRaw == NULL) return -1;
+  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
+
+static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
+  SSdbRaw *pCommitRaw = mndSubActionEncode(pSub);
+  if (pCommitRaw == NULL) return -1;
+  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
+  if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
+
+/*static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,*/
+/*const SMqConsumerEp *pConsumerEp);*/
+
+/*static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/
+/*const char *topicName);*/
+
+/*static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/
+/*const char *oldTopicName);*/
 
 int32_t mndInitSubscribe(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
@@ -74,16 +91,35 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndSubActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndSubActionDelete};
 
-  mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);
-  mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
-  mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);
-  mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);
+  /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);*/
+  /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);*/
+  /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);*/
+  /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);*/
+  /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);*/
+  /*mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);*/
+  /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);*/
+  mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq);
   return sdbSetTable(pMnode->pSdb, table);
 }
 
+static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) {
+  SMqSubscribeObj *pSub = tNewSubscribeObj(subKey);
+  if (pSub == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
+    tDeleteSubscribeObj(pSub);
+    taosMemoryFree(pSub);
+    return NULL;
+  }
+
+  return pSub;
+}
+
+#if 0
 static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) {
   SMqSubscribeObj *pSub = tNewSubscribeObj();
   if (pSub == NULL) {
@@ -212,7 +248,63 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
 
   return 0;
 }
+#endif
 
+static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const char *subKey, const SMqRebOutputVg *pRebVg) {
+  SMqRebVgReq req = {0};
+  req.oldConsumerId = pRebVg->oldConsumerId;
+  req.newConsumerId = pRebVg->newConsumerId;
+  req.vgId = pRebVg->pVgEp->vgId;
+  req.qmsg = req.qmsg;
+  strncpy(req.subKey, subKey, TSDB_SUBSCRIBE_KEY_LEN);
+
+  int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req);
+  void   *buf = taosMemoryMalloc(tlen);
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  SMsgHead *pMsgHead = (SMsgHead *)buf;
+
+  pMsgHead->contLen = htonl(tlen);
+  pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId);
+
+  void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  tEncodeSMqRebVgReq(&abuf, &req);
+  *pBuf = buf;
+  *pLen = tlen;
+
+  return 0;
+}
+
+static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const char *subKey,
+                                        const SMqRebOutputVg *pRebVg) {
+  ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId);
+
+  void   *buf;
+  int32_t tlen;
+  if (mndBuildSubChangeReq(&buf, &tlen, subKey, pRebVg) < 0) {
+    return -1;
+  }
+
+  int32_t vgId = pRebVg->pVgEp->vgId;
+  SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
+
+  STransAction action = {0};
+  action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
+  action.pCont = buf;
+  action.contLen = tlen;
+  action.msgType = TDMT_VND_MQ_VG_CHANGE;
+
+  mndReleaseVgroup(pMnode, pVgObj);
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(buf);
+    return -1;
+  }
+  return 0;
+}
+
+#if 0
 static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
   SMnode           *pMnode = pMsg->pNode;
   SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
@@ -312,6 +404,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
   pMsg->rspLen = tlen;
   return 0;
 }
+#endif
 
 static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) {
   int32_t i = 0;
@@ -337,6 +430,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
   return pRebSub;
 }
 
+#if 0
 static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   SMnode            *pMnode = pMsg->pNode;
   SSdb              *pSdb = pMnode->pSdb;
@@ -408,7 +502,9 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   }
   return 0;
 }
+#endif
 
+#if 0
 static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
   SMnode            *pMnode = pMsg->pNode;
   SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont;
@@ -422,7 +518,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
     if (pIter == NULL) break;
     SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter;
     SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
-    taosMemoryFreeClear(pRebSub->key);
 
     mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum,
           (int32_t)taosArrayGetSize(pSub->unassignedVg));
@@ -562,6 +657,346 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
   mndTransDrop(pTrans);
   return 0;
 }
+#endif
+
+static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
+  if (pInput->pTopic != NULL) {
+    // create subscribe
+    pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key);
+  } else {
+    pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub);
+  }
+  int32_t totalVgNum = pOutput->pSub->vgNum;
+
+  // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
+  SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+
+  // 2. check and get actual removed consumers, put their vg into hash
+  int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+  int32_t actualRemoved = 0;
+  for (int32_t i = 0; i < removedNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    if (pEpInSub) {
+      ASSERT(consumerId == pEpInSub->consumerId);
+      actualRemoved++;
+      int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+      for (int32_t j = 0; j < consumerVgNum; j++) {
+        SMqVgEp       *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        SMqRebOutputVg outputVg = {
+            .oldConsumerId = consumerId,
+            .newConsumerId = -1,
+            .pVgEp = pVgEp,
+        };
+        taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+      }
+      taosHashRelease(pOutput->pSub->consumerHash, pEpInSub);
+      taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+      // put into removed
+      taosArrayPush(pOutput->removedConsumers, &consumerId);
+    }
+  }
+  ASSERT(removedNum == actualRemoved);
+
+  // if previously no consumer, there are vgs not assigned
+  {
+    int64_t             key = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    for (int32_t i = 0; i < consumerVgNum; i++) {
+      SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+      SMqRebOutputVg rebOutput = {
+          .oldConsumerId = -1,
+          .newConsumerId = -1,
+          .pVgEp = pVgEp,
+      };
+      taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
+    }
+    taosHashRelease(pOutput->pSub->consumerHash, pEpInSub);
+  }
+
+  // 3. calc vg number of each consumer
+  int32_t actualConsumerNum = taosHashGetSize(pInput->pOldSub->consumerHash) - 1 +
+                              taosArrayGetSize(pInput->pRebInfo->newConsumers) -
+                              taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+  int32_t afterRebConsumerNum = taosHashGetSize(pOutput->pSub->consumerHash) - 1;
+  ASSERT(afterRebConsumerNum == actualConsumerNum);
+  // calc num
+  int32_t minVgCnt = totalVgNum / actualConsumerNum;
+  int32_t imbConsumerNum = totalVgNum % actualConsumerNum;
+
+  // 4. first scan: remove consumer more than wanted, put to remove hash
+  int32_t imbCnt = 0;
+  void   *pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    ASSERT(pEpInSub->consumerId > 0);
+    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    // all old consumers still existing are touched
+    // TODO optimize: touch only consumer whose vgs changed
+    taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId);
+    if (consumerVgNum > minVgCnt) {
+      if (imbCnt < imbConsumerNum) {
+        if (consumerVgNum == minVgCnt + 1) {
+          continue;
+        } else {
+          // pop until equal minVg + 1
+          while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) {
+            SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+            SMqRebOutputVg outputVg = {
+                .oldConsumerId = pEpInSub->consumerId,
+                .newConsumerId = -1,
+                .pVgEp = pVgEp,
+            };
+            taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+          }
+          imbCnt++;
+        }
+      } else {
+        // pop until equal minVg
+        while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) {
+          SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+          SMqRebOutputVg outputVg = {
+              .oldConsumerId = pEpInSub->consumerId,
+              .newConsumerId = -1,
+              .pVgEp = pVgEp,
+          };
+          taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+        }
+      }
+    }
+  }
+
+  // 5. add new consumer into sub
+  {
+    int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers);
+    for (int32_t i = 0; i < consumerNum; i++) {
+      int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
+      ASSERT(consumerId > 0);
+      SMqConsumerEpInSub newConsumerEp;
+      newConsumerEp.consumerId = consumerId;
+      newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
+      taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp,
+                  sizeof(SMqConsumerEpInSub));
+      taosArrayPush(pOutput->newConsumers, &consumerId);
+    }
+  }
+
+  // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer.
+  // All related vg should be put into rebVgs
+  SMqRebOutputVg *pRebVg = NULL;
+  void           *pRemovedIter = NULL;
+  pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    ASSERT(pEpInSub->consumerId > 0);
+    /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/
+    if (imbCnt < imbConsumerNum) {
+      imbCnt++;
+      // push until equal minVg + 1
+      while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt + 1) {
+        // iter hash and find one vg
+        pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+        ASSERT(pRemovedIter);
+        pRebVg = (SMqRebOutputVg *)pRemovedIter;
+        // push
+        taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+        pRebVg->newConsumerId = pEpInSub->consumerId;
+        taosArrayPush(pOutput->rebVgs, pRebVg);
+      }
+    } else {
+      // push until equal minVg
+      while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) {
+        // iter hash and find one vg
+        pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+        ASSERT(pRemovedIter);
+        pRebVg = (SMqRebOutputVg *)pRemovedIter;
+        // push
+        taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+        pRebVg->newConsumerId = pEpInSub->consumerId;
+        taosArrayPush(pOutput->rebVgs, pRebVg);
+      }
+    }
+  }
+
+  // 7. handle unassigned vg
+  if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) {
+    // if has consumer, vg should be all assigned
+    pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+    ASSERT(pRemovedIter == NULL);
+  } else {
+    // if all consumer is removed, put all vg into unassigned
+    int64_t             key = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    ASSERT(pEpInSub->consumerId == -1);
+
+    pIter = NULL;
+    SMqRebOutputVg *pRebOutput = NULL;
+    while (1) {
+      pIter = taosHashIterate(pHash, pIter);
+      if (pIter == NULL) break;
+      pRebOutput = (SMqRebOutputVg *)pIter;
+      ASSERT(pRebOutput->newConsumerId == -1);
+      taosArrayPush(pEpInSub->vgs, pRebOutput->pVgEp);
+    }
+  }
+
+  // 8. generate logs
+
+  // 9. clear
+  taosHashCleanup(pHash);
+
+  return 0;
+}
+
+static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebOutputObj *pOutput) {
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg);
+  if (pTrans == NULL) {
+    return -1;
+  }
+  // make txn:
+  // 1. redo action: action to all vg
+  const SArray *rebVgs = pOutput->rebVgs;
+  int32_t       vgNum = taosArrayGetSize(rebVgs);
+  for (int32_t i = 0; i < vgNum; i++) {
+    SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i);
+    if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub->key, pRebVg) < 0) {
+      goto REB_FAIL;
+    }
+  }
+
+  // 2. redo log: subscribe and vg assignment
+  // subscribe
+  if (mndSetSubRedoLogs(pMnode, pTrans, pOutput->pSub) != 0) {
+    goto REB_FAIL;
+  }
+
+  // 3. commit log: consumer to update status and epoch
+  // 3.1 set touched consumer
+  int32_t consumerNum = taosArrayGetSize(pOutput->touchedConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t         consumerId = *(int64_t *)taosArrayGet(pOutput->touchedConsumers, i);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH;
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+  // 3.2 set new consumer
+  consumerNum = taosArrayGetSize(pOutput->newConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__ADD;
+    char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
+    char  cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    taosArrayPush(pConsumerNew->rebNewTopics, &topic);
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+
+  // 3.3 set removed consumer
+  consumerNum = taosArrayGetSize(pOutput->removedConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE;
+    char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
+    char  cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    taosArrayPush(pConsumerNew->rebRemovedTopics, &topic);
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+  // 4. commit log: modification log
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+
+REB_FAIL:
+  mndTransDrop(pTrans);
+  return -1;
+}
+
+static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
+  SMnode            *pMnode = pMsg->pNode;
+  SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont;
+  void              *pIter = NULL;
+
+  mInfo("mq rebalance start");
+
+  while (1) {
+    pIter = taosHashIterate(pReq->rebSubHash, pIter);
+    if (pIter == NULL) break;
+    SMqRebInputObj   rebInput = {0};
+    SMqRebOutputObj  rebOutput = {0};
+    SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter;
+    SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
+
+    if (pSub == NULL) {
+      taosRLockLatch(&pSub->lock);
+      // split sub key and extract topic
+      char topic[TSDB_TOPIC_FNAME_LEN];
+      char cgroup[TSDB_CGROUP_LEN];
+      mndSplitSubscribeKey(pSub->key, topic, cgroup);
+      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+      ASSERT(pTopic);
+      taosRLockLatch(&pTopic->lock);
+      rebInput.pTopic = pTopic;
+    }
+
+    rebInput.pRebInfo = pRebSub;
+    rebInput.pOldSub = pSub;
+
+    int32_t             unassignedVgNum = 0;
+    int64_t             key = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &key, sizeof(int64_t));
+    if (pEpInSub != NULL) {
+      ASSERT(pEpInSub->consumerId == key);
+      unassignedVgNum = taosArrayGetSize(pEpInSub->vgs);
+    }
+    mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, unassignedVgNum);
+
+    // TODO replace assert with error check
+    ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
+    ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
+
+    if (rebInput.pTopic) {
+      SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic;
+      taosRUnLockLatch(&pTopic->lock);
+      mndReleaseTopic(pMnode, pTopic);
+      taosRUnLockLatch(&pSub->lock);
+      mndReleaseSubscribe(pMnode, pSub);
+    }
+  }
+
+  // reset flag
+  atomic_store_8(pReq->mqInReb, 0);
+  mInfo("mq rebalance completed successfully");
+  taosHashCleanup(pReq->rebSubHash);
+
+  return 0;
+}
 
 static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp) {
@@ -697,16 +1132,23 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) {
 
 static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) {
   mTrace("subscribe:%s, perform delete action", pSub->key);
-  tDeleteSMqSubscribeObj(pSub);
+  tDeleteSubscribeObj(pSub);
   return 0;
 }
 
 static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) {
   mTrace("subscribe:%s, perform update action", pOldSub->key);
+  taosWLockLatch(&pOldSub->lock);
+
+  SHashObj *tmp = pOldSub->consumerHash;
+  pOldSub->consumerHash = pNewSub->consumerHash;
+  pNewSub->consumerHash = tmp;
+
+  taosWUnLockLatch(&pOldSub->lock);
   return 0;
 }
 
-static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
+int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
   int32_t tlen = strlen(cgroup);
   memcpy(key, cgroup, tlen);
   key[tlen] = TMQ_SEPARATOR;
@@ -739,6 +1181,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) {
   sdbRelease(pSdb, pSub);
 }
 
+#if 0
 static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   SMnode         *pMnode = pMsg->pNode;
   char           *msgStr = pMsg->rpcMsg.pCont;
@@ -901,6 +1344,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer);
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
+#endif
 
 static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) {
   mndTransProcessRsp(pRsp);
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index a74b26a386..84f52cc2fc 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "tarray.h"
+#include "tcoding.h"
 
 SArray* taosArrayInit(size_t size, size_t elemSize) {
   assert(elemSize > 0);
@@ -312,7 +313,14 @@ void* taosArrayDestroy(SArray* pArray) {
   return NULL;
 }
 
-void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) {
+void taosArrayDestroyP(SArray* pArray, FDelete fp) {
+  for (int32_t i = 0; i < pArray->size; i++) {
+    fp(*(void**)TARRAY_GET_ELEM(pArray, i));
+  }
+  taosArrayDestroy(pArray);
+}
+
+void taosArrayDestroyEx(SArray* pArray, FDelete fp) {
   if (pArray == NULL) {
     return;
   }
@@ -421,6 +429,37 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void
   return;
 }
 
+SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) {
+  SArray* pArray = taosArrayInit(pSrc->size, pSrc->elemSize);
+  for (int32_t i = 0; i < pSrc->size; i++) {
+    void* clone = deepCopy(taosArrayGetP(pSrc, i));
+    taosArrayPush(pArray, &clone);
+  }
+  return pArray;
+}
+
+int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) {
+  int32_t tlen = 0;
+  int32_t sz = pArray->size;
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    void* data = taosArrayGetP(pArray, i);
+    tlen += encode(buf, data);
+  }
+  return tlen;
+}
+
+void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz) {
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  *pArray = taosArrayInit(sz, sizeof(void*));
+  for (int32_t i = 0; i < sz; i++) {
+    void* data = taosMemoryCalloc(1, dataSz);
+    buf = decode(buf, data);
+  }
+  return (void*)buf;
+}
+
 // order array<type *>
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) {
   taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param);

From 19bbd6806072955df859c4015496449a72293dd8 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 19 Apr 2022 17:11:53 +0800
Subject: [PATCH 11/22] remove commit

---
 tests/test/c/tmqSim.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index eaca8f151e..5ea15a6622 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -354,6 +354,7 @@ void loop_consume(tmq_t* tmq) {
       if (0 != g_stConfInfo.showMsgFlag) {
         /*msg_process(tmqMsg);*/
       }
+      /*tmq_commit(tmq, NULL, false);*/
       tmq_message_destroy(tmqMsg);
     } else {
       break;

From 412e2cbb0d3631017c81783fdfb8d18ca67f304f Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 19 Apr 2022 17:52:04 +0800
Subject: [PATCH 12/22] refactor: add test casese for json compare logic

---
 include/util/tdef.h                           |   5 -
 source/libs/parser/src/parUtil.c              |   4 +-
 source/libs/scalar/src/scalar.c               |  88 +++++++++------
 source/libs/scalar/src/sclvector.c            |  23 +++-
 .../libs/scalar/test/scalar/scalarTests.cpp   | 103 +++++++++++++++++-
 5 files changed, 177 insertions(+), 46 deletions(-)

diff --git a/include/util/tdef.h b/include/util/tdef.h
index eb7a51cd7e..4c3126caa9 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -272,11 +272,6 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
 #define TSDB_MAX_JSON_TAG_LEN   16384
-#define TSDB_JSON_PLACEHOLDER      0x7F
-#define TSDB_JSON_null             0x00
-#define TSDB_JSON_NOT_NULL         0x01
-#define TSDB_JSON_NULL             0x00
-
 
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index f3eb0c5ebc..0ff5843736 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -276,7 +276,7 @@ static bool isValidateTag(char *input) {
 
 int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
   // set json NULL data
-  uint8_t jsonNULL = TSDB_JSON_NULL;
+  uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
   int jsonIndex = startColId + 1;
   if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
     tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
@@ -328,7 +328,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
     strncpy(varDataVal(tagKV), jsonKey, keyLen);
     varDataSetLen(tagKV, keyLen);
     if(taosHashGetSize(keyHash) == 0){
-      uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
+      uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON;
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);   // add json type
     }
     taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index 9c4d2d94dd..2fd688a233 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -604,39 +604,55 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
   return DEAL_RES_CONTINUE;
 }
 
-//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
-//  STargetNode *target = (STargetNode *)pNode;
-//
-//  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
-//    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
-//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-//    return DEAL_RES_ERROR;
-//  }
-//
-//  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
-//  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
-//    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
-//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-//    return DEAL_RES_ERROR;
-//  }
-//
-//  // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
-//  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
-//
-//  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-//  if (NULL == res) {
-//    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
-//    ctx->code = TSDB_CODE_QRY_APP_ERROR;
-//    return DEAL_RES_ERROR;
-//  }
-//
-//  colDataAssign(col, res->columnData, res->numOfRows);
-//  block->info.rows = res->numOfRows;
-//
-//  sclFreeParam(res);
-//  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-//  return DEAL_RES_CONTINUE;
-//}
+EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
+  STargetNode *target = (STargetNode *)pNode;
+
+  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
+    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
+    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+    return DEAL_RES_ERROR;
+  }
+
+  int32_t index = -1;
+  for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
+    SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i);
+    if (pb->info.blockId == target->dataBlockId) {
+      index = i;
+      break;
+    }
+  }
+
+  if (index == -1) {
+    sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
+    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+    return DEAL_RES_ERROR;
+  }
+
+  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
+
+  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
+    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
+    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+    return DEAL_RES_ERROR;
+  }
+
+  // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
+  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
+
+  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+  if (NULL == res) {
+    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
+    ctx->code = TSDB_CODE_QRY_APP_ERROR;
+    return DEAL_RES_ERROR;
+  }
+
+  colDataAssign(col, res->columnData, res->numOfRows);
+  block->info.rows = res->numOfRows;
+
+  sclFreeParam(res);
+  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+  return DEAL_RES_CONTINUE;
+}
 
 EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
   if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
@@ -656,9 +672,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
     return sclWalkOperator(pNode, ctx);
   }
 
-//  if (QUERY_NODE_TARGET == nodeType(pNode)) {
-//    return sclWalkTarget(pNode, ctx);
-//  }
+  if (QUERY_NODE_TARGET == nodeType(pNode)) {
+    return sclWalkTarget(pNode, ctx);
+  }
 
   sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
   ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index e97831b1f6..7ad200befc 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -767,6 +767,17 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
 }
 
 char *getJsonValue(char *json, char *key){    //todo
+  int16_t cols = kvRowNCols(json);
+  for (int i = 0; i < cols; ++i) {
+    SColIdx *pColIdx = kvRowColIdxAt(json, i);
+    char *data = kvRowColVal(json, pColIdx);
+    if(i == 0 && *data == TSDB_DATA_TYPE_NULL){
+      return NULL;
+    }
+    if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){
+      return data + varDataTLen(data);
+    }
+  }
   return NULL;
 }
 
@@ -844,7 +855,11 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i);
+        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+          colDataAppendNULL(pOutputCol, i);
+          continue;  // TODO set null or ignore
+        }
+        *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
 
       pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -957,7 +972,11 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i);
+        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+          colDataAppendNULL(pOutputCol, i);
+          continue;  // TODO set null or ignore
+        }
+        *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
 
       pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp
index 61ef2fdce2..975effc63a 100644
--- a/source/libs/scalar/test/scalar/scalarTests.cpp
+++ b/source/libs/scalar/test/scalar/scalarTests.cpp
@@ -92,6 +92,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s
     blockDataEnsureCapacity(res, rows);
 
     *dataBlockId = taosArrayGetSize(pBlockList) - 1;
+    res->info.blockId = *dataBlockId;
     *slotId = 0;
   } else {
     SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
@@ -124,6 +125,56 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
   
   *pNode = (SNode *)vnode;
 }
+void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
+  SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+  SColumnNode *rnode = (SColumnNode *)node;
+  rnode->node.resType.type = dataType;
+  rnode->node.resType.bytes = dataBytes;
+  rnode->dataBlockId = 0;
+
+  if (NULL == *block) {
+    SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
+    res->info.numOfCols = 3;
+    res->info.rows = rowNum;
+    res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
+    for (int32_t i = 0; i < 2; ++i) {
+      SColumnInfoData idata = {{0}};
+      idata.info.type  = TSDB_DATA_TYPE_NULL;
+      idata.info.bytes = 10;
+      idata.info.colId = i + 1;
+
+      int32_t size = idata.info.bytes * rowNum;
+      idata.pData = (char *)taosMemoryCalloc(1, size);
+      taosArrayPush(res->pDataBlock, &idata);
+    }
+
+    SColumnInfoData idata = {{0}};
+    idata.info.type  = dataType;
+    idata.info.bytes = dataBytes;
+    idata.info.colId = 3;
+    int32_t size = idata.info.bytes * rowNum;
+    idata.pData = (char *)taosMemoryCalloc(1, size);
+    taosArrayPush(res->pDataBlock, &idata);
+
+    blockDataEnsureCapacity(res, rowNum);
+
+    SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
+    for (int32_t i = 0; i < rowNum; ++i) {
+      uint32_t len = pColumnInfoData->varmeta.length;
+      pColumnInfoData->varmeta.offset[i] = len;
+
+      memcpy(pColumnInfoData->pData + len, value, varDataLen(value));
+      pColumnInfoData->varmeta.length += varDataLen(value);
+    }
+
+    rnode->slotId = 2;
+    rnode->colId = 3;
+
+    *block = res;
+  }
+
+  *pNode = (SNode *)rnode;
+}
 
 void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
   SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
@@ -909,6 +960,56 @@ TEST(constantTest, greater_and_lower) {
   nodesDestroyNode(res);
 }
 
+TEST(columnTest, smallint_value_add_json_column) {
+  scltInitLogFile();
+
+  SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
+  int32_t leftv = 1;
+  char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}";
+  char key[10] = {0};
+  memcpy(varDataVal(key), "k1", 2);
+  varDataLen(key) = 2;
+
+  SKVRowBuilder kvRowBuilder;
+  tdInitKVRowBuilder(&kvRowBuilder);
+  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+
+  double eRes[5] = {1.0, -4, -3, 24, 101};
+
+  SSDataBlock *src = NULL;
+  int32_t rowNum = 1;
+
+  scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key);
+  scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row);
+  scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
+
+  SNode *right = NULL;
+  scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv);
+  scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right);
+
+  SArray *blockList = taosArrayInit(2, POINTER_BYTES);
+  taosArrayPush(blockList, &src);
+  SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
+  int16_t dataBlockId = 0, slotId = 0;
+  scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
+  scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
+
+  int32_t code = scalarCalculate(opNode, blockList, NULL);
+  ASSERT_EQ(code, 0);
+
+  SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
+  ASSERT_EQ(res->info.rows, rowNum);
+  SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
+  ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
+  for (int32_t i = 0; i < rowNum; ++i) {
+    ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]);
+  }
+
+  taosArrayDestroyEx(blockList, scltFreeDataBlock);
+  nodesDestroyNode(opNode);
+}
+
 TEST(columnTest, smallint_value_add_int_column) {
   scltInitLogFile();
   
@@ -928,7 +1029,7 @@ TEST(columnTest, smallint_value_add_int_column) {
   int16_t dataBlockId = 0, slotId = 0;
   scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
   scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
-  
+
   int32_t code = scalarCalculate(opNode, blockList, NULL);
   ASSERT_EQ(code, 0);
 

From 2133a2de1d5ab0c761bf9fb9b85c0f5c139beba8 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 19 Apr 2022 20:30:06 +0800
Subject: [PATCH 13/22] refactor: add test casese for json compare logic

---
 include/common/tdataformat.h                  |   2 +-
 source/common/src/tdatablock.c                |  21 ++-
 source/libs/scalar/src/sclvector.c            |   4 +-
 source/libs/scalar/test/scalar/CMakeLists.txt |   3 +-
 .../libs/scalar/test/scalar/scalarTests.cpp   | 143 +++++++-----------
 5 files changed, 80 insertions(+), 93 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 2dd887ae74..a0f4351bbf 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -502,7 +502,7 @@ typedef struct {
 
 #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
-#define kvRowLen(r)            (*(TDRowLenT *)(r))
+#define kvRowLen(r)            (*(uint16_t *)(r))
 #define kvRowNCols(r)          (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
 #define kvRowSetLen(r, len)    kvRowLen(r) = (len)
 #define kvRowSetNCols(r, n)    kvRowNCols(r) = (n)
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 14b7fe5d0e..5bdf7ddb85 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
+    int32_t dataLen = varDataTLen(pData);
+    if(type == TSDB_DATA_TYPE_JSON) {
+      if(*pData == TSDB_DATA_TYPE_NULL) {
+        dataLen = 0;
+      }else if(*pData == TSDB_DATA_TYPE_NCHAR) {
+        dataLen = varDataTLen(pData+CHAR_BYTES);
+      }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+        dataLen = LONG_BYTES;
+      }else if(*pData == TSDB_DATA_TYPE_BOOL) {
+        dataLen = CHAR_BYTES;
+      }
+      dataLen += CHAR_BYTES;
+    }
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;
-    if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) {
+    if (pAttr->allocLen < pAttr->length + dataLen) {
       uint32_t newSize = pAttr->allocLen;
       if (newSize == 0) {
         newSize = 8;
       }
 
-      while (newSize < pAttr->length + varDataTLen(pData)) {
+      while (newSize < pAttr->length + dataLen) {
         newSize = newSize * 1.5;
       }
 
@@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
     uint32_t len = pColumnInfoData->varmeta.length;
     pColumnInfoData->varmeta.offset[currentRow] = len;
 
-    memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData));
-    pColumnInfoData->varmeta.length += varDataTLen(pData);
+    memcpy(pColumnInfoData->pData + len, pData, dataLen);
+    pColumnInfoData->varmeta.length += dataLen;
   } else {
     memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
   }
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 7ad200befc..c2eafedf9b 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -774,7 +774,7 @@ char *getJsonValue(char *json, char *key){    //todo
     if(i == 0 && *data == TSDB_DATA_TYPE_NULL){
       return NULL;
     }
-    if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){
+    if(memcmp(key, data, varDataTLen(data)) == 0){
       return data + varDataTLen(data);
     }
   }
@@ -803,7 +803,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
       pOutputCol->hasNull = true;
       continue;
     }
-    colDataAppend(pOutputCol, i, pLeftData, false);
+    colDataAppend(pOutputCol, i, value, false);
   }
 }
 
diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt
index 6cbac1e25d..03418d17ac 100644
--- a/source/libs/scalar/test/scalar/CMakeLists.txt
+++ b/source/libs/scalar/test/scalar/CMakeLists.txt
@@ -8,11 +8,12 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         scalarTest
-        PUBLIC os util common gtest qcom function nodes scalar
+        PUBLIC os util common gtest qcom function nodes scalar parser
 )
 
 TARGET_INCLUDE_DIRECTORIES(
         scalarTest
         PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
+        PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
         PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
 )
diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp
index 975effc63a..8049e80590 100644
--- a/source/libs/scalar/test/scalar/scalarTests.cpp
+++ b/source/libs/scalar/test/scalar/scalarTests.cpp
@@ -125,56 +125,6 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
   
   *pNode = (SNode *)vnode;
 }
-void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
-  SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
-  SColumnNode *rnode = (SColumnNode *)node;
-  rnode->node.resType.type = dataType;
-  rnode->node.resType.bytes = dataBytes;
-  rnode->dataBlockId = 0;
-
-  if (NULL == *block) {
-    SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
-    res->info.numOfCols = 3;
-    res->info.rows = rowNum;
-    res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
-    for (int32_t i = 0; i < 2; ++i) {
-      SColumnInfoData idata = {{0}};
-      idata.info.type  = TSDB_DATA_TYPE_NULL;
-      idata.info.bytes = 10;
-      idata.info.colId = i + 1;
-
-      int32_t size = idata.info.bytes * rowNum;
-      idata.pData = (char *)taosMemoryCalloc(1, size);
-      taosArrayPush(res->pDataBlock, &idata);
-    }
-
-    SColumnInfoData idata = {{0}};
-    idata.info.type  = dataType;
-    idata.info.bytes = dataBytes;
-    idata.info.colId = 3;
-    int32_t size = idata.info.bytes * rowNum;
-    idata.pData = (char *)taosMemoryCalloc(1, size);
-    taosArrayPush(res->pDataBlock, &idata);
-
-    blockDataEnsureCapacity(res, rowNum);
-
-    SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
-    for (int32_t i = 0; i < rowNum; ++i) {
-      uint32_t len = pColumnInfoData->varmeta.length;
-      pColumnInfoData->varmeta.offset[i] = len;
-
-      memcpy(pColumnInfoData->pData + len, value, varDataLen(value));
-      pColumnInfoData->varmeta.length += varDataLen(value);
-    }
-
-    rnode->slotId = 2;
-    rnode->colId = 3;
-
-    *block = res;
-  }
-
-  *pNode = (SNode *)rnode;
-}
 
 void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
   SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
@@ -960,54 +910,77 @@ TEST(constantTest, greater_and_lower) {
   nodesDestroyNode(res);
 }
 
-TEST(columnTest, smallint_value_add_json_column) {
-  scltInitLogFile();
+void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
+  char key[32] = {0};
+  memcpy(varDataVal(key), key, strlen(key));
+  varDataLen(key) = strlen(key);
 
   SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
-  int32_t leftv = 1;
-  char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}";
-  char key[10] = {0};
-  memcpy(varDataVal(key), "k1", 2);
-  varDataLen(key) = 2;
+  scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key);
+  scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json);
+  scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
+}
+
+void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorType opType, int32_t rightType, void *rightData){
+  SNode *right = NULL;
+  scltMakeValueNode(&right, rightType, rightData);
+  scltMakeOpNode(opNode, opType, resType, *opNode, right);
+
+  SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes);
+  int16_t dataBlockId = 0, slotId = 0;
+  scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, 1, &colInfo);
+  scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
+}
+
+void makeCalculate(void *json, void *key, void *value, double exceptValue, EOperatorType opType){
+  SArray *blockList = taosArrayInit(2, POINTER_BYTES);
+  SSDataBlock *src = NULL;
+  SNode *opNode = NULL;
+
+  makeJsonArrow(&src, &opNode, row, key);
+  taosArrayPush(blockList, &src);
+
+  makeOperator(&opNode, blockList, TSDB_DATA_TYPE_DOUBLE, opType, TSDB_DATA_TYPE_INT, value);
+
+  int32_t code = scalarCalculate(opNode, blockList, NULL);
+  ASSERT_EQ(code, 0);
+
+  SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
+  ASSERT_EQ(res->info.rows, 1);
+  SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
+  ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
+  ASSERT_EQ(*((double *)colDataGetData(column, 0)), exceptValue);
+
+  taosArrayDestroyEx(blockList, scltFreeDataBlock);
+  nodesDestroyNode(opNode);
+}
+
+TEST(columnTest, int_value_add_json_column) {
+  scltInitLogFile();
+  char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
+  char *key = "k1";
 
   SKVRowBuilder kvRowBuilder;
   tdInitKVRowBuilder(&kvRowBuilder);
   parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
   SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
 
-  double eRes[5] = {1.0, -4, -3, 24, 101};
+  int32_t leftv = 1;
+  makeCalculate(row, key, &leftv, 5.0, OP_TYPE_ADD);
 
-  SSDataBlock *src = NULL;
-  int32_t rowNum = 1;
+  leftv = 8;
+  makeCalculate(row, key, &leftv, -4, OP_TYPE_SUB);
 
-  scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key);
-  scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row);
-  scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
+  leftv = 2;
+  makeCalculate(row, key, &leftv, 8.0, OP_TYPE_MULTI);
 
-  SNode *right = NULL;
-  scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv);
-  scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right);
+  leftv = 2;
+  makeCalculate(row, key, &leftv, 2.0, OP_TYPE_DIV);
 
-  SArray *blockList = taosArrayInit(2, POINTER_BYTES);
-  taosArrayPush(blockList, &src);
-  SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
-  int16_t dataBlockId = 0, slotId = 0;
-  scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
-  scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
+  leftv = 3;
+  makeCalculate(row, key, &leftv, 1.0, OP_TYPE_MODE);
 
-  int32_t code = scalarCalculate(opNode, blockList, NULL);
-  ASSERT_EQ(code, 0);
-
-  SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
-  ASSERT_EQ(res->info.rows, rowNum);
-  SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
-  ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
-  for (int32_t i = 0; i < rowNum; ++i) {
-    ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]);
-  }
-
-  taosArrayDestroyEx(blockList, scltFreeDataBlock);
-  nodesDestroyNode(opNode);
+  makeCalculate(row, key, &leftv, -4, OP_TYPE_MINUS);
 }
 
 TEST(columnTest, smallint_value_add_int_column) {

From a016f1157a0abaa2001a7706bdb5ae3d8db9572f Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 20 Apr 2022 16:52:32 +0800
Subject: [PATCH 14/22] refactor: add unit test for json compare

---
 include/common/tdatablock.h                   |  32 +-
 source/libs/scalar/src/sclvector.c            | 335 +++++++-----------
 .../libs/scalar/test/scalar/scalarTests.cpp   | 235 ++++++++++--
 3 files changed, 358 insertions(+), 244 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index e40efd815d..1d6f496eea 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -57,11 +57,30 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
 #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
 #define colDataSetNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] = -1)
 
+#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
+
+#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
+
+#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
+// SColumnInfoData, rowNumber
+#define colDataGetData(p1_, r_)                                        \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
+                                        :  colDataGetNumData(p1_, r_))
+
 static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
+  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){
+    if(colDataIsNull_var(pColumnInfoData, row)){
+      return true;
+    }
+    char *data = colDataGetVarData(pColumnInfoData, row);
+    return (*data == TSDB_DATA_TYPE_NULL);
+  } 
+  
   if (!pColumnInfoData->hasNull) {
     return false;
   }
-  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+
+  if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
     return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
@@ -99,16 +118,6 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 }
 
-#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
-
-#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
-
-#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
-// SColumnInfoData, rowNumber
-#define colDataGetData(p1_, r_)                                        \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
-                                        :  colDataGetNumData(p1_, r_))
-
 static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
   // There is a placehold for each NULL value of binary or nchar type.
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@@ -271,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
 #endif
 
 #endif /*_TD_COMMON_EP_H_*/
+
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index c2eafedf9b..ae8c395997 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -28,6 +28,75 @@
 #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
 #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
 
+void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
+  switch (outType) {
+    case TSDB_DATA_TYPE_BOOL: {
+      GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_TINYINT: {
+      GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UTINYINT: {
+      GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT: {
+      GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT: {
+      GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UBIGINT: {
+      GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      GET_TYPED_DATA(*((float *)outData), float, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      GET_TYPED_DATA(*((double *)outData), double, inType, inData);
+      break;
+    }
+    default:{
+      ASSERT(0);
+    }
+  }
+}
+
+void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
+  char *tmp = taosMemoryMalloc(varDataTLen(inData));
+  int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
+  if (len < 0) {
+    sclError("castConvert taosUcs4ToMbs error 1");
+  }
+
+  tmp[len] = 0;
+
+  ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
+  double value = strtod(tmp, NULL);
+
+  *((double *)outData) = value;
+  taosMemoryFreeClear(tmp);
+}
+
 typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
 
 int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
@@ -64,6 +133,20 @@ int64_t getVectorBigintValue_BOOL(void *src, int32_t index) {
   return (int64_t)*((bool *)src + index);
 }
 
+int64_t getVectorBigintValue_JSON(void *src, int32_t index){
+  ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index));
+  char *data = colDataGetVarData((SColumnInfoData*)src, index);
+  double out = 0;
+  if (*data == TSDB_DATA_TYPE_NULL){
+    return 0;
+  } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
+    convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+  } else {
+    convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+  }
+  return (int64_t)out;
+}
+
 _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
     _getBigintValue_fn_t p = NULL;
     if(srcType==TSDB_DATA_TYPE_TINYINT) {
@@ -90,6 +173,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
         p = getVectorBigintValue_BIGINT;
     }else if(srcType==TSDB_DATA_TYPE_BOOL) {
         p = getVectorBigintValue_BOOL;
+    }else if(srcType==TSDB_DATA_TYPE_JSON) {
+        p = getVectorBigintValue_JSON;
     }else {
         assert(0);
     }
@@ -200,75 +285,6 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn
   taosMemoryFree(t);
 }
 
-void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
-  switch (outType) {
-    case TSDB_DATA_TYPE_BOOL: {
-      GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_TINYINT: {
-      GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_SMALLINT: {
-      GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_INT: {
-      GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_TIMESTAMP: {
-      GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_UTINYINT: {
-      GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_USMALLINT: {
-      GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_UINT: {
-      GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_UBIGINT: {
-      GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_FLOAT: {
-      GET_TYPED_DATA(*((float *)outData), float, inType, inData);
-      break;
-    }
-    case TSDB_DATA_TYPE_DOUBLE: {
-      GET_TYPED_DATA(*((double *)outData), double, inType, inData);
-      break;
-    }
-    default:{
-      ASSERT(0);
-    }
-  }
-}
-
-void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
-  char *tmp = taosMemoryMalloc(varDataTLen(inData));
-  int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
-  if (len < 0) {
-    sclError("castConvert taosUcs4ToMbs error 1");
-  }
-
-  tmp[len] = 0;
-
-  ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
-  double value = strtod(tmp, NULL);
-
-  *((double *)outData) = value;
-  taosMemoryFreeClear(tmp);
-}
-
 //TODO opt performance, tmp is not needed.
 int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
   ASSERT(!IS_VAR_DATA_TYPE(outType));
@@ -345,12 +361,13 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
 }
 
 double getVectorDoubleValue_JSON(void *src, int32_t index){
-  ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index));
   char *data = colDataGetVarData((SColumnInfoData*)src, index);
   double out = 0;
-  if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
+  if (*data == TSDB_DATA_TYPE_NULL){
+    return out;
+  } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
     convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
-  } {
+  } else {
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   }
   return out;
@@ -569,24 +586,24 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
 }
 
 int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = {
-/*         NULL BOOL TINY SMAL INT  BIG  FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG VARB JSON DECI BLOB */
+/*         NULL BOOL TINY SMAL INT  BIG  FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */
 /*NULL*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-/*BOOL*/   0,   0,   0,   3,   4,   5,   6,   7,   7,   9,   7,   0,   12,  13,  14,  7,   0,   0,   0,
-/*TINY*/   0,   0,   0,   3,   4,   5,   6,   7,   7,   9,   7,   3,   4,   5,   7,   7,   0,   0,   0,
-/*SMAL*/   0,   0,   0,   0,   4,   5,   6,   7,   7,   9,   7,   3,   4,   5,   7,   7,   0,   0,   0,
-/*INT */   0,   0,   0,   0,   0,   5,   6,   7,   7,   9,   7,   4,   4,   5,   7,   7,   0,   0,   0,
-/*BIGI*/   0,   0,   0,   0,   0,   0,   6,   7,   7,   0,   7,   5,   5,   5,   7,   7,   0,   0,   0,
-/*FLOA*/   0,   0,   0,   0,   0,   0,   0,   7,   7,   6,   7,   6,   6,   6,   6,   7,   0,   0,   0,
-/*DOUB*/   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   7,   7,   7,   7,   0,   0,   0,
+/*BOOL*/   0,   0,   0,   3,   4,   5,   6,   7,   7,   9,   7,   0,   12,  13,  14,  0,   7,   0,   0,
+/*TINY*/   0,   0,   0,   3,   4,   5,   6,   7,   7,   9,   7,   3,   4,   5,   7,   0,   7,   0,   0,
+/*SMAL*/   0,   0,   0,   0,   4,   5,   6,   7,   7,   9,   7,   3,   4,   5,   7,   0,   7,   0,   0,
+/*INT */   0,   0,   0,   0,   0,   5,   6,   7,   7,   9,   7,   4,   4,   5,   7,   0,   7,   0,   0,
+/*BIGI*/   0,   0,   0,   0,   0,   0,   6,   7,   7,   0,   7,   5,   5,   5,   7,   0,   7,   0,   0,
+/*FLOA*/   0,   0,   0,   0,   0,   0,   0,   7,   7,   6,   7,   6,   6,   6,   6,   0,   7,   0,   0,
+/*DOUB*/   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   7,   7,   7,   0,   7,   0,   0,
 /*VARC*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   0,   7,   7,   7,   7,   0,   0,   0,   0,
-/*TIME*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   9,   9,   9,   7,   7,   0,   0,   0,
+/*TIME*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   9,   9,   9,   7,   0,   7,   0,   0,
 /*NCHA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   0,   0,   0,   0,
-/*UTIN*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   12,  13,  14,  7,   0,   0,   0,
-/*USMA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   13,  14,  7,   0,   0,   0,
-/*UINT*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   14,  7,   0,   0,   0,
-/*UBIG*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   0,   0,   0,
-/*VARB*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+/*UTIN*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   12,  13,  14,  0,   7,   0,   0,
+/*USMA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   13,  14,  0,   7,   0,   0,
+/*UINT*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   14,  0,   7,   0,   0,
+/*UBIG*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   0,   0,
 /*JSON*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+/*VARB*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
 /*DECI*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
 /*BLOB*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0
 };
@@ -677,7 +694,7 @@ enum {
 static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) {
   SColumnInfoData* pCol = pParam->columnData;
 
-  if (IS_VAR_DATA_TYPE(pCol->info.type) || pCol->info.type != TSDB_DATA_TYPE_JSON) {
+  if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) {
     pDest->numOfRows = pParam->numOfRows;
 
     SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
@@ -718,10 +735,6 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
       *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
                 + getVectorDoubleValueFnRight(RIGHT_COL, 0);
     }
-    pOutputCol->hasNull = pLeftCol->hasNull;
-    if (pOutputCol->hasNull) {
-      memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
-    }
   }
 }
 
@@ -735,12 +748,12 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
+      if (colDataIsNull_s(pLeftCol, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
       *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0);
     }
-    pOutputCol->hasNull = pLeftCol->hasNull;
-    if (pOutputCol->hasNull) {
-      memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
-    }
   }
 }
 
@@ -771,8 +784,11 @@ char *getJsonValue(char *json, char *key){    //todo
   for (int i = 0; i < cols; ++i) {
     SColIdx *pColIdx = kvRowColIdxAt(json, i);
     char *data = kvRowColVal(json, pColIdx);
-    if(i == 0 && *data == TSDB_DATA_TYPE_NULL){
-      return NULL;
+    if(i == 0){
+      if(*data == TSDB_DATA_TYPE_NULL) {
+        return NULL;
+      }
+      continue;
     }
     if(memcmp(key, data, varDataTLen(data)) == 0){
       return data + varDataTLen(data);
@@ -798,7 +814,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
     }
     char *pLeftData = colDataGetVarData(pLeft->columnData, i);
     char *value = getJsonValue(pLeftData, pRightData);
-    if (!value || *value == TSDB_DATA_TYPE_NULL) {
+    if (!value) {
       colDataSetNull_var(pOutputCol, i);
       pOutputCol->hasNull = true;
       continue;
@@ -835,14 +851,6 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
         }
         *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i);
       }
-      pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-      if (pOutputCol->hasNull) {
-        int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-        for (int32_t j = 0; j < numOfBitLen; ++j) {
-          pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
-        }
-      }
-
     } else if (pLeft->numOfRows == 1) {
       vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
     } else if (pRight->numOfRows == 1) {
@@ -861,15 +869,6 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
         }
         *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
-
-      pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-      if (pOutputCol->hasNull) {
-        int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-        for (int32_t j = 0; j < numOfBitLen; ++j) {
-          pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
-        }
-      }
-
     } else if (pLeft->numOfRows == 1) {
       vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
     } else if (pRight->numOfRows == 1) {
@@ -899,10 +898,6 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
       *output = (getVectorDoubleValueFnLeft(LEFT_COL, i)
                  - getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor;
     }
-    pOutputCol->hasNull = pLeftCol->hasNull;
-    if (pOutputCol->hasNull) {
-      memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
-    }
   }
 }
 
@@ -916,12 +911,12 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
+      if (colDataIsNull_s(pLeftCol, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
       *output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor;
     }
-    pOutputCol->hasNull = pLeftCol->hasNull;
-    if (pOutputCol->hasNull) {
-      memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
-    }
   }
 }
 
@@ -951,15 +946,6 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
         }
         *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i);
       }
-
-      pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-      if (pOutputCol->hasNull) {
-        int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-        for (int32_t j = 0; j < numOfBitLen; ++j) {
-          pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
-        }
-      }
-
     } else if (pLeft->numOfRows == 1) {
       vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
     } else if (pRight->numOfRows == 1) {
@@ -978,15 +964,6 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
         }
         *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
-
-      pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-      if (pOutputCol->hasNull) {
-        int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-        for (int32_t j = 0; j < numOfBitLen; ++j) {
-          pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
-        }
-      }
-
     } else if (pLeft->numOfRows == 1) {
       vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i);
     } else if (pRight->numOfRows == 1) {
@@ -1016,10 +993,6 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
       *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
                 * getVectorDoubleValueFnRight(RIGHT_COL, 0);
     }
-    pOutputCol->hasNull = pLeftCol->hasNull;
-    if (pOutputCol->hasNull) {
-      memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
-    }
   }
 }
 
@@ -1047,15 +1020,6 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
       *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
                 * getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
-
-    pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-    if (pOutputCol->hasNull) {
-      int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-      for (int32_t j = 0; j < numOfBitLen; ++j) {
-        pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
-      }
-    }
-
   } else if (pLeft->numOfRows == 1) {
     vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
   } else if (pRight->numOfRows == 1) {
@@ -1090,15 +1054,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
       *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
                  /getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
-
-    pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-    if (pOutputCol->hasNull) {
-      int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-      for (int32_t j = 0; j < numOfBitLen; ++j) {
-        pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
-      }
-    }
-
   } else if (pLeft->numOfRows == 1) {
     if (colDataIsNull_s(pLeftCol, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
@@ -1111,10 +1066,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
         *output = getVectorDoubleValueFnLeft(LEFT_COL, 0)
                   / getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
-      pOutputCol->hasNull = pRightCol->hasNull;
-      if (pOutputCol->hasNull) {
-        memcpy(pOutputCol->nullbitmap, pRightCol->nullbitmap, BitmapLen(pRight->numOfRows));
-      }
     }
   } else if (pRight->numOfRows == 1) {
     if (colDataIsNull_s(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
@@ -1128,10 +1079,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
         *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
                   / getVectorDoubleValueFnRight(RIGHT_COL, 0);
       }
-      pOutputCol->hasNull = pLeftCol->hasNull;
-      if (pOutputCol->hasNull) {
-        memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows));
-      }
     }
   }
 
@@ -1240,11 +1187,6 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
     *output = - getVectorDoubleValueFnLeft(LEFT_COL, i);
   }
 
-  pOutputCol->hasNull = pLeftCol->hasNull;
-  if (pOutputCol->hasNull) {
-    memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows));
-  }
-
   doReleaseVec(pLeftCol,  leftConvert);
 }
 
@@ -1315,10 +1257,6 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh
       }
       *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0);
     }
-    pOutputCol->hasNull = pLeftCol->hasNull;
-    if (pOutputCol->hasNull) {
-      memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
-    }
   }
 }
 
@@ -1345,15 +1283,6 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
       }
       *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i);
     }
-
-    pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-    if (pOutputCol->hasNull) {
-      int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-      for (int32_t j = 0; j < numOfBitLen; ++j) {
-        pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] & pRightCol->nullbitmap[j];
-      }
-    }
-
   } else if (pLeft->numOfRows == 1) {
     vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
   } else if (pRight->numOfRows == 1) {
@@ -1381,10 +1310,6 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight
       }
       *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx;
     }
-    pOutputCol->hasNull = pLeftCol->hasNull;
-    if (pOutputCol->hasNull) {
-      memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows));
-    }
   }
 }
 
@@ -1411,14 +1336,6 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
       }
       *output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i);
     }
-
-    pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
-    if (pOutputCol->hasNull) {
-      int32_t numOfBitLen = BitmapLen(pLeft->numOfRows);
-      for (int32_t j = 0; j < numOfBitLen; ++j) {
-        pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j];
-      }
-    }
   } else if (pLeft->numOfRows == 1) {
     vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
   } else if (pRight->numOfRows == 1) {
@@ -1439,6 +1356,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
   if (pRight->pHashFilter != NULL) {
     for (; i >= 0 && i < pLeft->numOfRows; i += step) {
       if (colDataIsNull_s(pLeft->columnData, i)) {
+        colDataAppendNULL(pOut->columnData, i);
         continue;
       }
 
@@ -1452,6 +1370,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step) {
       if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOut->columnData, i);
         continue;  // TODO set null or ignore
       }
 
@@ -1463,6 +1382,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       bool isJsonnull = false;
       convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
       if(isJsonnull){
+        colDataAppendNULL(pOut->columnData, i);
         continue;  // TODO set null or ignore
       }
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
@@ -1472,6 +1392,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
     ASSERT(pLeft->pHashFilter == NULL);
     for (; i >= 0 && i < pLeft->numOfRows; i += step) {
       if (colDataIsNull_s(pLeft->columnData, i)) {
+        colDataAppendNULL(pOut->columnData, i);
         continue;
       }
 
@@ -1482,6 +1403,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       bool isJsonnull = false;
       convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
       if(isJsonnull){
+        colDataAppendNULL(pOut->columnData, i);
         continue;  // TODO set null or ignore
       }
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
@@ -1490,6 +1412,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
   } else if (pLeft->numOfRows == 1) {
     for (; i >= 0 && i < pRight->numOfRows; i += step) {
       if (colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOut->columnData, i);
         continue;
       }
 
@@ -1500,6 +1423,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       bool isJsonnull = false;
       convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
       if(isJsonnull){
+        colDataAppendNULL(pOut->columnData, i);
         continue;  // TODO set null or ignore
       }
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
@@ -1589,10 +1513,6 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
 void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
-    if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){
-      char *data = colDataGetVarData(pLeft->columnData, i);
-      v = (*data == TSDB_DATA_TYPE_NULL)? 1:0;
-    }
     colDataAppendInt8(pOut->columnData, i, &v);
   }
   pOut->numOfRows = pLeft->numOfRows;
@@ -1601,10 +1521,6 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
 void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1;
-    if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){
-      char *data = colDataGetVarData(pLeft->columnData, i);
-      v = (*data == TSDB_DATA_TYPE_NULL)? 0:1;
-    }
     colDataAppendInt8(pOut->columnData, i, &v);
   }
   pOut->numOfRows = pLeft->numOfRows;
@@ -1671,3 +1587,4 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
       return NULL;
   }
 }
+
diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp
index 8049e80590..502ab7edfe 100644
--- a/source/libs/scalar/test/scalar/scalarTests.cpp
+++ b/source/libs/scalar/test/scalar/scalarTests.cpp
@@ -38,6 +38,7 @@
 #include "scalar.h"
 #include "nodes.h"
 #include "tlog.h"
+#include "parUtil.h"
 
 #define _DEBUG_PRINT_ 0
 
@@ -911,17 +912,32 @@ TEST(constantTest, greater_and_lower) {
 }
 
 void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
-  char key[32] = {0};
-  memcpy(varDataVal(key), key, strlen(key));
-  varDataLen(key) = strlen(key);
+  char keyVar[32] = {0};
+  memcpy(varDataVal(keyVar), key, strlen(key));
+  varDataLen(keyVar) = strlen(key);
 
-  SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
-  scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key);
+  SNode *pLeft = NULL, *pRight = NULL;
+  scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar);
   scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json);
   scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
 }
 
-void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorType opType, int32_t rightType, void *rightData){
+void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){
+  int32_t resType = TSDB_DATA_TYPE_NULL;
+  if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI ||
+      opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
+    resType = TSDB_DATA_TYPE_DOUBLE;
+  }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
+    resType = TSDB_DATA_TYPE_BIGINT;
+  }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL ||
+             opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL ||
+             opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
+             opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
+             opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH ||
+             opType == OP_TYPE_NMATCH){
+    resType = TSDB_DATA_TYPE_BOOL;
+  }
+
   SNode *right = NULL;
   scltMakeValueNode(&right, rightType, rightData);
   scltMakeOpNode(opNode, opType, resType, *opNode, right);
@@ -932,15 +948,15 @@ void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorT
   scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
 }
 
-void makeCalculate(void *json, void *key, void *value, double exceptValue, EOperatorType opType){
+void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){
   SArray *blockList = taosArrayInit(2, POINTER_BYTES);
   SSDataBlock *src = NULL;
   SNode *opNode = NULL;
 
-  makeJsonArrow(&src, &opNode, row, key);
+  makeJsonArrow(&src, &opNode, json, (char*)key);
   taosArrayPush(blockList, &src);
 
-  makeOperator(&opNode, blockList, TSDB_DATA_TYPE_DOUBLE, opType, TSDB_DATA_TYPE_INT, value);
+  makeOperator(&opNode, blockList, opType, rightType, rightData);
 
   int32_t code = scalarCalculate(opNode, blockList, NULL);
   ASSERT_EQ(code, 0);
@@ -948,39 +964,209 @@ void makeCalculate(void *json, void *key, void *value, double exceptValue, EOper
   SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
   ASSERT_EQ(res->info.rows, 1);
   SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
-  ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
-  ASSERT_EQ(*((double *)colDataGetData(column, 0)), exceptValue);
+
+  if(colDataIsNull_f(column->nullbitmap, 0)){
+    ASSERT_EQ(DBL_MAX, exceptValue);
+    printf("result:NULL\n");
+
+  }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
+             opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
+    double tmp = *((double *)colDataGetData(column, 0));
+    ASSERT_TRUE(tmp == exceptValue);
+    printf("result:%lf\n", tmp);
+  }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
+    ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
+    printf("result:%ld\n", *((int64_t *)colDataGetData(column, 0)));
+  }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN ||
+             opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
+             opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
+             opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){
+    ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
+    printf("result:%d\n", *((bool *)colDataGetData(column, 0)));
+  }
 
   taosArrayDestroyEx(blockList, scltFreeDataBlock);
   nodesDestroyNode(opNode);
 }
 
-TEST(columnTest, int_value_add_json_column) {
+TEST(columnTest, json_column_arith_op) {
   scltInitLogFile();
   char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
-  char *key = "k1";
 
   SKVRowBuilder kvRowBuilder;
   tdInitKVRowBuilder(&kvRowBuilder);
   parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
   SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
 
-  int32_t leftv = 1;
-  makeCalculate(row, key, &leftv, 5.0, OP_TYPE_ADD);
+  const int32_t len = 8;
+  EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
+                         OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
+  int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9};
 
-  leftv = 8;
-  makeCalculate(row, key, &leftv, -4, OP_TYPE_SUB);
+  printf("--------------------json int---------------------\n");
+  char *key = "k1";
+  double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
+  }
 
-  leftv = 2;
-  makeCalculate(row, key, &leftv, 8.0, OP_TYPE_MULTI);
+  printf("--------------------json string---------------------\n");
 
-  leftv = 2;
-  makeCalculate(row, key, &leftv, 2.0, OP_TYPE_DIV);
+  key = "k2";
+  double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
+  }
 
-  leftv = 3;
-  makeCalculate(row, key, &leftv, 1.0, OP_TYPE_MODE);
+  printf("---------------------json null--------------------\n");
 
-  makeCalculate(row, key, &leftv, -4, OP_TYPE_MINUS);
+  key = "k3";
+  double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
+  }
+
+  printf("---------------------json bool--------------------\n");
+
+  key = "k4";
+  double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
+  }
+
+  printf("----------------------json double-------------------\n");
+
+  key = "k5";
+  double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
+  }
+
+  printf("---------------------json not exist--------------------\n");
+
+  key = "k10";
+  double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
+  }
+}
+
+void *prepareNchar(char* rightData){
+  int32_t len = 0;
+  int32_t inputLen = strlen(rightData);
+
+  char* t = (char*)taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
+  taosMbsToUcs4(rightData, inputLen, (TdUcs4*) varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len);
+  varDataSetLen(t, len);
+  return t;
+}
+
+TEST(columnTest, json_column_logic_op) {
+  scltInitLogFile();
+  char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
+
+  SKVRowBuilder kvRowBuilder;
+  tdInitKVRowBuilder(&kvRowBuilder);
+  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+
+  const int32_t len = 9;
+  const int32_t len1 = 4;
+  EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL,
+                          OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH};
+
+  int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0};
+  char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"};
+
+  printf("--------------------json int---------------------\n");
+  char *key = "k1";
+  bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
+  }
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]);
+    taosMemoryFree(rightData);
+  }
+
+  printf("--------------------json string---------------------\n");
+
+  key = "k2";
+  bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
+  }
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]);
+    taosMemoryFree(rightData);
+  }
+
+  printf("--------------------json null---------------------\n");
+
+  key = "k3";
+  double eRes2[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
+  }
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]);
+    taosMemoryFree(rightData);
+  }
+
+  printf("--------------------json bool---------------------\n");
+
+  key = "k4";
+  bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
+  }
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]);
+    taosMemoryFree(rightData);
+  }
+
+  printf("--------------------json double---------------------\n");
+
+  key = "k5";
+  bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
+  }
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]);
+    taosMemoryFree(rightData);
+  }
+
+  printf("--------------------json double---------------------\n");
+
+  key = "k6";
+  bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, true};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
+  }
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]);
+    taosMemoryFree(rightData);
+  }
+
+  printf("---------------------json not exist--------------------\n");
+
+  key = "k10";
+  double eRes10[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]);
+  }
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]);
+    taosMemoryFree(rightData);
+  }
 }
 
 TEST(columnTest, smallint_value_add_int_column) {
@@ -3171,3 +3357,4 @@ int main(int argc, char** argv) {
 }
 
 #pragma GCC diagnostic pop
+

From 1ab6342018927dfcbce4c7f30f7f3ec0c08a44ca Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 20 Apr 2022 18:54:15 +0800
Subject: [PATCH 15/22] refactor: fix error in merge

---
 source/libs/parser/src/parUtil.c   | 1 -
 source/libs/scalar/src/sclvector.c | 1 -
 tools/shell/src/shellEngine.c      | 3 ++-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 0ff5843736..7f945eb70c 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -262,7 +262,6 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
     j++;
   }
   dst[j] = '\0';
-  strtrim(dst);
   return j;
 }
 
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index ae8c395997..d484738328 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -287,7 +287,6 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn
 
 //TODO opt performance, tmp is not needed.
 int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
-  ASSERT(!IS_VAR_DATA_TYPE(outType));
   int32_t bufSize = pIn->columnData->info.bytes;
   char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE);
 
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index e8f53ef981..ca5bf69795 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -761,7 +761,8 @@ static int calcColWidth(TAOS_FIELD *field, int precision) {
         return TMAX(field->bytes, width);
       }
 
-    case TSDB_DATA_TYPE_NCHAR: {
+    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:{
       int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
       if (bytes > tsMaxBinaryDisplayWidth) {
         return TMAX(tsMaxBinaryDisplayWidth, width);

From 32c24eedb87a247478327ea88170512e23c68356 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 20 Apr 2022 19:05:00 +0800
Subject: [PATCH 16/22] fix: invalid comment len

---
 source/dnode/mnode/impl/src/mndStb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index d8b94f5ffe..6bdf4575a8 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -348,7 +348,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
   memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema));
   memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema));
   if (pNew->commentLen != 0) {
-    memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN);
+    memcpy(pOld->comment, pNew->comment, pNew->commentLen);
   }
   if (pNew->ast1Len != 0) {
     memcpy(pOld->pAst1, pNew->pAst1, pNew->ast1Len);

From da232cbeaf2a906fe9c8cf74537cb869b6543baa Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Wed, 20 Apr 2022 20:02:10 +0800
Subject: [PATCH 17/22] [test: check error from valgrind log file]

---
 tests/script/sh/checkValgrind.sh          | 38 ++++++++++++
 tests/script/tsim/valgrind/checkError.sim | 76 +++++++++++++++++++++++
 tests/tsim/src/simExe.c                   |  6 ++
 3 files changed, 120 insertions(+)
 create mode 100755 tests/script/sh/checkValgrind.sh
 create mode 100644 tests/script/tsim/valgrind/checkError.sim

diff --git a/tests/script/sh/checkValgrind.sh b/tests/script/sh/checkValgrind.sh
new file mode 100755
index 0000000000..17d7ce1aff
--- /dev/null
+++ b/tests/script/sh/checkValgrind.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set +e
+#set -x
+
+NODE_NAME=
+
+while getopts "n:" arg
+do
+  case $arg in
+    n)
+      NODE_NAME=$OPTARG
+      ;;
+    ?)
+      echo "unkown argument"
+      ;;
+  esac
+done
+
+SCRIPT_DIR=`dirname $0`
+cd $SCRIPT_DIR/../
+SCRIPT_DIR=`pwd`
+
+IN_TDINTERNAL="community"
+if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  cd ../../..
+else
+  cd ../../
+fi
+
+TAOS_DIR=`pwd`
+LOG_DIR=$TAOS_DIR/sim/$NODE_NAME/log
+#CFG_DIR=$TAOS_DIR/sim/$NODE_NAME/cfg
+
+#echo ---- $LOG_DIR
+
+errors=`grep "ERROR SUMMARY:" ${LOG_DIR}/valgrind-taosd-*.log | cut -d ' ' -f 2,3,4,5 | tr -d "\n"`
+echo $errors
diff --git a/tests/script/tsim/valgrind/checkError.sim b/tests/script/tsim/valgrind/checkError.sim
new file mode 100644
index 0000000000..3b66cb4a8c
--- /dev/null
+++ b/tests/script/tsim/valgrind/checkError.sim
@@ -0,0 +1,76 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+#system sh/deploy.sh -n dnode2 -i 2
+#system sh/deploy.sh -n dnode3 -i 3
+#system sh/deploy.sh -n dnode4 -i 4
+#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+#system sh/exec.sh -n dnode2 -s start
+#system sh/exec.sh -n dnode3 -s start
+#system sh/exec.sh -n dnode4 -s start
+
+sleep 2000
+
+#$loop_cnt = 0
+#check_dnode_ready:
+#	$loop_cnt = $loop_cnt + 1
+#	sleep 200
+#	if $loop_cnt == 10 then
+#	  print ====> dnode not ready!
+#		return -1
+#	endi
+#sql show dnodes
+#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
+#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
+#if $data[0][0] != 1 then
+#  return -1
+#endi
+#if $data[0][4] != ready then
+#  goto check_dnode_ready
+#endi
+#
+##sql connect
+#sql create dnode $hostname port 7200
+#sql create dnode $hostname port 7300
+#sql create dnode $hostname port 7400
+#
+#$loop_cnt = 0
+#check_dnode_ready_1:
+#$loop_cnt = $loop_cnt + 1
+#sleep 200
+#if $loop_cnt == 10 then
+#  print ====> dnodes not ready!
+#	return -1
+#endi
+#sql show dnodes
+#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
+#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
+#if $data[0][4] != ready then
+#  goto check_dnode_ready_1
+#endi
+#if $data[1][4] != ready then
+#  goto check_dnode_ready_1
+#endi
+#if $data[2][4] != ready then
+#  goto check_dnode_ready_1
+#endi
+#if $data[3][4] != ready then
+#  goto check_dnode_ready_1
+#endi
+
+#=========== please add any actions above =================
+
+print ====> stop all dondes to output valgrind log file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+
+print ====> start to check if there are ERRORS in vagrind log file for each dnode
+# -n : dnode[x] be check
+system_content sh/checkValgrind.sh -n dnode1 
+print cmd return result----> [ $system_content ]
+if $system_content != @ERROR SUMMARY: 0 errors@ then
+  return -1
+endi 
diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c
index 8b3edb6035..dbc3c2c460 100644
--- a/tests/tsim/src/simExe.c
+++ b/tests/tsim/src/simExe.c
@@ -441,6 +441,12 @@ void simStoreSystemContentResult(SScript *script, char *filename) {
   // if ((fd = fopen(filename, "r")) != NULL) {
   if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) {
     taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1);
+    int32_t len = strlen(script->system_ret_content);
+    for (int32_t i = 0; i < len; ++i) {
+      if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') {
+        script->system_ret_content[i] = 0;
+      }
+    }
     taosCloseFile(&pFile);
     char rmCmd[MAX_FILE_NAME_LEN] = {0};
     sprintf(rmCmd, "rm -f %s", filename);

From 3ecf6df9e362ba5b18a54985048b6462775a7fb0 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Wed, 20 Apr 2022 20:29:50 +0800
Subject: [PATCH 18/22] [test: add valgrind case]

---
 tests/script/jenkins/basic.txt            |  3 +++
 tests/script/tsim/valgrind/checkError.sim | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 64ea437648..506abf3fc5 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -93,4 +93,7 @@
 # --- sma
 ./test.sh -f tsim/sma/tsmaCreateInsertData.sim
 
+# --- valgrind
+./test.sh -f tsim/valgrind/checkError.sim -v
+
 #======================b1-end===============
diff --git a/tests/script/tsim/valgrind/checkError.sim b/tests/script/tsim/valgrind/checkError.sim
index 3b66cb4a8c..80700dcbce 100644
--- a/tests/script/tsim/valgrind/checkError.sim
+++ b/tests/script/tsim/valgrind/checkError.sim
@@ -71,6 +71,13 @@ print ====> start to check if there are ERRORS in vagrind log file for each dnod
 # -n : dnode[x] be check
 system_content sh/checkValgrind.sh -n dnode1 
 print cmd return result----> [ $system_content ]
-if $system_content != @ERROR SUMMARY: 0 errors@ then
-  return -1
+if $system_content == @ERROR SUMMARY: 0 errors@ then
+  return 0
 endi 
+
+$null=
+if $system_content == $null then
+  return 0
+endi 
+
+return -1
\ No newline at end of file

From 29d994d1dd5040f1585008207dc26e1ab57006c4 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Wed, 20 Apr 2022 20:34:45 +0800
Subject: [PATCH 19/22] feat: taos shell refactor (#11688)

* [TD-13558]<feature>: taos shell refactor

add taosTools as submodule

* add tools/taos-tools

* add more client interface for taosTools compile

* update taos-tools

* update taos-tools

* refactor shell

* [TD-13558]<feature>: taos shell test speed

* [TD-13558]<feature>: taos -n startup works

* taos -n rpc works

* taos -n server works

* cleanup code since no endPort in 3.0

* update taos-tools

* [TD-13558]<feature>: taos -C works

* improve taos shell -c WIP

* update taos-tools

* add demoapi.c

* adjust show databases result for 3.0

* test: add platform logic

* add nchar

* adjust taos_fetch_lengths

* print fields

* remove show databases check from insert cases

* fix lua example compile for 3.0

still not work

* remove lua.py from smoketest

* use get_column_data_offset() to get offset and convert length

* add int, float, timestamp value convert

* improve tests/pytest/insert/binary.py

* improve example/src/demoapi.c
---
 example/src/demoapi.c         |  2 +-
 tests/pytest/insert/binary.py | 28 +++++++++++++++-------------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/example/src/demoapi.c b/example/src/demoapi.c
index 45da88fa1b..afa9e19d4d 100644
--- a/example/src/demoapi.c
+++ b/example/src/demoapi.c
@@ -176,7 +176,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
     }
 
     if (block) {
-        warnPrint("%s", "call taos_fetch_block()\n");
+        warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
         int rows = 0;
         while ((rows = taos_fetch_block(res, &row))) {
             int *lengths = taos_fetch_lengths(res);
diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py
index ffd1d6cb8c..28621f777b 100644
--- a/tests/pytest/insert/binary.py
+++ b/tests/pytest/insert/binary.py
@@ -35,8 +35,8 @@ class TDTestCase:
         tdSql.prepare()
 
         tdLog.info('=============== step1')
-        tdLog.info('create table tb (ts timestamp, speed binary(5))')
-        tdSql.execute('create table tb (ts timestamp, speed binary(5))')
+        tdLog.info('create table tb (ts timestamp, speed binary(10))')
+        tdSql.execute('create table tb (ts timestamp, speed binary(10))')
         tdLog.info("insert into tb values (now, ) -x step1")
         tdSql.error("insert into tb values (now, )")
         tdLog.info('=============== step2')
@@ -49,27 +49,29 @@ class TDTestCase:
         tdLog.info("tdSql.checkData(0, 0, '1234')")
         tdSql.checkData(0, 0, '1234')
         tdLog.info('=============== step3')
-        tdLog.info("insert into tb values (now+2a, '23456')")
-        tdSql.execute("insert into tb values (now+2a, '23456')")
+        tdLog.info("insert into tb values (now+2a, '0123456789')")
+        tdSql.execute("insert into tb values (now+2a, '0123456789')")
         tdLog.info('select speed from tb order by ts desc')
         tdSql.query('select speed from tb order by ts desc')
         tdLog.info('tdSql.checkRow(2)')
         tdSql.checkRows(2)
         tdLog.info('==> $data00')
-        tdLog.info("tdSql.checkData(0, 0, '23456')")
-        tdSql.checkData(0, 0, '23456')
+        tdLog.info("tdSql.checkData(0, 0, '0123456789')")
+        tdSql.checkData(0, 0, '0123456789')
         tdLog.info('=============== step4')
-        tdLog.info("insert into tb values (now+3a, '345678')")
-        tdSql.error("insert into tb values (now+3a, '345678')")
+        tdLog.info("insert into tb values (now+3a, '01234567890')")
+        tdSql.error("insert into tb values (now+3a, '01234567890')")
         tdLog.info("insert into tb values (now+3a, '34567')")
         tdSql.execute("insert into tb values (now+3a, '34567')")
+        tdLog.info("insert into tb values (now+4a, NULL)")
+        tdSql.execute("insert into tb values (now+4a, NULL)")
         tdLog.info('select speed from tb order by ts desc')
         tdSql.query('select speed from tb order by ts desc')
-        tdLog.info('tdSql.checkRow(3)')
-        tdSql.checkRows(3)
-        tdLog.info('==> $data00')
-        tdLog.info("tdSql.checkData(0, 0, '34567')")
-        tdSql.checkData(0, 0, '34567')
+        tdSql.checkRows(4)
+        tdLog.info("tdSql.checkData(0, 0, '0123456789')")
+        tdSql.checkData(0, 0, '0123456789')
+        tdLog.info("tdSql.checkData(3, 0, None)")
+        tdSql.checkData(3, 0, None)
         tdLog.info("insert into tb values (now+4a, \"'';\")")
 
         if platform.system() == "Linux":

From d75ab9b3ff053db2ea017bd1c1fdb5b70f4037a1 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Wed, 20 Apr 2022 21:36:55 +0800
Subject: [PATCH 20/22] refactor

---
 example/src/tmq.c                           |   3 +-
 include/common/tmsg.h                       |  79 +++++-
 source/client/inc/clientInt.h               |  60 ++---
 source/client/src/clientMain.c              |  35 ++-
 source/client/src/tmq.c                     | 210 ++++++----------
 source/dnode/mgmt/mgmt_mnode/src/mmHandle.c |   1 +
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |  11 +-
 source/dnode/mnode/impl/src/mndConsumer.c   |  15 +-
 source/dnode/mnode/impl/src/mndDef.c        |  29 ++-
 source/dnode/mnode/impl/src/mndScheduler.c  |  17 +-
 source/dnode/mnode/impl/src/mndSubscribe.c  |  74 +++---
 source/dnode/mnode/impl/src/mnode.c         |   6 +-
 source/dnode/vnode/src/inc/tq.h             |  49 ++--
 source/dnode/vnode/src/tq/tq.c              | 251 +++++++++-----------
 source/dnode/vnode/src/tq/tqMetaStore.c     |   1 -
 source/dnode/vnode/src/vnd/vnodeSvr.c       |  10 +-
 source/os/src/osFile.c                      |  40 ++--
 source/util/src/tarray.c                    |   7 +-
 tests/script/tsim/tmq/basic1.sim            |   4 +-
 tests/test/c/tmqSim.c                       |   6 +-
 20 files changed, 480 insertions(+), 428 deletions(-)

diff --git a/example/src/tmq.c b/example/src/tmq.c
index fdd26bc95d..4847e243a1 100644
--- a/example/src/tmq.c
+++ b/example/src/tmq.c
@@ -22,6 +22,7 @@
 static int  running = 1;
 static void msg_process(TAOS_RES* msg) {
   char buf[1024];
+  memset(buf, 0, 1024);
   printf("topic: %s\n", tmq_get_topic_name(msg));
   printf("vg:%d\n", tmq_get_vgroup_id(msg));
   while (1) {
@@ -220,7 +221,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
       msg_process(tmqmessage);
       tmq_message_destroy(tmqmessage);
 
-      if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
+      /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/
     }
   }
 
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index ecb5cd3d61..2b480a4067 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2396,6 +2396,7 @@ typedef struct {
   int64_t consumerId;
 } SMqRspHead;
 
+#if 0
 typedef struct {
   SMsgHead head;
 
@@ -2409,6 +2410,17 @@ typedef struct {
   uint64_t reqId;
   char     topic[TSDB_TOPIC_FNAME_LEN];
 } SMqPollReq;
+#endif
+
+typedef struct {
+  SMsgHead head;
+  char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int32_t  epoch;
+  uint64_t reqId;
+  int64_t  consumerId;
+  int64_t  blockingTime;
+  int64_t  currentOffset;
+} SMqPollReqV2;
 
 typedef struct {
   int32_t vgId;
@@ -2482,6 +2494,71 @@ static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRs
   return (void*)buf;
 }
 
+typedef struct {
+  SMqRspHead head;
+  int64_t    reqOffset;
+  int64_t    rspOffset;
+  int32_t    skipLogNum;
+  int32_t    blockNum;
+  int8_t     withTbName;
+  int8_t     withSchema;
+  int8_t     withTag;
+  int8_t     withTagSchema;
+  SArray*    blockDataLen;    // SArray<int32_t>
+  SArray*    blockData;       // SArray<SRetrieveTableRsp*>
+  SArray*    blockTbName;     // SArray<char*>
+  SArray*    blockSchema;     // SArray<SSchemaWrapper>
+  SArray*    blockTags;       // SArray<kvrow>
+  SArray*    blockTagSchema;  // SArray<kvrow>
+} SMqDataBlkRsp;
+
+static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
+  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
+  tlen += taosEncodeFixedI32(buf, pRsp->blockNum);
+  if (pRsp->blockNum != 0) {
+    tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
+    tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTag);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
+      void*   data = taosArrayGetP(pRsp->blockData, i);
+      tlen += taosEncodeFixedI32(buf, bLen);
+      tlen += taosEncodeBinary(buf, data, bLen);
+    }
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) {
+  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
+  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
+  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
+  buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
+  pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  if (pRsp->blockNum != 0) {
+    buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
+    buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTag);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = 0;
+      void*   data = NULL;
+      buf = taosDecodeFixedI32(buf, &bLen);
+      buf = taosDecodeBinary(buf, &data, bLen);
+      taosArrayPush(pRsp->blockDataLen, &bLen);
+      taosArrayPush(pRsp->blockData, &data);
+    }
+  }
+  return (void*)buf;
+}
+
 typedef struct {
   SMqRspHead head;
   char       cgroup[TSDB_CGROUP_LEN];
@@ -2489,7 +2566,7 @@ typedef struct {
 } SMqCMGetSubEpRsp;
 
 static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
-  taosMemoryFree(pSubTopicEp->schema.pSchema);
+  // taosMemoryFree(pSubTopicEp->schema.pSchema);
   taosArrayDestroy(pSubTopicEp->vgs);
 }
 
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 772ff5e69a..9363cf00a2 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -44,7 +44,7 @@ extern "C" {
   } while (0)
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
-#define HEARTBEAT_INTERVAL 1500  // ms
+#define HEARTBEAT_INTERVAL         1500  // ms
 
 enum {
   RES_TYPE__QUERY = 1,
@@ -187,11 +187,13 @@ typedef struct SRequestSendRecvBody {
 } SRequestSendRecvBody;
 
 typedef struct {
-  int8_t  resType;
-  char*   topic;
-  SArray* res;  // SArray<SReqResultInfo>
-  int32_t resIter;
-  int32_t vgId;
+  int8_t         resType;
+  char           topic[TSDB_TOPIC_FNAME_LEN];
+  int32_t        vgId;
+  SSchemaWrapper schema;
+  int32_t        resIter;
+  SMqDataBlkRsp  rsp;
+  SReqResultInfo resInfo;
 } SMqRspObj;
 
 typedef struct SRequestObj {
@@ -211,16 +213,24 @@ typedef struct SRequestObj {
   SRequestSendRecvBody body;
 } SRequestObj;
 
+void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
+void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
+void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
+void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
+
 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  int32_t    resIter = msg->resIter == -1 ? 0 : msg->resIter;
-  return (SReqResultInfo*)taosArrayGet(msg->res, resIter);
+  return (SReqResultInfo*)&msg->resInfo;
 }
 
-static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) {
+static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  if (++msg->resIter < taosArrayGetSize(msg->res)) {
-    return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter);
+  msg->resIter++;
+  if (msg->resIter < msg->rsp.blockNum) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
+    setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4);
+    return &msg->resInfo;
   }
   return NULL;
 }
@@ -238,25 +248,25 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t
 int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
-int   taos_init();
+int taos_init();
 
-void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
-void  destroyTscObj(void* pObj);
-STscObj *acquireTscObj(int64_t rid);
-int32_t releaseTscObj(int64_t rid);
+void*    createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
+void     destroyTscObj(void* pObj);
+STscObj* acquireTscObj(int64_t rid);
+int32_t  releaseTscObj(int64_t rid);
 
 uint64_t generateRequestId();
 
-void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
-void  destroyRequest(SRequestObj* pRequest);
-SRequestObj *acquireRequest(int64_t rid);
-int32_t releaseRequest(int64_t rid);
+void*        createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
+void         destroyRequest(SRequestObj* pRequest);
+SRequestObj* acquireRequest(int64_t rid);
+int32_t      releaseRequest(int64_t rid);
 
 char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
 void  resetConnectDB(STscObj* pTscObj);
 
-int  taos_options_imp(TSDB_OPTION option, const char* str);
+int taos_options_imp(TSDB_OPTION option, const char* str);
 
 void* openTransporter(const char* user, const char* auth, int32_t numOfThreads);
 
@@ -273,12 +283,6 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
-void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
-void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
-void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
-void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
-int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
-
 // --- heartbeat
 // global, called by mgmt
 int  hbMgrInit();
@@ -290,7 +294,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
 void       appHbMgrCleanup(void);
 
 // conn level
-int  hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
+int  hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
 void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
 
 int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen);
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 903018d5c3..6472dea556 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -14,12 +14,12 @@
  */
 
 #include "catalog.h"
-#include "scheduler.h"
 #include "clientInt.h"
-#include "clientStmt.h"
 #include "clientLog.h"
+#include "clientStmt.h"
 #include "os.h"
 #include "query.h"
+#include "scheduler.h"
 #include "tglobal.h"
 #include "tmsg.h"
 #include "tref.h"
@@ -177,25 +177,24 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return doFetchRows(pRequest, true, true);
 
   } else if (TD_RES_TMQ(res)) {
-    SMqRspObj *msg = ((SMqRspObj *)res);
-    if (msg->resIter == -1) msg->resIter++;
-    SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter);
+    SMqRspObj      *msg = ((SMqRspObj *)res);
+    SReqResultInfo *pResultInfo;
+    if (msg->resIter == -1) {
+      pResultInfo = tmqGetNextResInfo(res, true);
+    } else {
+      pResultInfo = tmqGetCurResInfo(res);
+    }
     if (pResultInfo->current < pResultInfo->numOfRows) {
       doSetOneRowPtr(pResultInfo);
       pResultInfo->current += 1;
       return pResultInfo->row;
     } else {
-      msg->resIter++;
-      if (msg->resIter < taosArrayGetSize(msg->res)) {
-        pResultInfo = taosArrayGet(msg->res, msg->resIter);
-        doSetOneRowPtr(pResultInfo);
-        pResultInfo->current += 1;
-        return pResultInfo->row;
-      } else {
-        return NULL;
-      }
+      pResultInfo = tmqGetNextResInfo(res, true);
+      if (pResultInfo == NULL) return NULL;
+      doSetOneRowPtr(pResultInfo);
+      pResultInfo->current += 1;
+      return pResultInfo->row;
     }
-
   } else {
     // assert to avoid un-initialization error
     ASSERT(0);
@@ -455,7 +454,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
     (*numOfRows) = pResultInfo->numOfRows;
     return pRequest->code;
   } else if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true);
     if (pResultInfo == NULL) return -1;
 
     pResultInfo->current = pResultInfo->numOfRows;
@@ -474,7 +473,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
   }
 
   if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false);
     if (pResultInfo == NULL) {
       (*numOfRows) = 0;
       return 0;
@@ -710,10 +709,8 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
   return stmtBindBatch(stmt, bind);
 }
 
-
 TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
   // TODO
   return NULL;
 }
 
-
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index 9ea7a85d9b..90a589c36c 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -72,25 +72,25 @@ struct tmq_conf_t {
 
 struct tmq_t {
   // conf
-  char           groupId[TSDB_CGROUP_LEN];
-  char           clientId[256];
-  int8_t         autoCommit;
-  int8_t         inWaiting;
+  char   groupId[TSDB_CGROUP_LEN];
+  char   clientId[256];
+  int8_t autoCommit;
+  /*int8_t         inWaiting;*/
   int64_t        consumerId;
   int32_t        epoch;
   int32_t        resetOffsetCfg;
   int64_t        status;
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
-  int32_t        nextTopicIdx;
-  int8_t         epStatus;
-  int32_t        epSkipCnt;
-  int32_t        waitingRequest;
-  int32_t        readyRequest;
-  SArray*        clientTopics;  // SArray<SMqClientTopic>
-  STaosQueue*    mqueue;        // queue of tmq_message_t
-  STaosQall*     qall;
-  tsem_t         rspSem;
+  /*int32_t        nextTopicIdx;*/
+  int8_t  epStatus;
+  int32_t epSkipCnt;
+  /*int32_t        waitingRequest;*/
+  /*int32_t        readyRequest;*/
+  SArray*     clientTopics;  // SArray<SMqClientTopic>
+  STaosQueue* mqueue;        // queue of tmq_message_t
+  STaosQall*  qall;
+  tsem_t      rspSem;
   // stat
   int64_t pollCnt;
 };
@@ -134,7 +134,7 @@ typedef struct {
   int32_t         epoch;
   SMqClientVg*    vgHandle;
   SMqClientTopic* topicHandle;
-  SMqPollRspV2    msg;
+  SMqDataBlkRsp   msg;
 } SMqPollRspWrapper;
 
 typedef struct {
@@ -145,6 +145,7 @@ typedef struct {
 
 typedef struct {
   tmq_t*  tmq;
+  int32_t code;
   int32_t sync;
   tsem_t  rspSem;
 } SMqAskEpCbParam;
@@ -327,12 +328,12 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
     return NULL;
   }
   pTmq->pTscObj = (STscObj*)conn;
-  pTmq->inWaiting = 0;
+  /*pTmq->inWaiting = 0;*/
   pTmq->status = 0;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
-  pTmq->waitingRequest = 0;
-  pTmq->readyRequest = 0;
+  /*pTmq->waitingRequest = 0;*/
+  /*pTmq->readyRequest = 0;*/
   pTmq->epStatus = 0;
   pTmq->epSkipCnt = 0;
   // set conf
@@ -372,12 +373,12 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ);
   if (pTmq->pTscObj == NULL) return NULL;
 
-  pTmq->inWaiting = 0;
+  /*pTmq->inWaiting = 0;*/
   pTmq->status = 0;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
-  pTmq->waitingRequest = 0;
-  pTmq->readyRequest = 0;
+  /*pTmq->waitingRequest = 0;*/
+  /*pTmq->readyRequest = 0;*/
   pTmq->epStatus = 0;
   pTmq->epSkipCnt = 0;
   // set conf
@@ -862,7 +863,6 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
 #endif
 
 int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
-  /*printf("recv poll\n");*/
   SMqPollCbParam* pParam = (SMqPollCbParam*)param;
   SMqClientVg*    pVg = pParam->pVg;
   SMqClientTopic* pTopic = pParam->pTopic;
@@ -875,17 +875,15 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
-    /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
-    /*tsem_post(&tmq->rspSem);*/
+    // do not write into queue since updating epoch reset
     tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
             tmqEpoch);
+    /*tsem_post(&tmq->rspSem);*/
     return 0;
   }
 
   if (msgEpoch != tmqEpoch) {
     tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
-  } else {
-    atomic_sub_fetch_32(&tmq->waitingRequest, 1);
   }
 
 #if 0
@@ -907,45 +905,33 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 #endif
 
-  /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
-  /*tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));*/
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper));
   if (pRspWrapper == NULL) {
     tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
     goto CREATE_MSG_FAIL;
   }
+
   pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP;
   pRspWrapper->vgHandle = pVg;
   pRspWrapper->topicHandle = pTopic;
-  /*memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));*/
-  memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
-  tDecodeSMqPollRspV2(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
-  // TODO: alloc mem
-  /*pRsp->*/
-  /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
 
-#if 0
-  if (pRsp->msg.numOfTopics == 0) {
-    /*printf("no data\n");*/
-    taosFreeQitem(pRsp);
-    goto CREATE_MSG_FAIL;
-  }
-#endif
+  memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
+
+  tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
 
   tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId,
            pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
-  atomic_add_fetch_32(&tmq->readyRequest, 1);
   /*tsem_post(&tmq->rspSem);*/
-  return 0;
 
+  return 0;
 CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
   /*tsem_post(&tmq->rspSem);*/
-  return code;
+  return -1;
 }
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
@@ -1028,6 +1014,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
 int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
   tmq_t*           tmq = pParam->tmq;
+  pParam->code = code;
   if (code != 0) {
     tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync);
     goto END;
@@ -1067,6 +1054,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
 
     taosWriteQitem(tmq->mqueue, pWrapper);
     /*tsem_post(&tmq->rspSem);*/
+    taosMemoryFree(pParam);
   }
 
 END:
@@ -1078,7 +1066,8 @@ END:
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
-  int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
+  int32_t code = 0;
+  int8_t  epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
   if (epStatus == 1) {
     int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
     tscTrace("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
@@ -1135,8 +1124,12 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
-  if (sync) tsem_wait(&pParam->rspSem);
-  return 0;
+  if (sync) {
+    tsem_wait(&pParam->rspSem);
+    code = pParam->code;
+    taosMemoryFree(pParam);
+  }
+  return code;
 }
 
 tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
@@ -1162,7 +1155,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
   return TMQ_RESP_ERR__FAIL;
 }
 
-SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
   int64_t reqOffset;
   if (pVg->currentOffset >= 0) {
     reqOffset = pVg->currentOffset;
@@ -1174,13 +1167,18 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
     reqOffset = tmq->resetOffsetCfg;
   }
 
-  SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq));
+  SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2));
   if (pReq == NULL) {
     return NULL;
   }
 
-  strcpy(pReq->topic, pTopic->topicName);
-  strcpy(pReq->cgroup, tmq->groupId);
+  /*strcpy(pReq->topic, pTopic->topicName);*/
+  /*strcpy(pReq->cgroup, tmq->groupId);*/
+
+  int32_t tlen = strlen(tmq->groupId);
+  memcpy(pReq->subKey, tmq->groupId, tlen);
+  pReq->subKey[tlen] = TMQ_SEPARATOR;
+  strcpy(pReq->subKey + tlen + 1, pTopic->topicName);
 
   pReq->blockingTime = blockingTime;
   pReq->consumerId = tmq->consumerId;
@@ -1189,102 +1187,27 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
   pReq->reqId = generateRequestId();
 
   pReq->head.vgId = htonl(pVg->vgId);
-  pReq->head.contLen = htonl(sizeof(SMqPollReq));
+  pReq->head.contLen = htonl(sizeof(SMqPollReqV2));
   return pReq;
 }
 
 SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
   pRspObj->resType = RES_TYPE__TMQ;
-  pRspObj->topic = strdup(pWrapper->topicHandle->topicName);
-  pRspObj->resIter = -1;
+  strncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
   pRspObj->vgId = pWrapper->vgHandle->vgId;
-  SMqPollRspV2* pRsp = &pWrapper->msg;
-  int32_t       blockNum = taosArrayGetSize(pRsp->blockPos);
-  pRspObj->res = taosArrayInit(blockNum, sizeof(SReqResultInfo));
-  for (int32_t i = 0; i < blockNum; i++) {
-    int32_t            pos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
-    SRetrieveTableRsp* pRetrieve = POINTER_SHIFT(pRsp->blockData, pos);
-    SReqResultInfo     resInfo = {0};
-    resInfo.totalRows = 0;
-    resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-    setResSchemaInfo(&resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
-    setQueryResultFromRsp(&resInfo, pRetrieve, true);
-    taosArrayPush(pRspObj->res, &resInfo);
-  }
+  pRspObj->resIter = -1;
+  memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp));
+
+  /*SRetrieveTableRsp* pRetrieve = taosArrayGetP(pWrapper->msg.blockData, 0);*/
+  pRspObj->resInfo.totalRows = 0;
+  pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
+  setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
+
+  taosFreeQitem(pWrapper);
   return pRspObj;
 }
 
-#if 0
-tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
-  tmq_message_t* msg = NULL;
-  for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
-    SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
-      SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-      int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
-      /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
-      /*continue;*/
-      /*}*/
-      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
-      if (pReq == NULL) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
-        return NULL;
-      }
-
-      SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
-      if (pParam == NULL) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
-        return NULL;
-      }
-      pParam->tmq = tmq;
-      pParam->pVg = pVg;
-      pParam->epoch = tmq->epoch;
-      pParam->sync = 1;
-      pParam->msg = &msg;
-      tsem_init(&pParam->rspSem, 0, 0);
-
-      SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
-      if (sendInfo == NULL) {
-        return NULL;
-      }
-
-      sendInfo->msgInfo = (SDataBuf){
-          .pData = pReq,
-          .len = sizeof(SMqPollReq),
-          .handle = NULL,
-      };
-      sendInfo->requestId = generateRequestId();
-      sendInfo->requestObjRefId = 0;
-      sendInfo->param = pParam;
-      sendInfo->fp = tmqPollCb;
-      sendInfo->msgType = TDMT_VND_CONSUME;
-
-      int64_t transporterId = 0;
-      /*printf("send poll\n");*/
-      atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
-      pVg->pollCnt++;
-      tmq->pollCnt++;
-
-      tsem_wait(&pParam->rspSem);
-      tmq_message_t* nmsg = NULL;
-      while (1) {
-        taosReadQitem(tmq->mqueue, (void**)&nmsg);
-        if (nmsg == NULL) continue;
-        while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {
-          taosReadQitem(tmq->mqueue, (void**)&nmsg);
-        }
-        return nmsg;
-      }
-    }
-  }
-  return NULL;
-}
-#endif
-
 int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
   /*printf("call poll\n");*/
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
@@ -1306,7 +1229,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 #endif
       }
       atomic_store_32(&pVg->vgSkipCnt, 0);
-      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         /*tsem_post(&tmq->rspSem);*/
@@ -1337,7 +1260,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       sendInfo->msgInfo = (SDataBuf){
           .pData = pReq,
-          .len = sizeof(SMqPollReq),
+          .len = sizeof(SMqPollReqV2),
           .handle = NULL,
       };
       sendInfo->requestId = pReq->reqId;
@@ -1348,7 +1271,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
-      atomic_add_fetch_32(&tmq->waitingRequest, 1);
+      /*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/
       tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
                pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
@@ -1390,7 +1313,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
 
     if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
-      atomic_sub_fetch_32(&tmq->readyRequest, 1);
+      /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
       /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
       if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) {
         /*printf("epoch match\n");*/
@@ -1398,7 +1321,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
         pVg->currentOffset = pollRspWrapper->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        if (pollRspWrapper->msg.dataLen == 0) {
+        if (pollRspWrapper->msg.blockNum == 0) {
           taosFreeQitem(pollRspWrapper);
           rspWrapper = NULL;
           continue;
@@ -1454,7 +1377,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
   // TODO: put into another thread or delayed queue
   int64_t status = atomic_load_64(&tmq->status);
-  tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
+  while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) {
+    tscDebug("not ready, retry\n");
+    taosSsleep(1);
+  }
 
   rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
   if (rspObj) {
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
index 38885333c9..4cf4a405e0 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@@ -186,6 +186,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE);
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 751edd6f98..c9bc3514cc 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -321,7 +321,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/
   dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -334,10 +334,11 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/
+  /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/
+  /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/
+  /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);*/
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 8a66e5cb69..2ce7f0ae5f 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -127,7 +127,6 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
     pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
     if (pIter == NULL) break;
 
-    taosRLockLatch(&pConsumer->lock);
     int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
     int32_t status = atomic_load_32(&pConsumer->status);
     if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) {
@@ -143,6 +142,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
     if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) {
       // do nothing
     } else if (status == MQ_CONSUMER_STATUS__LOST) {
+      taosRLockLatch(&pConsumer->lock);
       int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics);
       for (int32_t i = 0; i < topicNum; i++) {
         char  key[TSDB_SUBSCRIBE_KEY_LEN];
@@ -151,7 +151,9 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
         SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
         taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
       }
+      taosRUnLockLatch(&pConsumer->lock);
     } else if (status == MQ_CONSUMER_STATUS__MODIFY) {
+      taosRLockLatch(&pConsumer->lock);
       int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
       for (int32_t i = 0; i < newTopicNum; i++) {
         char  key[TSDB_SUBSCRIBE_KEY_LEN];
@@ -169,11 +171,11 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
         SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
         taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
       }
+      taosRUnLockLatch(&pConsumer->lock);
     } else {
       // do nothing
     }
 
-    taosRUnLockLatch(&pConsumer->lock);
     mndReleaseConsumer(pMnode, pConsumer);
   }
 
@@ -188,7 +190,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   } else {
     taosHashCleanup(pRebMsg->rebSubHash);
     rpcFreeCont(pRebMsg);
-    mInfo("mq rebalance finished, no modification");
+    mTrace("mq rebalance finished, no modification");
     atomic_store_8(&mqInRebFlag, 0);
   }
   return 0;
@@ -213,12 +215,12 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
 
   // 1. check consumer status
   int32_t status = atomic_load_32(&pConsumer->status);
+
   if (status == MQ_CONSUMER_STATUS__LOST) {
-    // recover consumer
+    // TODO: recover consumer
   }
 
   if (status != MQ_CONSUMER_STATUS__READY) {
-    mndReleaseConsumer(pMnode, pConsumer);
     terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
     return -1;
   }
@@ -228,11 +230,13 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
   // 2. check epoch, only send ep info when epoches do not match
   if (epoch != serverEpoch) {
     taosRLockLatch(&pConsumer->lock);
+    mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
     int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics);
 
     rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp));
     if (rsp.topics == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
+      taosRUnLockLatch(&pConsumer->lock);
       goto FAIL;
     }
 
@@ -265,6 +269,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
       topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
       if (topicEp.vgs == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
+        taosRUnLockLatch(&pConsumer->lock);
         goto FAIL;
       }
 
diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index dd2d6801e1..37c819ae60 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -250,13 +250,28 @@ void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) {
 int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId);
-  tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);
+  int32_t sz = taosArrayGetSize(pEpInSub->vgs);
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i);
+    tlen += tEncodeSMqVgEp(buf, pVgEp);
+  }
+  /*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/
   return tlen;
 }
 
 void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) {
   buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId);
-  buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));
+  /*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  pEpInSub->vgs = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
+    buf = tDecodeSMqVgEp(buf, pVgEp);
+    taosArrayPush(pEpInSub->vgs, &pVgEp);
+  }
+
   return (void *)buf;
 }
 
@@ -268,10 +283,12 @@ SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
   pSubNew->vgNum = -1;
   pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
   // TODO set free fp
-  SMqConsumerEpInSub *pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));
-  pEpInSub->vgs = taosArrayInit(0, sizeof(SMqVgEp));
+  SMqConsumerEpInSub epInSub = {
+      .consumerId = -1,
+      .vgs = taosArrayInit(0, sizeof(void *)),
+  };
   int64_t unexistKey = -1;
-  taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), pEpInSub, sizeof(SMqConsumerEpInSub));
+  taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
   return pSubNew;
 }
 
@@ -287,7 +304,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
   void               *pIter = NULL;
   SMqConsumerEpInSub *pEpInSub = NULL;
   while (1) {
-    pIter = taosHashIterate(pSubNew->consumerHash, pIter);
+    pIter = taosHashIterate(pSub->consumerHash, pIter);
     if (pIter == NULL) break;
     pEpInSub = (SMqConsumerEpInSub *)pIter;
     SMqConsumerEpInSub newEp = {
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 5629421a84..b3583af1dc 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -434,7 +434,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     return -1;
   }
 
-  ASSERT(pSub->vgNum == 0);
+  ASSERT(pSub->vgNum == -1);
+
+  pSub->vgNum = 0;
 
   int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
   if (levelNum != 1) {
@@ -455,6 +457,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
 
   int64_t             unexistKey = -1;
   SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
+  ASSERT(pEpInSub);
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
 
   void* pIter = NULL;
   while (1) {
@@ -492,10 +497,18 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     }
     taosArrayPush(pEpInSub->vgs, &pVgEp);
 
+    ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
     /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
   }
 
-  taosHashRelease(pSub->consumerHash, pEpInSub);
+  ASSERT(pEpInSub->vgs->size > 0);
+  pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
+
+  ASSERT(pEpInSub->vgs->size > 0);
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
   qDestroyQueryPlan(pPlan);
 
   return 0;
diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index 48c9804619..7b2ec83230 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -109,6 +109,7 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
 
   if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
     tDeleteSubscribeObj(pSub);
@@ -116,6 +117,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
     return NULL;
   }
 
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
   return pSub;
 }
 
@@ -255,7 +258,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const char *subK
   req.oldConsumerId = pRebVg->oldConsumerId;
   req.newConsumerId = pRebVg->newConsumerId;
   req.vgId = pRebVg->pVgEp->vgId;
-  req.qmsg = req.qmsg;
+  req.qmsg = pRebVg->pVgEp->qmsg;
   strncpy(req.subKey, subKey, TSDB_SUBSCRIBE_KEY_LEN);
 
   int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req);
@@ -663,14 +666,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   if (pInput->pTopic != NULL) {
     // create subscribe
     pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key);
+    ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1);
   } else {
     pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub);
   }
   int32_t totalVgNum = pOutput->pSub->vgNum;
 
+  mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum);
+
   // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
   SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
 
+  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
   // 2. check and get actual removed consumers, put their vg into hash
   int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
   int32_t actualRemoved = 0;
@@ -692,18 +699,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
         };
         taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
       }
-      taosHashRelease(pOutput->pSub->consumerHash, pEpInSub);
       taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
       // put into removed
       taosArrayPush(pOutput->removedConsumers, &consumerId);
     }
   }
   ASSERT(removedNum == actualRemoved);
+  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
 
   // if previously no consumer, there are vgs not assigned
   {
-    int64_t             key = -1;
-    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t));
+    int64_t             unexistKey = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
     ASSERT(pEpInSub);
     int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
     for (int32_t i = 0; i < consumerVgNum; i++) {
@@ -715,18 +722,22 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       };
       taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
     }
-    taosHashRelease(pOutput->pSub->consumerHash, pEpInSub);
   }
 
   // 3. calc vg number of each consumer
-  int32_t actualConsumerNum = taosHashGetSize(pInput->pOldSub->consumerHash) - 1 +
-                              taosArrayGetSize(pInput->pRebInfo->newConsumers) -
-                              taosArrayGetSize(pInput->pRebInfo->removedConsumers);
-  int32_t afterRebConsumerNum = taosHashGetSize(pOutput->pSub->consumerHash) - 1;
-  ASSERT(afterRebConsumerNum == actualConsumerNum);
+  int32_t oldSz = 0;
+  if (pInput->pOldSub) {
+    oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1;
+  }
+  int32_t afterRebConsumerNum =
+      oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+  int32_t minVgCnt = 0;
+  int32_t imbConsumerNum = 0;
   // calc num
-  int32_t minVgCnt = totalVgNum / actualConsumerNum;
-  int32_t imbConsumerNum = totalVgNum % actualConsumerNum;
+  if (afterRebConsumerNum) {
+    minVgCnt = totalVgNum / afterRebConsumerNum;
+    imbConsumerNum = totalVgNum % afterRebConsumerNum;
+  }
 
   // 4. first scan: remove consumer more than wanted, put to remove hash
   int32_t imbCnt = 0;
@@ -735,6 +746,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
     if (pIter == NULL) break;
     SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    if (pEpInSub->consumerId == -1) continue;
     ASSERT(pEpInSub->consumerId > 0);
     int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
     // all old consumers still existing are touched
@@ -783,6 +795,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
       taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp,
                   sizeof(SMqConsumerEpInSub));
+      /*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
+      /*ASSERT(pTestNew->consumerId == consumerId);*/
+      /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/
       taosArrayPush(pOutput->newConsumers, &consumerId);
     }
   }
@@ -796,6 +811,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
     if (pIter == NULL) break;
     SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    if (pEpInSub->consumerId == -1) continue;
     ASSERT(pEpInSub->consumerId > 0);
     /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/
     if (imbCnt < imbConsumerNum) {
@@ -833,8 +849,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     ASSERT(pRemovedIter == NULL);
   } else {
     // if all consumer is removed, put all vg into unassigned
-    int64_t             key = -1;
-    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t));
+    int64_t             unexistKey = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
     ASSERT(pEpInSub);
     ASSERT(pEpInSub->consumerId == -1);
 
@@ -845,7 +861,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       if (pIter == NULL) break;
       pRebOutput = (SMqRebOutputVg *)pIter;
       ASSERT(pRebOutput->newConsumerId == -1);
-      taosArrayPush(pEpInSub->vgs, pRebOutput->pVgEp);
+      taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp);
+      taosArrayPush(pOutput->rebVgs, pRebOutput);
     }
   }
 
@@ -895,7 +912,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO
   // 3.2 set new consumer
   consumerNum = taosArrayGetSize(pOutput->newConsumers);
   for (int32_t i = 0; i < consumerNum; i++) {
-    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i);
+    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i);
     ASSERT(consumerId > 0);
     SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
     SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
@@ -948,17 +965,22 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
   while (1) {
     pIter = taosHashIterate(pReq->rebSubHash, pIter);
     if (pIter == NULL) break;
-    SMqRebInputObj   rebInput = {0};
-    SMqRebOutputObj  rebOutput = {0};
+    SMqRebInputObj rebInput = {0};
+
+    SMqRebOutputObj rebOutput = {0};
+    rebOutput.newConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.removedConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.touchedConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg));
+
     SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter;
     SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
 
     if (pSub == NULL) {
-      taosRLockLatch(&pSub->lock);
       // split sub key and extract topic
       char topic[TSDB_TOPIC_FNAME_LEN];
       char cgroup[TSDB_CGROUP_LEN];
-      mndSplitSubscribeKey(pSub->key, topic, cgroup);
+      mndSplitSubscribeKey(pRebSub->key, topic, cgroup);
       SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
       ASSERT(pTopic);
       taosRLockLatch(&pTopic->lock);
@@ -968,24 +990,16 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
     rebInput.pRebInfo = pRebSub;
     rebInput.pOldSub = pSub;
 
-    int32_t             unassignedVgNum = 0;
-    int64_t             key = -1;
-    SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &key, sizeof(int64_t));
-    if (pEpInSub != NULL) {
-      ASSERT(pEpInSub->consumerId == key);
-      unassignedVgNum = taosArrayGetSize(pEpInSub->vgs);
-    }
-    mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, unassignedVgNum);
-
     // TODO replace assert with error check
     ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
+    ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);
     ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
 
     if (rebInput.pTopic) {
       SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic;
       taosRUnLockLatch(&pTopic->lock);
       mndReleaseTopic(pMnode, pTopic);
-      taosRUnLockLatch(&pSub->lock);
+    } else {
       mndReleaseSubscribe(pMnode, pSub);
     }
   }
diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c
index 3ac1c522b2..5a61cefee1 100644
--- a/source/dnode/mnode/impl/src/mnode.c
+++ b/source/dnode/mnode/impl/src/mnode.c
@@ -22,12 +22,14 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndFunc.h"
+#include "mndGrant.h"
 #include "mndInfoSchema.h"
-#include "mndPerfSchema.h"
 #include "mndMnode.h"
 #include "mndOffset.h"
+#include "mndPerfSchema.h"
 #include "mndProfile.h"
 #include "mndQnode.h"
+#include "mndQuery.h"
 #include "mndShow.h"
 #include "mndSma.h"
 #include "mndSnode.h"
@@ -40,8 +42,6 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
-#include "mndQuery.h"
-#include "mndGrant.h"
 
 #define MQ_TIMER_MS    3000
 #define TRNAS_TIMER_MS 6000
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index ed33473b16..5e074eeb12 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -16,6 +16,13 @@
 #ifndef _TD_VNODE_TQ_H_
 #define _TD_VNODE_TQ_H_
 
+#include "executor.h"
+#include "os.h"
+#include "thash.h"
+#include "tmsg.h"
+#include "ttimer.h"
+#include "wal.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -30,12 +37,6 @@ extern "C" {
 #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
-enum {
-  TQ_STREAM_TOKEN__DATA = 1,
-  TQ_STREAM_TOKEN__WATERMARK,
-  TQ_STREAM_TOKEN__CHECKPOINT,
-};
-
 #define TQ_BUFFER_SIZE 4
 
 #define TQ_BUCKET_MASK 0xFF
@@ -151,22 +152,27 @@ typedef struct {
 } STqMetaStore;
 
 typedef struct {
-  SMemAllocatorFactory* pAllocatorFactory;
-  SMemAllocator*        pAllocator;
-} STqMemRef;
+  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int64_t consumerId;
+  int32_t epoch;
+  char*   qmsg;
+  // SRWLatch        lock;
+  SWalReadHandle* pReadHandle;
+  // number should be identical to fetch thread num
+  qTaskInfo_t task[4];
+} STqExec;
 
 struct STQ {
   // the collection of groups
   // the handle of meta kvstore
   bool          writeTrigger;
   char*         path;
-  STqMemRef     tqMemRef;
   STqMetaStore* tqMeta;
-  // STqPushMgr*   tqPushMgr;
-  SHashObj* pStreamTasks;
-  SVnode*   pVnode;
-  SWal*     pWal;
-  SMeta*    pVnodeMeta;
+  SHashObj*     tqMetaNew;  // subKey -> tqExec
+  SHashObj*     pStreamTasks;
+  SVnode*       pVnode;
+  SWal*         pWal;
+  SMeta*        pVnodeMeta;
 };
 
 typedef struct {
@@ -230,10 +236,6 @@ typedef struct {
   // TODO sync function
 } STqStreamPusher;
 
-typedef struct {
-  int8_t type;  // mq or stream
-} STqPusher;
-
 typedef struct {
   SHashObj* pHash;  // <id, STqPush*>
 } STqPushMgr;
@@ -257,9 +259,10 @@ int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
 int tqCommit(STQ*);
 
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
-int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
-int32_t tqProcessRebReq(STQ* pTq, char* msg);
-int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
+// int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
+// int32_t tqProcessRebReq(STQ* pTq, char* msg);
+// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
@@ -314,4 +317,4 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet
 }
 #endif
 
-#endif /*_TD_VNODE_TQ_H_*/
\ No newline at end of file
+#endif /*_TD_VNODE_TQ_H_*/
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 0aa6023eaf..16c5a4752e 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -29,31 +29,14 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMe
   pTq->pVnode = pVnode;
   pTq->pWal = pWal;
   pTq->pVnodeMeta = pVnodeMeta;
-#if 0
-  pTq->tqMemRef.pAllocatorFactory = allocFac;
-  pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
-  if (pTq->tqMemRef.pAllocator == NULL) {
-    // TODO: error code of buffer pool
-  }
-#endif
   pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer,
                             (FTqDelete)taosMemoryFree, 0);
   if (pTq->tqMeta == NULL) {
     taosMemoryFree(pTq);
-#if 0
-    allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator);
-#endif
     return NULL;
   }
 
-#if 0
-  pTq->tqPushMgr = tqPushMgrOpen();
-  if (pTq->tqPushMgr == NULL) {
-    // free store
-    taosMemoryFree(pTq);
-    return NULL;
-  }
-#endif
+  pTq->tqMetaNew = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
 
   pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
 
@@ -248,16 +231,15 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 
   return 0;
 }
-#if 0
+
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
-  SMqPollReq* pReq = pMsg->pCont;
-  int64_t     consumerId = pReq->consumerId;
-  int64_t     fetchOffset;
-  int64_t     blockingTime = pReq->blockingTime;
-  int32_t     reqEpoch = pReq->epoch;
+  SMqPollReqV2* pReq = pMsg->pCont;
+  int64_t       consumerId = pReq->consumerId;
+  int32_t       reqEpoch = pReq->epoch;
+  int64_t       fetchOffset;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-    fetchOffset = 0;
+    fetchOffset = walGetFirstVer(pTq->pWal);
   } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
     fetchOffset = walGetLastVer(pTq->pWal);
   } else {
@@ -267,65 +249,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
          TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
-  SMqPollRsp rsp = {
-      /*.consumerId = consumerId,*/
-      .numOfTopics = 0,
-      .pBlockData = NULL,
-  };
+  STqExec* pExec = taosHashGet(pTq->tqMetaNew, pReq->subKey, strlen(pReq->subKey));
+  ASSERT(pExec);
 
-  STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
-  if (pConsumer == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch);
+  int32_t consumerEpoch = atomic_load_32(&pExec->epoch);
   while (consumerEpoch < reqEpoch) {
-    consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch);
+    consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch);
   }
 
-  STqTopic* pTopic = NULL;
-  int32_t   sz = taosArrayGetSize(pConsumer->topics);
-  for (int32_t i = 0; i < sz; i++) {
-    STqTopic* topic = taosArrayGet(pConsumer->topics, i);
-    // TODO race condition
-    ASSERT(pConsumer->consumerId == consumerId);
-    if (strcmp(topic->topicName, pReq->topic) == 0) {
-      pTopic = topic;
-      break;
-    }
-  }
-  if (pTopic == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
-          TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
-         TD_VID(pTq->pVnode));
-
+  SMqDataBlkRsp rsp = {0};
   rsp.reqOffset = pReq->currentOffset;
-  rsp.skipLogNum = 0;
+  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
+  rsp.blockData = taosArrayInit(0, sizeof(void*));
 
   while (1) {
-    /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
-    // TODO
-    consumerEpoch = atomic_load_32(&pConsumer->epoch);
+    consumerEpoch = atomic_load_32(&pExec->epoch);
     if (consumerEpoch > reqEpoch) {
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
              consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
+
     SWalReadHead* pHead;
-    if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
+    if (walReadWithHandle_s(pExec->pReadHandle, fetchOffset, &pHead) < 0) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
@@ -333,101 +279,86 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
              TD_VID(pTq->pVnode), fetchOffset);
       break;
     }
+
     vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
            TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
-    /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
-    /*pHead = pTopic->pReadhandle->pHead;*/
+
     if (pHead->msgType == TDMT_VND_SUBMIT) {
       SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
-      qTaskInfo_t task = pTopic->buffer.output[workerId].task;
+      qTaskInfo_t task = pExec->task[workerId];
       ASSERT(task);
       qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
-      SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
       while (1) {
         SSDataBlock* pDataBlock = NULL;
-        uint64_t     ts;
+        uint64_t     ts = 0;
         if (qExecTask(task, &pDataBlock, &ts) < 0) {
-          ASSERT(false);
-        }
-        if (pDataBlock == NULL) {
-          /*pos = fetchOffset % TQ_BUFFER_SIZE;*/
-          break;
+          ASSERT(0);
         }
+        if (pDataBlock == NULL) break;
 
-        taosArrayPush(pRes, pDataBlock);
+        ASSERT(pDataBlock->info.rows != 0);
+        ASSERT(pDataBlock->info.numOfCols != 0);
+
+        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
+        void*              buf = taosMemoryCalloc(1, dataStrLen);
+        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+        pRetrieve->useconds = ts;
+        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+        pRetrieve->compressed = 0;
+        pRetrieve->completed = 1;
+        pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
+
+        // TODO enable compress
+        int32_t actualLen = 0;
+        blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
+        actualLen += sizeof(SRetrieveTableRsp);
+        ASSERT(actualLen <= dataStrLen);
+        taosArrayPush(rsp.blockDataLen, &actualLen);
+        taosArrayPush(rsp.blockData, &buf);
+        rsp.blockNum++;
       }
-
-      if (taosArrayGetSize(pRes) == 0) {
-        vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
-               pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
-        fetchOffset++;
-        rsp.skipLogNum++;
-        taosArrayDestroy(pRes);
-        continue;
-      }
-      rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper;
-      rsp.rspOffset = fetchOffset;
-
-      rsp.numOfTopics = 1;
-      rsp.pBlockData = pRes;
-
-      int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
-      void*   buf = rpcMallocCont(tlen);
-      if (buf == NULL) {
-        pMsg->code = -1;
-        taosMemoryFree(pHead);
-        return -1;
-      }
-      ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
-      ((SMqRspHead*)buf)->epoch = pReq->epoch;
-      ((SMqRspHead*)buf)->consumerId = consumerId;
-
-      void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-      tEncodeSMqPollRsp(&abuf, &rsp);
-      /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/
-      pMsg->pCont = buf;
-      pMsg->contLen = tlen;
-      pMsg->code = 0;
-      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
-             pHead->msgType, consumerId, pReq->epoch);
-      tmsgSendRsp(pMsg);
-      taosMemoryFree(pHead);
-      return 0;
-    } else {
-      taosMemoryFree(pHead);
-      fetchOffset++;
-      rsp.skipLogNum++;
     }
+
+    // TODO batch optimization
+    if (rsp.blockNum != 0) break;
+    rsp.skipLogNum++;
+    fetchOffset++;
   }
 
-  /*if (blockingTime != 0) {*/
-  /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
-  /*} else {*/
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
+  ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
+  ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
+
+  if (rsp.blockNum != 0)
+    rsp.rspOffset = fetchOffset;
+  else
+    rsp.rspOffset = fetchOffset - 1;
+
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     pMsg->code = -1;
     return -1;
   }
+
   ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
   ((SMqRspHead*)buf)->epoch = pReq->epoch;
-  rsp.rspOffset = fetchOffset - 1;
+  ((SMqRspHead*)buf)->consumerId = consumerId;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqPollRsp(&abuf, &rsp);
-  rsp.pBlockData = NULL;
+  tEncodeSMqDataBlkRsp(&abuf, &rsp);
   pMsg->pCont = buf;
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
-         pReq->epoch);
-  /*}*/
 
+  vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+         TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
+
+  // TODO destroy
   return 0;
 }
-#endif
 
+#if 0
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
@@ -436,7 +367,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   int32_t     reqEpoch = pReq->epoch;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-    fetchOffset = 0;
+    fetchOffset = walGetFirstVer(pTq->pWal);
   } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
     fetchOffset = walGetLastVer(pTq->pWal);
   } else {
@@ -635,7 +566,56 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   return 0;
 }
+#endif
 
+// TODO: persist meta into tdb
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
+  SMqRebVgReq req;
+  tDecodeSMqRebVgReq(msg, &req);
+  // todo lock
+  STqExec* pExec = taosHashGet(pTq->tqMetaNew, req.subKey, strlen(req.subKey));
+  if (pExec == NULL) {
+    ASSERT(req.oldConsumerId == -1);
+    ASSERT(req.newConsumerId != -1);
+    STqExec exec = {0};
+    pExec = &exec;
+    /*taosInitRWLatch(&pExec->lock);*/
+
+    memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
+    pExec->consumerId = req.newConsumerId;
+    pExec->epoch = -1;
+    pExec->qmsg = req.qmsg;
+    req.qmsg = NULL;
+    pExec->pReadHandle = walOpenReadHandle(pTq->pVnode->pWal);
+    for (int32_t i = 0; i < 4; i++) {
+      STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
+      SReadHandle    handle = {
+             .reader = pReadHandle,
+             .meta = pTq->pVnodeMeta,
+      };
+      pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);
+      ASSERT(pExec->task[i]);
+    }
+    taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
+    return 0;
+  } else {
+    if (req.newConsumerId != -1) {
+      /*taosWLockLatch(&pExec->lock);*/
+      ASSERT(pExec->consumerId == req.oldConsumerId);
+      // TODO handle qmsg and exec modification
+      atomic_store_64(&pExec->consumerId, req.newConsumerId);
+      atomic_add_fetch_32(&pExec->epoch, 1);
+      /*taosWUnLockLatch(&pExec->lock);*/
+      return 0;
+    } else {
+      // TODO
+      /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/
+      return 0;
+    }
+  }
+}
+
+#if 0
 int32_t tqProcessRebReq(STQ* pTq, char* msg) {
   SMqMVRebReq req = {0};
   terrno = TSDB_CODE_SUCCESS;
@@ -754,6 +734,7 @@ int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) {
   terrno = TSDB_CODE_SUCCESS;
   return 0;
 }
+#endif
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
   if (pTask->execType == TASK_EXEC__NONE) return 0;
diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c
index 357917e0ba..809fd7cb06 100644
--- a/source/dnode/vnode/src/tq/tqMetaStore.c
+++ b/source/dnode/vnode/src/tq/tqMetaStore.c
@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #include "vnodeInt.h"
-// TODO:replace by an abstract file layer
 // #include <fcntl.h>
 // #include <string.h>
 // #include <unistd.h>
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index fface9d6c5..1486d18b5c 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -80,6 +80,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       pRsp->msgType = TDMT_VND_SUBMIT_RSP;
       vnodeProcessSubmitReq(pVnode, ptr, pRsp);
       break;
+    case TDMT_VND_MQ_VG_CHANGE:
+      if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
+        // TODO: handle error
+      }
+      break;
+#if 0
     case TDMT_VND_MQ_SET_CONN: {
       if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
         // TODO: handle error
@@ -93,6 +100,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
       }
     } break;
+#endif
     case TDMT_VND_TASK_DEPLOY: {
       if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
@@ -307,4 +315,4 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg
   pRsp->contLen = sizeof(SSubmitRsp);
 
   return 0;
-}
\ No newline at end of file
+}
diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c
index c2c4af18e5..3d3d13fcb6 100644
--- a/source/os/src/osFile.c
+++ b/source/os/src/osFile.c
@@ -38,7 +38,7 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
 #include <sys/file.h>
 
 #if !defined(_TD_DARWIN_64)
-    #include <sys/sendfile.h>
+#include <sys/sendfile.h>
 #endif
 #include <sys/stat.h>
 #include <unistd.h>
@@ -58,9 +58,9 @@ typedef int32_t FileFd;
 
 typedef struct TdFile {
   TdThreadRwlock rwlock;
-  int      refId;
-  FileFd   fd;
-  FILE    *fp;
+  int            refId;
+  FileFd         fd;
+  FILE          *fp;
 } * TdFilePtr, TdFile;
 
 #define FILE_WITH_LOCK 1
@@ -182,7 +182,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
   return 0;
 #else
   struct stat fileStat;
-  int32_t code = stat(path, &fileStat);
+  int32_t     code = stat(path, &fileStat);
   if (code < 0) {
     return code;
   }
@@ -203,7 +203,7 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) {
   return 0;
 #else
   struct stat fileStat;
-  int32_t code = stat(path, &fileStat);
+  int32_t     code = stat(path, &fileStat);
   if (code < 0) {
     return code;
   }
@@ -226,7 +226,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   return NULL;
 #else
-  int fd = -1;
+  int   fd = -1;
   FILE *fp = NULL;
   if (tdFileOptions & TD_FILE_STREAM) {
     char *mode = NULL;
@@ -325,7 +325,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
   int64_t leftbytes = count;
   int64_t readbytes;
   char   *tbuf = (char *)buf;
@@ -365,7 +365,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
   int64_t ret = pread(pFile->fd, buf, count, offset);
 #if FILE_WITH_LOCK
   taosThreadRwlockUnlock(&(pFile->rwlock));
@@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  /*assert(pFile->fd >= 0); // Please check if you have closed the file.*/
 
   int64_t nleft = count;
   int64_t nwritten = 0;
@@ -414,7 +414,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
   int64_t ret = lseek(pFile->fd, (long)offset, whence);
 #if FILE_WITH_LOCK
   taosThreadRwlockUnlock(&(pFile->rwlock));
@@ -429,10 +429,10 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
 
   struct stat fileStat;
-  int32_t code = fstat(pFile->fd, &fileStat);
+  int32_t     code = fstat(pFile->fd, &fileStat);
   if (code < 0) {
     return code;
   }
@@ -456,7 +456,7 @@ int32_t taosLockFile(TdFilePtr pFile) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
 
   return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB);
 #endif
@@ -469,7 +469,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
 
   return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB);
 #endif
@@ -529,7 +529,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
 
   return ftruncate(pFile->fd, l_size);
 #endif
@@ -637,7 +637,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co
   }
   off_t len = count;
   while (len > 0) {
-    char buf[1024 * 16];
+    char  buf[1024 * 16];
     off_t n = sizeof(buf);
     if (len < n) n = len;
     size_t m = fread(buf, 1, n, in_file);
@@ -662,7 +662,7 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
   }
   off_t len = count;
   while (len > 0) {
-    char buf[1024 * 16];
+    char  buf[1024 * 16];
     off_t n = sizeof(buf);
     if (len < n) n = len;
     size_t m = read(sfd, buf, n);
@@ -750,7 +750,7 @@ void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) {
   if (pFile == NULL) {
     return NULL;
   }
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
 
   void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0);
   return ptr;
@@ -811,4 +811,4 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) {
 
 bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); };
 
-#endif // WINDOWS
+#endif  // WINDOWS
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 0181b63380..1c655fc2bf 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -76,7 +76,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
 }
 
 void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) {
-  if (pArray == NULL || pData == NULL) {
+  if (pData == NULL) {
     return NULL;
   }
 
@@ -318,7 +318,6 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) {
   pArray->size = 0;
 }
 
-
 void* taosArrayDestroy(SArray* pArray) {
   if (pArray) {
     taosMemoryFree(pArray->pData);
@@ -445,7 +444,8 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void
 }
 
 SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) {
-  SArray* pArray = taosArrayInit(pSrc->size, pSrc->elemSize);
+  ASSERT(pSrc->elemSize == sizeof(void*));
+  SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*));
   for (int32_t i = 0; i < pSrc->size; i++) {
     void* clone = deepCopy(taosArrayGetP(pSrc, i));
     taosArrayPush(pArray, &clone);
@@ -471,6 +471,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t
   for (int32_t i = 0; i < sz; i++) {
     void* data = taosMemoryCalloc(1, dataSz);
     buf = decode(buf, data);
+    taosArrayPush(*pArray, &data);
   }
   return (void*)buf;
 }
diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim
index f7bd273624..60064b174e 100644
--- a/tests/script/tsim/tmq/basic1.sim
+++ b/tests/script/tsim/tmq/basic1.sim
@@ -33,8 +33,8 @@ endi
 sql connect
 
 $dbNamme = d0
-print =============== create database , vgroup 1
-sql create database $dbNamme vgroups 1
+print =============== create database , vgroup 4
+sql create database $dbNamme vgroups 4
 sql use $dbNamme
 
 print =============== create super table
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index 5ea15a6622..9b627e187c 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -276,7 +276,7 @@ tmq_t* build_consumer() {
   }
   tmq_conf_set(conf, "td.connect.user", "root");
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
-  tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
+  ASSERT(TMQ_CONF_OK == tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName));
   tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
   assert(tmq);
   tmq_conf_destroy(conf);
@@ -437,12 +437,14 @@ void* threadFunc(void* param) {
   pInfo->consumeMsgCnt = parallel_consume(tmq, 1);
   //}
 
+#if 0
   err = tmq_unsubscribe(tmq);
   if (err) {
     printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
     pInfo->consumeMsgCnt = -1;
     return NULL;
   }
+#endif
 
   return NULL;
 }
@@ -485,11 +487,13 @@ int main(int32_t argc, char* argv[]) {
     totalMsgs = parallel_consume(tmq, 0);
   }
 
+#if 0
   err = tmq_unsubscribe(tmq);
   if (err) {
     printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
     exit(-1);
   }
+#endif
 
   if (g_stConfInfo.numOfTopic1) {
     for (int32_t i = 0; i < numOfThreads; i++) {

From f7a63661ac5f592093b4ab3b45aa0e7a14db8a80 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Wed, 20 Apr 2022 22:08:04 +0800
Subject: [PATCH 21/22] fix

---
 source/dnode/mnode/impl/src/mndConsumer.c | 8 ++++----
 tests/test/c/tmqSim.c                     | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 2ce7f0ae5f..7f5df5d356 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -376,7 +376,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
     if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
 
   } else {
-    taosRLockLatch(&pConsumerOld->lock);
+    /*taosRLockLatch(&pConsumerOld->lock);*/
     int32_t status = atomic_load_32(&pConsumerOld->status);
     if (status != MQ_CONSUMER_STATUS__READY) {
       terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
@@ -440,7 +440,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
 
 SUBSCRIBE_OVER:
   if (pConsumerOld) {
-    taosRUnLockLatch(&pConsumerOld->lock);
+    /*taosRUnLockLatch(&pConsumerOld->lock);*/
     mndReleaseConsumer(pMnode, pConsumerOld);
   }
   if (pConsumerNew) {
@@ -628,9 +628,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
 
     // remove from removed topic
     for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) {
-      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i);
       if (strcmp(removedTopic, topic) == 0) {
-        taosArrayRemove(pOldConsumer->rebNewTopics, i);
+        taosArrayRemove(pOldConsumer->rebRemovedTopics, i);
         taosMemoryFree(topic);
         break;
       }
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index 9b627e187c..1b8c0d527d 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -437,7 +437,7 @@ void* threadFunc(void* param) {
   pInfo->consumeMsgCnt = parallel_consume(tmq, 1);
   //}
 
-#if 0
+#if 1
   err = tmq_unsubscribe(tmq);
   if (err) {
     printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
@@ -487,7 +487,7 @@ int main(int32_t argc, char* argv[]) {
     totalMsgs = parallel_consume(tmq, 0);
   }
 
-#if 0
+#if 1
   err = tmq_unsubscribe(tmq);
   if (err) {
     printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));

From ce6459a9f859b7ceb19ae4e9aef13adf3f1fcb7f Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Thu, 21 Apr 2022 01:47:29 +0800
Subject: [PATCH 22/22] fix(tmq): rebalance

---
 source/dnode/mnode/impl/inc/mndOffset.h    |  2 +-
 source/dnode/mnode/impl/src/mndOffset.c    | 18 ++++++++---
 source/dnode/mnode/impl/src/mndSubscribe.c | 36 +++++++++++++++++++---
 source/dnode/vnode/src/tq/tq.c             | 29 +++++++++--------
 4 files changed, 62 insertions(+), 23 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h
index e18eec973f..556bfe6a53 100644
--- a/source/dnode/mnode/impl/inc/mndOffset.h
+++ b/source/dnode/mnode/impl/inc/mndOffset.h
@@ -31,7 +31,7 @@ void          mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset);
 SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset);
 SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw);
 
-int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
+int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
 
 static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
   return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);
diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c
index 31c16e1e1d..dad912b4e6 100644
--- a/source/dnode/mnode/impl/src/mndOffset.c
+++ b/source/dnode/mnode/impl/src/mndOffset.c
@@ -130,8 +130,7 @@ OFFSET_DECODE_OVER:
   return pRow;
 }
 
-int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
-  int32_t code = 0;
+int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
   int32_t sz = taosArrayGetSize(vgs);
   for (int32_t i = 0; i < sz; i++) {
     SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i);
@@ -170,13 +169,22 @@ static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) {
     if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
       return -1;
     }
+    bool          create = false;
     SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
-    ASSERT(pOffsetObj);
+    if (pOffsetObj == NULL) {
+      pOffsetObj = taosMemoryMalloc(sizeof(SMqOffset));
+      memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN);
+      create = true;
+    }
     pOffsetObj->offset = pOffset->offset;
     SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
     sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
     mndTransAppendRedolog(pTrans, pOffsetRaw);
-    mndReleaseOffset(pMnode, pOffsetObj);
+    if (create) {
+      taosMemoryFree(pOffsetObj);
+    } else {
+      mndReleaseOffset(pMnode, pOffsetObj);
+    }
   }
 
   if (mndTransPrepare(pMnode, pTrans) != 0) {
@@ -201,7 +209,7 @@ static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) {
 
 static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) {
   mTrace("offset:%s, perform update action", pOldOffset->key);
-  pOldOffset->offset = pNewOffset->offset;
+  atomic_store_64(&pOldOffset->offset, pNewOffset->offset);
   return 0;
 }
 
diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index 7b2ec83230..f708d4ffc1 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -813,6 +813,20 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
     if (pEpInSub->consumerId == -1) continue;
     ASSERT(pEpInSub->consumerId > 0);
+
+    // push until equal minVg
+    while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) {
+      // iter hash and find one vg
+      pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+      ASSERT(pRemovedIter);
+      pRebVg = (SMqRebOutputVg *)pRemovedIter;
+      // push
+      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pEpInSub->consumerId;
+      taosArrayPush(pOutput->rebVgs, pRebVg);
+    }
+
+#if 0
     /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/
     if (imbCnt < imbConsumerNum) {
       imbCnt++;
@@ -840,13 +854,25 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
         taosArrayPush(pOutput->rebVgs, pRebVg);
       }
     }
+#endif
   }
 
   // 7. handle unassigned vg
   if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) {
-    // if has consumer, vg should be all assigned
-    pRemovedIter = taosHashIterate(pHash, pRemovedIter);
-    ASSERT(pRemovedIter == NULL);
+    // if has consumer, assign all left vg
+    while (1) {
+      pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+      if (pRemovedIter == NULL) break;
+      pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+      ASSERT(pIter);
+      pRebVg = (SMqRebOutputVg *)pRemovedIter;
+      SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+      if (pEpInSub->consumerId == -1) continue;
+      ASSERT(pEpInSub->consumerId > 0);
+      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pEpInSub->consumerId;
+      taosArrayPush(pOutput->rebVgs, pRebVg);
+    }
   } else {
     // if all consumer is removed, put all vg into unassigned
     int64_t             unexistKey = -1;
@@ -992,7 +1018,9 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
 
     // TODO replace assert with error check
     ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
-    ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);
+    // if add more consumer to balanced subscribe,
+    // possibly no vg is changed
+    /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
     ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
 
     if (rebInput.pTopic) {
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 16c5a4752e..c7c8054120 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -355,6 +355,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
          TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
 
   // TODO destroy
+  taosArrayDestroy(rsp.blockData);
+  taosArrayDestroy(rsp.blockDataLen);
   return 0;
 }
 
@@ -599,19 +601,20 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
     taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
     return 0;
   } else {
-    if (req.newConsumerId != -1) {
-      /*taosWLockLatch(&pExec->lock);*/
-      ASSERT(pExec->consumerId == req.oldConsumerId);
-      // TODO handle qmsg and exec modification
-      atomic_store_64(&pExec->consumerId, req.newConsumerId);
-      atomic_add_fetch_32(&pExec->epoch, 1);
-      /*taosWUnLockLatch(&pExec->lock);*/
-      return 0;
-    } else {
-      // TODO
-      /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/
-      return 0;
-    }
+    /*if (req.newConsumerId != -1) {*/
+    /*taosWLockLatch(&pExec->lock);*/
+    ASSERT(pExec->consumerId == req.oldConsumerId);
+    // TODO handle qmsg and exec modification
+    atomic_store_32(&pExec->epoch, -1);
+    atomic_store_64(&pExec->consumerId, req.newConsumerId);
+    atomic_add_fetch_32(&pExec->epoch, 1);
+    /*taosWUnLockLatch(&pExec->lock);*/
+    return 0;
+    /*} else {*/
+    // TODO
+    /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/
+    /*return 0;*/
+    /*}*/
   }
 }