Merge branch '3.0' of github.com:taosdata/TDengine into feature/TD-14761

Jenkinsfile2

@@ -168,13 +168,6 @@ pipeline {
     stages {
         stage('run test') {
             parallel {
-                stage('windows test') {
-                    agent {label " windows11 "}
-                    steps {
-                        pre_test_win()
-                        pre_test_build_win()
-                    }
-                }
                 stage('linux test') {
                     agent{label " slave3_0 || slave15 || slave16 || slave17 "}
                     options { skipDefaultCheckout() } 

example/src/tstream.c

@@ -81,9 +81,9 @@ int32_t create_stream() {
   /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
   /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
-  pRes =
+  pRes = taos_query(
-      taos_query(pConn,
+      pConn,
-                 "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
+      "create stream stream1 trigger window_close into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
       "from tu1 interval(10m)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));

include/common/tdataformat.h

@@ -374,8 +374,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
 typedef struct SDataCol {
   int8_t          type;        // column type
   uint8_t         bitmap : 1;  // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
-  uint8_t         bitmapMode : 1;  // default is 0(2 bits), otherwise 1(1 bit)
+  uint8_t         reserve : 7;
-  uint8_t         reserve : 6;
   int16_t         colId;      // column ID
   int32_t         bytes;      // column data bytes defined
   int32_t         offset;     // data offset in a SDataRow (including the header size)
@@ -387,8 +386,6 @@ typedef struct SDataCol {
   TSKEY           ts;         // only used in last NULL column
 } SDataCol;
 
-
-
 #define isAllRowsNull(pCol) ((pCol)->len == 0)
 #define isAllRowsNone(pCol) ((pCol)->len == 0)
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
@@ -482,7 +479,7 @@ void       tdResetDataCols(SDataCols *pCols);
 int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 SDataCols *tdFreeDataCols(SDataCols *pCols);
-int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer);
+int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer);
 
 // ----------------- K-V data row structure
 /* |<-------------------------------------- len -------------------------------------------->|

include/common/tmsg.h

@@ -672,7 +672,6 @@ typedef struct {
   SArray* pArray;  // Array of SUseDbRsp
 } SUseDbBatchRsp;
 
-
 int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp);
 int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp);
 void    tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp);
@@ -685,7 +684,6 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp
 int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp);
 void    tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp);
 
-
 typedef struct {
   char db[TSDB_DB_FNAME_LEN];
 } SCompactDbReq;
@@ -1554,7 +1552,9 @@ typedef struct SVDropStbReq {
 int32_t tEncodeSVDropStbReq(SCoder* pCoder, const SVDropStbReq* pReq);
 int32_t tDecodeSVDropStbReq(SCoder* pCoder, SVDropStbReq* pReq);
 
+#define TD_CREATE_IF_NOT_EXISTS 0x1
 typedef struct SVCreateTbReq {
+  int32_t     flags;
   tb_uid_t    uid;
   int64_t     ctime;
   const char* name;

include/common/trow.h

@@ -42,7 +42,7 @@ extern "C" {
  * @brief value type
  *  - for data from client input and STSRow in memory, 3 types of value none/null/norm available
  */
-#define TD_VTYPE_NORM 0x00U  // normal val: not none, not null(no need assign value)
+#define TD_VTYPE_NORM 0x00U  // normal val: not none, not null
 #define TD_VTYPE_NULL 0x01U  // null val
 #define TD_VTYPE_NONE 0x02U  // none or unknown/undefined
 #define TD_VTYPE_MAX  0x03U  //
@@ -140,8 +140,6 @@ typedef struct {
   };
   /// row total length
   uint32_t len;
-  /// row version
-  uint64_t ver;
   /// the inline data, maybe a tuple or a k-v tuple
   char data[];
 } STSRow;
@@ -176,7 +174,7 @@ typedef struct {
 #define TD_ROW_DATA(r)     ((r)->data)
 #define TD_ROW_LEN(r)      ((r)->len)
 #define TD_ROW_KEY(r)      ((r)->ts)
-#define TD_ROW_VER(r)      ((r)->ver)
+// #define TD_ROW_VER(r)      ((r)->ver)
 #define TD_ROW_KEY_ADDR(r) (r)
 
 // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
@@ -241,13 +239,13 @@ static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colI
 static FORCE_INLINE bool    tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode);
 bool                        tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode);
 int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
-                             int8_t bitmapMode);
+                             int8_t bitmapMode, bool isMerge);
 static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
                                                   bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset);
 static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
                                                   bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset,
                                                   col_id_t colId);
-int32_t                     tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols);
+int32_t                     tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge);
 
 /**
  * @brief
@@ -718,6 +716,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
       terrno = TSDB_CODE_INVALID_PARA;
       return terrno;
   }
+
   return TSDB_CODE_SUCCESS;
 }
 

include/libs/function/tudf.h

@@ -29,7 +29,11 @@ extern "C" {
 #endif
 
 #define UDF_LISTEN_PIPE_NAME_LEN 32
-#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
+#ifdef _WIN32
+#define UDF_LISTEN_PIPE_NAME_PREFIX "\\\\?\\pipe\\udfd.sock"
+#else
+#define UDF_LISTEN_PIPE_NAME_PREFIX ".udfd.sock."
+#endif
 #define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
 
 //======================================================================================
@@ -129,8 +133,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
 // begin API to UDF writer.
 
 // dynamic lib init and destroy
-typedef int32_t (*TUdfSetupFunc)();
+typedef int32_t (*TUdfInitFunc)();
-typedef int32_t (*TUdfTeardownFunc)();
+typedef int32_t (*TUdfDestroyFunc)();
 
 //TODO: add API to check function arguments type, number etc.
 
@@ -242,7 +246,6 @@ static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRo
   return 0;
 }
 
-typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
 typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
 
 typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);

include/libs/index/index.h

@@ -56,7 +56,8 @@ typedef enum {
   QUERY_LESS_EQUAL,
   QUERY_GREATER_THAN,
   QUERY_GREATER_EQUAL,
-  QUERY_RANGE
+  QUERY_RANGE,
+  QUERY_MAX
 } EIndexQueryType;
 
 /*
@@ -178,8 +179,8 @@ void        indexOptsDestroy(SIndexOpts* opts);
  * @param:
  */
 
-SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType,
+SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName,
-                            const char* colName, int32_t nColName, const char* colVal, int32_t nColVal);
+                            int32_t nColName, const char* colVal, int32_t nColVal);
 void        indexTermDestroy(SIndexTerm* p);
 
 /*

include/libs/scalar/scalar.h

@@ -91,6 +91,8 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
+int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
 #ifdef __cplusplus
 }
 #endif

include/util/taoserror.h

@@ -621,6 +621,14 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x2639)
 #define TSDB_CODE_PAR_INVALID_DROP_STABLE       TAOS_DEF_ERROR_CODE(0, 0x263A)
 #define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE   TAOS_DEF_ERROR_CODE(0, 0x263B)
+#define TSDB_CODE_PAR_DUPLICATED_COLUMN         TAOS_DEF_ERROR_CODE(0, 0x263C)
+#define TSDB_CODE_PAR_INVALID_TAGS_LENGTH       TAOS_DEF_ERROR_CODE(0, 0x263D)
+#define TSDB_CODE_PAR_INVALID_ROW_LENGTH        TAOS_DEF_ERROR_CODE(0, 0x263E)
+#define TSDB_CODE_PAR_INVALID_COLUMNS_NUM       TAOS_DEF_ERROR_CODE(0, 0x263F)
+#define TSDB_CODE_PAR_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x2640)
+#define TSDB_CODE_PAR_INVALID_FIRST_COLUMN      TAOS_DEF_ERROR_CODE(0, 0x2641)
+#define TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN    TAOS_DEF_ERROR_CODE(0, 0x2642)
+#define TSDB_CODE_PAR_INVALID_TAGS_NUM          TAOS_DEF_ERROR_CODE(0, 0x2643)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tdef.h

@@ -29,6 +29,8 @@ extern "C" {
 #define TSKEY_MAX         (INT64_MAX - 1)
 #define TSKEY_INITIAL_VAL TSKEY_MIN
 
+#define TD_VER_MAX UINT64_MAX  // TODO: use the real max version from query handle
+
 // Bytes for each type.
 extern const int32_t TYPE_BYTES[15];
 

source/client/test/clientTests.cpp

@@ -587,15 +587,34 @@ TEST(testCase, projection_query_tables) {
   }
   taos_free_result(pRes);
 
+  pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+  }
+  taos_free_result(pRes);
+
   pRes = taos_query(pConn, "create table tu using st1 tags(1)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
   }
   taos_free_result(pRes);
 
-  for(int32_t i = 0; i < 10000; ++i) {
+  pRes = taos_query(pConn, "create table tu2 using st2 tags(1)");
-    char sql[512] = {0};
+  if (taos_errno(pRes) != 0) {
-    sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
+    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+  }
+  taos_free_result(pRes);
+
+  for(int32_t i = 0; i < 10000000; i += 20) {
+    char sql[1024] = {0};
+    sprintf(sql,
+            "insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+            "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+            "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+            "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)",
+            i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7,
+            i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14,
+            i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19);
     TAOS_RES* p = taos_query(pConn, sql);
     if (taos_errno(p) != 0) {
       printf("failed to insert data, reason:%s\n", taos_errstr(p));
@@ -604,24 +623,44 @@ TEST(testCase, projection_query_tables) {
     taos_free_result(p);
   }
 
-  pRes = taos_query(pConn, "select * from tu");
+  printf("start to insert next table\n");
-  if (taos_errno(pRes) != 0) {
+
-    printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
+  for(int32_t i = 0; i < 10000000; i += 20) {
-    taos_free_result(pRes);
+    char sql[1024] = {0};
-    ASSERT_TRUE(false);
+    sprintf(sql,
+            "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+            "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+            "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+            "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)",
+            i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7,
+            i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14,
+            i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19);
+    TAOS_RES* p = taos_query(pConn, sql);
+    if (taos_errno(p) != 0) {
+      printf("failed to insert data, reason:%s\n", taos_errstr(p));
     }
 
-  TAOS_ROW    pRow = NULL;
+    taos_free_result(p);
-  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-  int32_t     numOfFields = taos_num_fields(pRes);
-
-  char str[512] = {0};
-  while ((pRow = taos_fetch_row(pRes)) != NULL) {
-    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-    printf("%s\n", str);
   }
 
-  taos_free_result(pRes);
+//  pRes = taos_query(pConn, "select * from tu");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
+//    taos_free_result(pRes);
+//    ASSERT_TRUE(false);
+//  }
+
+//  TAOS_ROW    pRow = NULL;
+//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+//  int32_t     numOfFields = taos_num_fields(pRes);
+//
+//  char str[512] = {0};
+//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+//    printf("%s\n", str);
+//  }
+
+//  taos_free_result(pRes);
   taos_close(pConn);
 }
 
@@ -659,7 +698,7 @@ TEST(testCase, agg_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  TAOS_RES* pRes = taos_query(pConn, "use db");
+  TAOS_RES* pRes = taos_query(pConn, "use abc1");
   if (taos_errno(pRes) != 0) {
     printf("failed to use db, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);

source/common/src/tdatablock.c

@@ -363,9 +363,9 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd
 
   for (int32_t i = 0; i < pDest->info.numOfCols; ++i) {
     int32_t mapIndex = i;
-    if (pIndexMap) {
+//    if (pIndexMap) {
-      mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i);
+//      mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i);
-    }
+//    }
 
     SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
     SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex);
@@ -493,12 +493,12 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
     for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) {
       bool isNull = false;
       if (pBlock->pBlockAgg == NULL) {
-        isNull = colDataIsNull(pColData, pBlock->info.rows, j, NULL);
+        isNull = colDataIsNull_s(pColData, pBlock->info.rows);
       } else {
         isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]);
       }
-      char* p = colDataGetData(pColData, j);
 
+      char* p = colDataGetData(pColData, j);
       colDataAppend(pDstCol, j - startIndex, p, isNull);
     }
   }
@@ -1481,8 +1481,8 @@ void blockDebugShowData(const SArray* dataBlocks) {
  *
  * TODO: colId should be set
  */
-int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid,
+int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
-                                        tb_uid_t suid) {
+                                    tb_uid_t uid, tb_uid_t suid) {
   int32_t sz = taosArrayGetSize(pDataBlocks);
   int32_t bufSize = sizeof(SSubmitReq);
   for (int32_t i = 0; i < sz; ++i) {
@@ -1536,7 +1536,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
           case TSDB_DATA_TYPE_TIMESTAMP:
             if (!isStartKey) {
               isStartKey = true;
-              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0);
+              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true,
+                                  0, 0);
             } else {
               tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k);
               break;
@@ -1629,14 +1630,14 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) {
     blkHead->uid = htobe64(pDataBlock->info.uid);
 
     int32_t rows = pDataBlock->info.rows;
-    int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+    /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/
     /*blkHead->dataLen = htonl(rows * maxLen);*/
     blkHead->dataLen = 0;
 
     void*   blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk));
     STSRow* rowData = blockData;
 
-    for (int32_t j = 0; j < pDataBlock->info.rows; j++) {
+    for (int32_t j = 0; j < rows; j++) {
       SRowBuilder rb = {0};
       tdSRowInit(&rb, pTSchema->version);
       tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen);

source/common/src/tmsg.c

@@ -1255,7 +1255,6 @@ int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp)
   return 0;
 }
 
-
 int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
@@ -2146,7 +2145,6 @@ void    tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){
   taosArrayDestroy(pRsp->pArray);
 }
 
-
 int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -3746,6 +3744,7 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
 int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) {
   if (tStartEncode(pCoder) < 0) return -1;
 
+  if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1;
   if (tEncodeI64(pCoder, pReq->uid) < 0) return -1;
   if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1;
 
@@ -3771,6 +3770,7 @@ int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) {
 
   if (tStartDecode(pCoder) < 0) return -1;
 
+  if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1;
   if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1;
   if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1;
 

source/common/src/trow.c

@@ -24,7 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{
                                    },
                                    {
                                        // 1 bit
-                                       TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I,
+                                       TD_VTYPE_NORM_BYTE_I,  // normal
+                                       TD_VTYPE_NULL_BYTE_I,
                                        TD_VTYPE_NULL_BYTE_I,  // padding
                                    }
 
@@ -33,6 +34,24 @@ const uint8_t tdVTypeByte[2][3] = {{
 // declaration
 static uint8_t tdGetBitmapByte(uint8_t byte);
 
+// static void dataColSetNEleNull(SDataCol *pCol, int nEle);
+
+/**
+ * @brief src2 data has more priority than src1
+ *
+ * @param target
+ * @param src1
+ * @param iter1
+ * @param limit1
+ * @param src2
+ * @param iter2
+ * @param limit2
+ * @param tRows
+ * @param update
+ */
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                               int limit2, int tRows, bool update);
+
 // implementation
 /**
  * @brief Compress bitmap bytes comprised of 2-bits to counterpart of 1-bit.
@@ -229,23 +248,23 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) {
 void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) {
   int32_t i = 0, j = 0;
   int32_t nBytes = TD_BITMAP_BYTES(nBits);
-  int32_t nStrictBytes = nBits / 4;
+  int32_t nRoundBytes = nBits / 4;
-  int32_t nPartialBits = nBits - nStrictBytes * 4;
+  int32_t nRemainderBits = nBits - nRoundBytes * 4;
 
-  switch (nPartialBits) {
+  switch (nRemainderBits) {
     case 0:
       // NOTHING TODO
       break;
     case 1: {
-      void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+      void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes);
       *(uint8_t *)lastByte &= 0xC0;
     } break;
     case 2: {
-      void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+      void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes);
       *(uint8_t *)lastByte &= 0xF0;
     } break;
     case 3: {
-      void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+      void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes);
       *(uint8_t *)lastByte &= 0xFC;
     } break;
     default:
@@ -266,10 +285,6 @@ void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) {
   }
 }
 
-// static void dataColSetNEleNull(SDataCol *pCol, int nEle);
-static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
-                               int limit2, int tRows, bool forceSetNull);
-
 static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
@@ -329,7 +344,7 @@ bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode
     if (*((uint8_t *)pBitmap) != vTypeByte) {
       return false;
     }
-    pBitmap = POINTER_SHIFT(pBitmap, 1);
+    pBitmap = POINTER_SHIFT(pBitmap, i);
   }
 
   int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
@@ -411,10 +426,11 @@ STSRow *tdRowDup(STSRow *row) {
  * @param numOfRows
  * @param maxPoints
  * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit)
+ * @param isMerge merge to current row
  * @return int
  */
 int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints,
-                         int8_t bitmapMode) {
+                         int8_t bitmapMode, bool isMerge) {
   TASSERT(pCol != NULL);
 
   // Assume that the columns not specified during insert/upsert mean None.
@@ -430,33 +446,58 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int
       dataColSetNEleNone(pCol, numOfRows, bitmapMode);
     }
   }
-  if (!tdValTypeIsNorm(valType)) {
+  const void *value = val;
+  if (!tdValTypeIsNorm(valType) || !val) {
     // TODO:
     // 1. back compatibility and easy to debug with codes of 2.0 to save NULL values.
     // 2. later on, considering further optimization, don't save Null/None for VarType.
-    val = getNullValue(pCol->type);
+    value = getNullValue(pCol->type);
   }
+  if (!isMerge) {
     if (IS_VAR_DATA_TYPE(pCol->type)) {
       // set offset
       pCol->dataOff[numOfRows] = pCol->len;
       // Copy data
-    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val));
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
       // Update the length
-    pCol->len += varDataTLen(val);
+      pCol->len += varDataTLen(value);
     } else {
       ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
-    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes);
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
       pCol->len += pCol->bytes;
     }
+  } else if (!tdValTypeIsNone(valType)) {
+    if (IS_VAR_DATA_TYPE(pCol->type)) {
+      // keep the last offset
+      // discard the last var data
+      int32_t lastVarLen = varDataTLen(POINTER_SHIFT(pCol->pData, pCol->dataOff[numOfRows]));
+      pCol->len -= lastVarLen;
+      // Copy data
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+      // Update the length
+      pCol->len += varDataTLen(value);
+    } else {
+      ASSERT(pCol->len - TYPE_BYTES[pCol->type] == TYPE_BYTES[pCol->type] * numOfRows);
+      memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), value, pCol->bytes);
+    }
+  }
+
 #ifdef TD_SUPPORT_BITMAP
+  if (!isMerge || !tdValTypeIsNone(valType)) {
     tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode);
+  }
 #endif
   return 0;
 }
 
 // internal
-static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) {
+#if 0
   ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow));
+#endif
+
+  // Multi-Version rows with the same key and different versions supported
+  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) <= TD_ROW_KEY(pRow));
 
   int   rcol = 1;
   int   dcol = 1;
@@ -464,12 +505,14 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
 
   SDataCol *pDataCol = &(pCols->cols[0]);
   ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
-  tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+  tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                       isMerge);
 
   while (dcol < pCols->numOfCols) {
     pDataCol = &(pCols->cols[dcol]);
     if (rcol >= schemaNCols(pSchema)) {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                           isMerge);
       ++dcol;
       continue;
     }
@@ -480,22 +523,26 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
       if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) {
         return terrno;
       }
-      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                           isMerge);
       ++dcol;
       ++rcol;
     } else if (pRowCol->colId < pDataCol->colId) {
       ++rcol;
     } else {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                           isMerge);
       ++dcol;
     }
   }
+#if 0  
   ++pCols->numOfRows;
+#endif
 
   return TSDB_CODE_SUCCESS;
 }
 // internal
-static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) {
   ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow));
 
   int   rcol = 0;
@@ -506,12 +553,14 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
 
   SDataCol *pDataCol = &(pCols->cols[0]);
   ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
-  tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+  tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                       isMerge);
 
   while (dcol < pCols->numOfCols) {
     pDataCol = &(pCols->cols[dcol]);
     if (rcol >= tRowCols || rcol >= tSchemaCols) {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                           isMerge);
       ++dcol;
       continue;
     }
@@ -527,17 +576,21 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
       if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) {
         return terrno;
       }
-      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                           isMerge);
       ++dcol;
       ++rcol;
     } else if (pIdx->colId < pDataCol->colId) {
       ++rcol;
     } else {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+                           isMerge);
       ++dcol;
     }
   }
+#if 0
   ++pCols->numOfRows;
+#endif
 
   return TSDB_CODE_SUCCESS;
 }
@@ -548,20 +601,30 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
  * @param pRow
  * @param pSchema
  * @param pCols
- * @param forceSetNull
  */
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) {
   if (TD_IS_TP_ROW(pRow)) {
-    return tdAppendTpRowToDataCol(pRow, pSchema, pCols);
+    return tdAppendTpRowToDataCol(pRow, pSchema, pCols, isMerge);
   } else if (TD_IS_KV_ROW(pRow)) {
-    return tdAppendKvRowToDataCol(pRow, pSchema, pCols);
+    return tdAppendKvRowToDataCol(pRow, pSchema, pCols, isMerge);
   } else {
     ASSERT(0);
   }
   return TSDB_CODE_SUCCESS;
 }
 
-int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull,
+/**
+ * @brief source data has more priority than target
+ *
+ * @param target
+ * @param source
+ * @param rowsToMerge
+ * @param pOffset
+ * @param update
+ * @param maxVer
+ * @return int
+ */
+int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool update,
                     TDRowVerT maxVer) {
   ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
   ASSERT(target->numOfCols == source->numOfCols);
@@ -576,17 +639,38 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *
   if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) {  // No overlap
     ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
     // TODO: filter the maxVer
-    for (int i = 0; i < rowsToMerge; i++) {
+    TSKEY lastKey = TSKEY_INITIAL_VAL;
+    for (int i = 0; i < rowsToMerge; ++i) {
+      bool merge = false;
       for (int j = 0; j < source->numOfCols; j++) {
         if (source->cols[j].len > 0 || target->cols[j].len > 0) {
           SCellVal sVal = {0};
           if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) {
             TASSERT(0);
           }
+
+          if (j == 0) {
+            if (lastKey == *(TSKEY *)sVal.val) {
+              if (!update) {
+                break;
+              }
+              merge = true;
+            } else if (lastKey != TSKEY_INITIAL_VAL) {
+              ++target->numOfRows;
+            }
+
+            lastKey = *(TSKEY *)sVal.val;
+          }
+          if (i == 0) {
+            (target->cols + j)->bitmap = (source->cols + j)->bitmap;
+          }
+
           tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
-                               target->bitmapMode);
+                               target->bitmapMode, merge);
         }
       }
+    }
+    if (lastKey != TSKEY_INITIAL_VAL) {
       ++target->numOfRows;
     }
     (*pOffset) += rowsToMerge;
@@ -596,7 +680,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *
 
     int iter1 = 0;
     tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows,
-                       pTarget->numOfRows + rowsToMerge, forceSetNull);
+                       pTarget->numOfRows + rowsToMerge, update);
   }
 
   tdFreeDataCols(pTarget);
@@ -607,67 +691,95 @@ _err:
   return -1;
 }
 
-// src2 data has more priority than src1
+static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, bool isMerge) {
+  for (int i = 0; i < src->numOfCols; ++i) {
+    ASSERT(target->cols[i].type == src->cols[i].type);
+    if (src->cols[i].len > 0 || target->cols[i].len > 0) {
+      SCellVal sVal = {0};
+      if (tdGetColDataOfRow(&sVal, src->cols + i, iter, src->bitmapMode) < 0) {
+        TASSERT(0);
+      }
+      if (isMerge) {
+        if (!tdValTypeIsNone(sVal.valType)) {
+          tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+                               target->bitmapMode, isMerge);
+        } else {
+          // Keep the origin value for None
+        }
+      } else {
+        tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+                             target->bitmapMode, isMerge);
+      }
+    }
+  }
+}
+/**
+ * @brief src2 data has more priority than src1
+ *
+ * @param target
+ * @param src1
+ * @param iter1
+ * @param limit1
+ * @param src2
+ * @param iter2
+ * @param limit2
+ * @param tRows
+ * @param update
+ */
 static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
-                               int limit2, int tRows, bool forceSetNull) {
+                               int limit2, int tRows, bool update) {
   tdResetDataCols(target);
+  target->bitmapMode = src1->bitmapMode;
   ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
+  int32_t nRows = 0;
 
-  while (target->numOfRows < tRows) {
+  // TODO: filter the maxVer
+  // TODO: handle the delete function
+  TSKEY lastKey = TSKEY_INITIAL_VAL;
+  while (nRows < tRows) {
     if (*iter1 >= limit1 && *iter2 >= limit2) break;
 
     TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
-    TKEY  tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
+    // TKEY  tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
     TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
     // TKEY  tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
 
-    ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
+    // ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
-    // TODO: filter the maxVer
-    if (key1 < key2) {
-      for (int i = 0; i < src1->numOfCols; ++i) {
-        ASSERT(target->cols[i].type == src1->cols[i].type);
-        if (src1->cols[i].len > 0 || target->cols[i].len > 0) {
-          SCellVal sVal = {0};
-          if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
-            TASSERT(0);
-          }
-          tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
-                               target->bitmapMode);
-        }
-      }
 
+    if (key1 <= key2) {
+      // select key1 if not delete
+      if (update && (lastKey == key1)) {
+        tdAppendValToDataCols(target, src1, *iter1, true);
+      } else if (lastKey != key1) {
+        if (lastKey != TSKEY_INITIAL_VAL) {
           ++target->numOfRows;
+        }
+        tdAppendValToDataCols(target, src1, *iter1, false);
+      }
+      ++nRows;
       ++(*iter1);
-    } else if (key1 >= key2) {
+      lastKey = key1;
-      // TODO: filter the maxVer
+    } else {
-      if ((key1 > key2) || ((key1 == key2) && !TKEY_IS_DELETED(key2))) {
+      // use key2 if not deleted
-        for (int i = 0; i < src2->numOfCols; ++i) {
+      // TODO: handle the delete function
-          SCellVal sVal = {0};
+      if (update && (lastKey == key2)) {
-          ASSERT(target->cols[i].type == src2->cols[i].type);
+        tdAppendValToDataCols(target, src2, *iter2, true);
-          if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) {
+      } else if (lastKey != key2) {
-            TASSERT(0);
+        if (lastKey != TSKEY_INITIAL_VAL) {
-          }
-          if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) {
-            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
-                                 target->bitmapMode);
-          } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
-            if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
-              TASSERT(0);
-            }
-            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
-                                 target->bitmapMode);
-          } else if (target->cols[i].len > 0) {
-            dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode);
-          }
-        }
           ++target->numOfRows;
         }
-
+        tdAppendValToDataCols(target, src2, *iter2, false);
-      ++(*iter2);
-      if (key1 == key2) ++(*iter1);
       }
 
-    ASSERT(target->numOfRows <= target->maxPoints);
+      ++nRows;
+      ++(*iter2);
+      lastKey = key2;
+    }
+
+    ASSERT(target->numOfRows <= target->maxPoints - 1);
+  }
+  if (lastKey != TSKEY_INITIAL_VAL) {
+    ++target->numOfRows;
   }
 }
 
@@ -777,7 +889,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
   pRet->sversion = pDataCols->sversion;
   if (keepData) pRet->numOfRows = pDataCols->numOfRows;
 
-  for (int i = 0; i < pDataCols->numOfCols; i++) {
+  for (int i = 0; i < pDataCols->numOfCols; ++i) {
     pRet->cols[i].type = pDataCols->cols[i].type;
     pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
     pRet->cols[i].colId = pDataCols->cols[i].colId;
@@ -797,8 +909,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
           memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
         }
         if (!TD_COL_ROWS_NORM(pRet->cols + i)) {
-          int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows);
+          memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, TD_BITMAP_BYTES(pDataCols->numOfRows));
-          memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes);
         }
       }
     }

source/common/src/ttypes.c

@@ -557,7 +557,7 @@ static const void *nullValues[] = {
 };
 
 const void *getNullValue(int32_t type) {
-  assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT);
+  assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT);  // TODO: extend the types
   return nullValues[type - 1];
 }
 

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -293,14 +293,10 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, DEFAULT_HANDLE);
-
   dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE);
-
   dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
-  // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
-  // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
 
   // sync integration
   dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);

source/dnode/mnode/impl/inc/mndAuth.h

@@ -31,6 +31,7 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser);
 
 int32_t mndCheckNodeAuth(SUserObj *pOperUser);
 int32_t mndCheckFuncAuth(SUserObj *pOperUser);
+int32_t mndCheckTransAuth(SUserObj *pOperUser);
 
 int32_t mndCheckCreateDbAuth(SUserObj *pOperUser);
 int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb);

source/dnode/mnode/impl/inc/mndDef.h

@@ -601,7 +601,6 @@ typedef struct {
   int32_t        triggerParam;
   int64_t        waterMark;
   char*          sql;
-  char*          logicalPlan;
   char*          physicalPlan;
   SArray*        tasks;  // SArray<SArray<SStreamTask>>
   SSchemaWrapper outputSchema;

source/dnode/mnode/impl/inc/mndStb.h

@@ -31,6 +31,11 @@ int32_t  mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t n
                             int32_t *pRspLen);
 int32_t  mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs);
 
+int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate);
+SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName);
+int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb);
+int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -27,10 +27,14 @@ void     mndCleanupVgroup(SMnode *pMnode);
 SVgObj  *mndAcquireVgroup(SMnode *pMnode, int32_t vgId);
 void     mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
 SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
-int32_t  mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
 SEpSet   mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
 int32_t  mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
 
+int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
+SArray *mndBuildDnodesArray(SMnode *pMnode);
+int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray);
+int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2);
+
 void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

source/dnode/mnode/impl/src/mndAuth.c

@@ -74,10 +74,7 @@ static int32_t mndProcessAuthReq(SNodeMsg *pReq) {
 }
 
 int32_t mndCheckCreateUserAuth(SUserObj *pOperUser) {
-  if (pOperUser->superUser) {
+  if (pOperUser->superUser) return 0;
-    return 0;
-  }
-
   terrno = TSDB_CODE_MND_NO_RIGHTS;
   return -1;
 }
@@ -118,28 +115,25 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb,
 }
 
 int32_t mndCheckDropUserAuth(SUserObj *pOperUser) {
-  if (pOperUser->superUser) {
+  if (pOperUser->superUser) return 0;
-    return 0;
-  }
-
   terrno = TSDB_CODE_MND_NO_RIGHTS;
   return -1;
 }
 
 int32_t mndCheckNodeAuth(SUserObj *pOperUser) {
-  if (pOperUser->superUser) {
+  if (pOperUser->superUser) return 0;
-    return 0;
-  }
-
   terrno = TSDB_CODE_MND_NO_RIGHTS;
   return -1;
 }
 
 int32_t mndCheckFuncAuth(SUserObj *pOperUser) {
-  if (pOperUser->superUser) {
+  if (pOperUser->superUser) return 0;
-    return 0;
+  terrno = TSDB_CODE_MND_NO_RIGHTS;
+  return -1;
 }
 
+int32_t mndCheckTransAuth(SUserObj *pOperUser) {
+  if (pOperUser->superUser) return 0;
   terrno = TSDB_CODE_MND_NO_RIGHTS;
   return -1;
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -261,6 +261,104 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
   sdbRelease(pSdb, pDb);
 }
 
+static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
+                                       bool isRedo) {
+  STransAction action = {0};
+
+  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+  if (pDnode == NULL) return -1;
+  action.epSet = mndGetDnodeEpset(pDnode);
+  mndReleaseDnode(pMnode, pDnode);
+
+  int32_t contLen = 0;
+  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
+  if (pReq == NULL) return -1;
+
+  action.pCont = pReq;
+  action.contLen = contLen;
+  action.msgType = TDMT_DND_CREATE_VNODE;
+  action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+
+  if (isRedo) {
+    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  } else {
+    if (mndTransAppendUndoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
+                                      bool isRedo) {
+  STransAction action = {0};
+
+  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+  if (pDnode == NULL) return -1;
+  action.epSet = mndGetDnodeEpset(pDnode);
+  mndReleaseDnode(pMnode, pDnode);
+
+  int32_t contLen = 0;
+  void   *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
+  if (pReq == NULL) return -1;
+
+  action.pCont = pReq;
+  action.contLen = contLen;
+  action.msgType = TDMT_VND_ALTER_VNODE;
+
+  if (isRedo) {
+    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  } else {
+    if (mndTransAppendUndoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
+                                     bool isRedo) {
+  STransAction action = {0};
+
+  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+  if (pDnode == NULL) return -1;
+  action.epSet = mndGetDnodeEpset(pDnode);
+  mndReleaseDnode(pMnode, pDnode);
+
+  int32_t contLen = 0;
+  void   *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
+  if (pReq == NULL) return -1;
+
+  action.pCont = pReq;
+  action.contLen = contLen;
+  action.msgType = TDMT_DND_DROP_VNODE;
+  action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+
+  if (isRedo) {
+    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  } else {
+    if (mndTransAppendUndoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
 static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) {
   char *pos = strstr(dbName, TS_PATH_DELIMITER);
   if (pos == NULL) {
@@ -278,6 +376,8 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) {
 }
 
 static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
+  terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
+
   if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1;
   if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1;
   if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
@@ -299,10 +399,12 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
   if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1;
   if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1;
+  if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
   if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
-  if (pCfg->strict > pCfg->replications) return -1;
   if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
   if (pCfg->hashMethod != 1) return -1;
+
+  terrno = 0;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -381,24 +483,8 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
     SVgObj *pVgroup = pVgroups + vg;
 
     for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
-      STransAction action = {0};
       SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
-
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) {
-      SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
-      if (pDnode == NULL) return -1;
-      action.epSet = mndGetDnodeEpset(pDnode);
-      mndReleaseDnode(pMnode, pDnode);
-
-      int32_t contLen = 0;
-      void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
-      if (pReq == NULL) return -1;
-
-      action.pCont = pReq;
-      action.contLen = contLen;
-      action.msgType = TDMT_DND_CREATE_VNODE;
-      action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
-      if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-        taosMemoryFree(pReq);
         return -1;
       }
     }
@@ -412,24 +498,8 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
     SVgObj *pVgroup = pVgroups + vg;
 
     for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
-      STransAction action = {0};
       SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
-
+      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, false) != 0) {
-      SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
-      if (pDnode == NULL) return -1;
-      action.epSet = mndGetDnodeEpset(pDnode);
-      mndReleaseDnode(pMnode, pDnode);
-
-      int32_t contLen = 0;
-      void   *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
-      if (pReq == NULL) return -1;
-
-      action.pCont = pReq;
-      action.contLen = contLen;
-      action.msgType = TDMT_DND_DROP_VNODE;
-      action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
-      if (mndTransAppendUndoAction(pTrans, &action) != 0) {
-        taosMemoryFree(pReq);
         return -1;
       }
     }
@@ -482,7 +552,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
   }
 
   if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) {
-    terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
     mError("db:%s, failed to create since %s", pCreate->db, terrstr());
     return -1;
   }
@@ -570,37 +639,37 @@ _OVER:
 static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
 
-  if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) {
+  if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) {
     pDb->cfg.buffer = pAlter->buffer;
     terrno = 0;
   }
 
-  if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) {
+  if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) {
     pDb->cfg.pages = pAlter->pages;
     terrno = 0;
   }
 
-  if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) {
+  if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) {
     pDb->cfg.pageSize = pAlter->pageSize;
     terrno = 0;
   }
 
-  if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {
+  if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {
     pDb->cfg.daysPerFile = pAlter->daysPerFile;
     terrno = 0;
   }
 
-  if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) {
+  if (pAlter->daysToKeep0 > 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) {
     pDb->cfg.daysToKeep0 = pAlter->daysToKeep0;
     terrno = 0;
   }
 
-  if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) {
+  if (pAlter->daysToKeep1 > 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) {
     pDb->cfg.daysToKeep1 = pAlter->daysToKeep1;
     terrno = 0;
   }
 
-  if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) {
+  if (pAlter->daysToKeep2 > 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) {
     pDb->cfg.daysToKeep2 = pAlter->daysToKeep2;
     terrno = 0;
   }
@@ -625,8 +694,9 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
     terrno = 0;
   }
 
-  if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) {
+  if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) {
     pDb->cfg.replications = pAlter->replications;
+    pDb->vgVersion++;
     terrno = 0;
   }
 
@@ -651,28 +721,49 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
   return 0;
 }
 
-static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
+static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
+  if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
     for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
-    STransAction action = {0};
       SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
-
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) {
-    SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
-    if (pDnode == NULL) return -1;
-    action.epSet = mndGetDnodeEpset(pDnode);
-    mndReleaseDnode(pMnode, pDnode);
-
-    int32_t contLen = 0;
-    void   *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
-    if (pReq == NULL) return -1;
-
-    action.pCont = pReq;
-    action.contLen = contLen;
-    action.msgType = TDMT_VND_ALTER_VNODE;
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
         return -1;
       }
     }
+  } else {
+    SVgObj newVgroup = {0};
+    memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
+    if (newVgroup.replica < pDb->cfg.replications) {
+      mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
+            pVgroup->vnodeGid[0].dnodeId);
+
+      if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) {
+        mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr());
+        return -1;
+      }
+      newVgroup.replica = pDb->cfg.replications;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1;
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
+    } else {
+      mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
+
+      SVnodeGid del1 = {0};
+      SVnodeGid del2 = {0};
+      if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1, &del2) != 0) {
+        mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr());
+        return -1;
+      }
+      newVgroup.replica = pDb->cfg.replications;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1;
+      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
+      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
+    }
+
+    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
+    if (pVgRaw == NULL) return -1;
+    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
+    if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1;
+  }
 
   return 0;
 }
@@ -680,6 +771,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj
 static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdb   *pSdb = pMnode->pSdb;
   void   *pIter = NULL;
+  SArray *pArray = mndBuildDnodesArray(pMnode);
 
   while (1) {
     SVgObj *pVgroup = NULL;
@@ -687,9 +779,10 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
     if (pIter == NULL) break;
 
     if (pVgroup->dbUid == pNew->uid) {
-      if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) {
+      if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) {
         sdbCancelFetch(pSdb, pIter);
         sdbRelease(pSdb, pVgroup);
+        taosArrayDestroy(pArray);
         return -1;
       }
     }
@@ -697,6 +790,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
     sdbRelease(pSdb, pVgroup);
   }
 
+  taosArrayDestroy(pArray);
   return 0;
 }
 
@@ -726,6 +820,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) {
   SDbObj     *pDb = NULL;
   SUserObj   *pUser = NULL;
   SAlterDbReq alterReq = {0};
+  SDbObj      dbObj = {0};
 
   if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -749,15 +844,17 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
-  SDbObj dbObj = {0};
   memcpy(&dbObj, pDb, sizeof(SDbObj));
-  dbObj.cfg.numOfRetensions = 0;
+  if (dbObj.cfg.pRetensions != NULL) {
-  dbObj.cfg.pRetensions = NULL;
+    dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions);
+    if (dbObj.cfg.pRetensions == NULL) goto _OVER;
+  }
 
   code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq);
-  if (code != 0) {
+  if (code != 0) goto _OVER;
-    goto _OVER;
+
-  }
+  code = mndCheckDbCfg(pMnode, &dbObj.cfg);
+  if (code != 0) goto _OVER;
 
   dbObj.cfgVersion++;
   dbObj.updateTime = taosGetTimestampMs();
@@ -771,6 +868,7 @@ _OVER:
 
   mndReleaseDb(pMnode, pDb);
   mndReleaseUser(pMnode, pUser);
+  taosArrayDestroy(dbObj.cfg.pRetensions);
 
   return code;
 }
@@ -899,24 +997,8 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
 
 static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
   for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
-    STransAction action = {0};
     SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
-
+    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) {
-    SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
-    if (pDnode == NULL) return -1;
-    action.epSet = mndGetDnodeEpset(pDnode);
-    mndReleaseDnode(pMnode, pDnode);
-
-    int32_t contLen = 0;
-    void   *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
-    if (pReq == NULL) return -1;
-
-    action.pCont = pReq;
-    action.contLen = contLen;
-    action.msgType = TDMT_DND_DROP_VNODE;
-    action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
       return -1;
     }
   }
@@ -988,6 +1070,17 @@ static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) {
   /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
   if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER;
 
+  SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser);
+  if (pUser != NULL) {
+    pUser->authVersion++;
+    SSdbRaw *pCommitRaw = mndUserActionEncode(pUser);
+    if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
+      mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
+      goto _OVER;
+    }
+    sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
+  }
+
   int32_t rspLen = 0;
   void   *pRsp = NULL;
   if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto _OVER;

source/dnode/mnode/impl/src/mndSma.c

@@ -21,7 +21,7 @@
 #include "mndInfoSchema.h"
 #include "mndMnode.h"
 #include "mndShow.h"
-#include "mndStb.c"
+#include "mndStb.h"
 #include "mndStream.h"
 #include "mndTrans.h"
 #include "mndUser.h"
@@ -419,7 +419,6 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
   streamObj.fixedSinkVgId = smaObj.dstVgId;
   streamObj.smaId = smaObj.uid;
   /*streamObj.physicalPlan = "";*/
-  streamObj.logicalPlan = "not implemented";
 
   int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg);

source/dnode/mnode/impl/src/mndStb.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "mndStb.h"
 #include "mndAuth.h"
 #include "mndDb.h"
@@ -342,7 +343,7 @@ void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) {
   sdbRelease(pSdb, pStb);
 }
 
-static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) {
+SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) {
   SName name = {0};
   tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
@@ -463,7 +464,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb,
   return pHead;
 }
 
-static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
+int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
   if (pCreate->igExists < 0 || pCreate->igExists > 1) {
     terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
     return -1;
@@ -634,91 +635,96 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) {
   return NULL;
 }
 
-static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
+int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) {
-  SStbObj stbObj = {0};
+  memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN);
-  memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
+  memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
-  memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN);
+  pDst->createdTime = taosGetTimestampMs();
-  stbObj.createdTime = taosGetTimestampMs();
+  pDst->updateTime = pDst->createdTime;
-  stbObj.updateTime = stbObj.createdTime;
+  pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
-  stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+  pDst->dbUid = pDb->uid;
-  stbObj.dbUid = pDb->uid;
+  pDst->version = 1;
-  stbObj.version = 1;
+  pDst->nextColId = 1;
-  stbObj.nextColId = 1;
+  pDst->xFilesFactor = pCreate->xFilesFactor;
-  stbObj.xFilesFactor = pCreate->xFilesFactor;
+  pDst->delay = pCreate->delay;
-  stbObj.delay = pCreate->delay;
+  pDst->ttl = pCreate->ttl;
-  stbObj.ttl = pCreate->ttl;
+  pDst->numOfColumns = pCreate->numOfColumns;
-  stbObj.numOfColumns = pCreate->numOfColumns;
+  pDst->numOfTags = pCreate->numOfTags;
-  stbObj.numOfTags = pCreate->numOfTags;
+  pDst->commentLen = pCreate->commentLen;
-  stbObj.commentLen = pCreate->commentLen;
+  if (pDst->commentLen > 0) {
-  if (stbObj.commentLen > 0) {
+    pDst->comment = taosMemoryCalloc(pDst->commentLen, 1);
-    stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1);
+    if (pDst->comment == NULL) {
-    if (stbObj.comment == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen);
+    memcpy(pDst->comment, pCreate->comment, pDst->commentLen);
   }
 
-  stbObj.ast1Len = pCreate->ast1Len;
+  pDst->ast1Len = pCreate->ast1Len;
-  if (stbObj.ast1Len > 0) {
+  if (pDst->ast1Len > 0) {
-    stbObj.pAst1 = taosMemoryCalloc(stbObj.ast1Len, 1);
+    pDst->pAst1 = taosMemoryCalloc(pDst->ast1Len, 1);
-    if (stbObj.pAst1 == NULL) {
+    if (pDst->pAst1 == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    memcpy(stbObj.pAst1, pCreate->pAst1, stbObj.ast1Len);
+    memcpy(pDst->pAst1, pCreate->pAst1, pDst->ast1Len);
   }
 
-  stbObj.ast2Len = pCreate->ast2Len;
+  pDst->ast2Len = pCreate->ast2Len;
-  if (stbObj.ast2Len > 0) {
+  if (pDst->ast2Len > 0) {
-    stbObj.pAst2 = taosMemoryCalloc(stbObj.ast2Len, 1);
+    pDst->pAst2 = taosMemoryCalloc(pDst->ast2Len, 1);
-    if (stbObj.pAst2 == NULL) {
+    if (pDst->pAst2 == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    memcpy(stbObj.pAst2, pCreate->pAst2, stbObj.ast2Len);
+    memcpy(pDst->pAst2, pCreate->pAst2, pDst->ast2Len);
   }
 
-  stbObj.pColumns = taosMemoryCalloc(1, stbObj.numOfColumns * sizeof(SSchema));
+  pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema));
-  stbObj.pTags = taosMemoryCalloc(1, stbObj.numOfTags * sizeof(SSchema));
+  pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema));
-  if (stbObj.pColumns == NULL || stbObj.pTags == NULL) {
+  if (pDst->pColumns == NULL || pDst->pTags == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  for (int32_t i = 0; i < stbObj.numOfColumns; ++i) {
+  for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
     SField  *pField = taosArrayGet(pCreate->pColumns, i);
-    SSchema *pSchema = &stbObj.pColumns[i];
+    SSchema *pSchema = &pDst->pColumns[i];
     pSchema->type = pField->type;
     pSchema->bytes = pField->bytes;
     pSchema->flags = pField->flags;
     memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
-    pSchema->colId = stbObj.nextColId;
+    pSchema->colId = pDst->nextColId;
-    stbObj.nextColId++;
+    pDst->nextColId++;
   }
 
-  for (int32_t i = 0; i < stbObj.numOfTags; ++i) {
+  for (int32_t i = 0; i < pDst->numOfTags; ++i) {
     SField  *pField = taosArrayGet(pCreate->pTags, i);
-    SSchema *pSchema = &stbObj.pTags[i];
+    SSchema *pSchema = &pDst->pTags[i];
     pSchema->type = pField->type;
     pSchema->bytes = pField->bytes;
     memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
-    pSchema->colId = stbObj.nextColId;
+    pSchema->colId = pDst->nextColId;
-    stbObj.nextColId++;
+    pDst->nextColId++;
+  }
+  return 0;
 }
 
+static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
+  SStbObj stbObj = {0};
+
   int32_t code = -1;
+
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg);
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
-  mndTransSetDbInfo(pTrans, pDb);
 
-  if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
+  if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) {
-  if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
+    goto _OVER;
-  if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
+  }
-  if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
+
-  if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
+  if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
+
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;
@@ -727,6 +733,15 @@ _OVER:
   mndTransDrop(pTrans);
   return code;
 }
+int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+  mndTransSetDbInfo(pTrans, pDb);
+  if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
+  if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
+  if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
+  if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
+  if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
+  return 0;
+}
 
 static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) {
   SMnode        *pMnode = pReq->pNode;

source/dnode/mnode/impl/src/mndStream.c

@@ -290,6 +290,83 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
   return 0;
 }
 
+static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) {
+  SStbObj  *pStb = NULL;
+  SDbObj   *pDb = NULL;
+  SUserObj *pUser = NULL;
+
+  SMCreateStbReq createReq = {0};
+  tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN);
+  createReq.numOfColumns = pStream->outputSchema.nCols;
+  createReq.numOfTags = 1;  // group id
+  createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
+  // build fields
+  taosArraySetSize(createReq.pColumns, createReq.numOfColumns);
+  for (int32_t i = 0; i < createReq.numOfColumns; i++) {
+    SField *pField = taosArrayGet(createReq.pColumns, i);
+    tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN);
+    pField->flags = pStream->outputSchema.pSchema[i].flags;
+    pField->type = pStream->outputSchema.pSchema[i].type;
+    pField->bytes = pStream->outputSchema.pSchema[i].bytes;
+  }
+  createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
+  taosArraySetSize(createReq.pTags, 1);
+  // build tags
+  SField *pField = taosArrayGet(createReq.pTags, 0);
+  strcpy(pField->name, "group_id");
+  pField->type = TSDB_DATA_TYPE_UBIGINT;
+  pField->flags = 0;
+  pField->bytes = 8;
+
+  if (mndCheckCreateStbReq(&createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _OVER;
+  }
+
+  pStb = mndAcquireStb(pMnode, createReq.name);
+  if (pStb != NULL) {
+    terrno = TSDB_CODE_MND_STB_ALREADY_EXIST;
+    goto _OVER;
+  }
+
+  pDb = mndAcquireDbByStb(pMnode, createReq.name);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    goto _OVER;
+  }
+
+  pUser = mndAcquireUser(pMnode, user);
+  if (pUser == NULL) {
+    goto _OVER;
+  }
+
+  if (mndCheckWriteAuth(pUser, pDb) != 0) {
+    goto _OVER;
+  }
+
+  int32_t numOfStbs = -1;
+  mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs);
+  if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) {
+    terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB;
+    goto _OVER;
+  }
+
+  SStbObj stbObj = {0};
+
+  if (mndBuildStbFromReq(pMnode, &stbObj, &createReq, pDb) != 0) {
+    goto _OVER;
+  }
+
+  if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
+
+  return 0;
+_OVER:
+  mndReleaseStb(pMnode, pStb);
+  mndReleaseDb(pMnode, pDb);
+  mndReleaseUser(pMnode, pUser);
+  return -1;
+}
+
 static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) {
   mDebug("stream:%s to create", pCreate->name);
   SStreamObj streamObj = {0};
@@ -307,11 +384,10 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   streamObj.fixedSinkVgId = 0;
   streamObj.smaId = 0;
   /*streamObj.physicalPlan = "";*/
-  streamObj.logicalPlan = "not implemented";
   streamObj.trigger = pCreate->triggerType;
   streamObj.waterMark = pCreate->watermark;
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
   if (pTrans == NULL) {
     mError("stream:%s, failed to create since %s", pCreate->name, terrstr());
     return -1;
@@ -324,6 +400,12 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
     return -1;
   }
 
+  if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) {
+    mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);

source/dnode/mnode/impl/src/mndTrans.c

@@ -1336,8 +1336,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
-  if (!pUser->superUser) {
+  if (mndCheckTransAuth(pUser) != 0) {
-    terrno = TSDB_CODE_MND_NO_RIGHTS;
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndUser.c

@@ -106,6 +106,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
   SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER)
   SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
   SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
 
@@ -161,6 +162,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER)
 
   int32_t numOfReadDbs = 0;
   int32_t numOfWriteDbs = 0;
@@ -711,7 +713,8 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) {
   sdbCancelFetch(pSdb, pIter);
 }
 
-int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) {
+int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
+                                int32_t *pRspLen) {
   SUserAuthBatchRsp batchRsp = {0};
   batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp));
   if (batchRsp.pArray == NULL) {
@@ -740,7 +743,6 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_
       goto _OVER;
     }
 
-
     taosArrayPush(batchRsp.pArray, &rsp);
     mndReleaseUser(pMnode, pUser);
   }
@@ -776,6 +778,3 @@ _OVER:
   tFreeSUserAuthBatchRsp(&batchRsp);
   return code;
 }
-
-
-

source/dnode/mnode/impl/src/mndVgroup.c

@@ -370,7 +370,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2
   return true;
 }
 
-static SArray *mndBuildDnodesArray(SMnode *pMnode) {
+SArray *mndBuildDnodesArray(SMnode *pMnode) {
   SSdb   *pSdb = pMnode->pSdb;
   int32_t numOfDnodes = mndGetDnodeSize(pMnode);
 
@@ -421,7 +421,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr
       pVgid->role = TAOS_SYNC_STATE_FOLLOWER;
     }
 
-    mDebug("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId);
+    mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId);
     pDnode->numOfVnodes++;
   }
 
@@ -440,11 +440,9 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
   }
 
   pArray = mndBuildDnodesArray(pMnode);
-  if (pArray == NULL) {
+  if (pArray == NULL) goto _OVER;
-    goto _OVER;
-  }
 
-  mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray),
+  mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray),
         pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications);
 
   int32_t  allocedVgroups = 0;
@@ -483,7 +481,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
   *ppVgroups = pVgroups;
   code = 0;
 
-  mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications);
+  mInfo("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications);
 
 _OVER:
   if (code != 0) taosMemoryFree(pVgroups);
@@ -491,6 +489,88 @@ _OVER:
   return code;
 }
 
+int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
+  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
+  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, i);
+    mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes);
+  }
+
+  int32_t maxPos = 1;
+  for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, d);
+
+    bool used = false;
+    for (int32_t vn = 0; vn < maxPos; ++vn) {
+      if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) {
+        used = true;
+        break;
+      }
+    }
+    if (used) continue;
+
+    if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      return -1;
+    }
+
+    SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos];
+    pVgid->dnodeId = pDnode->id;
+    pVgid->role = TAOS_SYNC_STATE_FOLLOWER;
+    pDnode->numOfVnodes++;
+
+    mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId);
+    maxPos++;
+    if (maxPos == 3) return 0;
+  }
+
+  terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+  return -1;
+}
+
+int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) {
+  int32_t removedNum = 0;
+
+  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
+  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, i);
+    mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes);
+  }
+
+  for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, d);
+
+    for (int32_t vn = 0; vn < TSDB_MAX_REPLICA; ++vn) {
+      SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
+      if (pVgid->dnodeId == pDnode->id) {
+        if (removedNum == 0) *del1 = *pVgid;
+        if (removedNum == 1) *del2 = *pVgid;
+
+        mInfo("db:%s, vgId:%d, vn:%d dnode:%d is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
+        memset(pVgid, 0, sizeof(SVnodeGid));
+        removedNum++;
+        pDnode->numOfVnodes--;
+
+        if (removedNum == 2) goto _OVER;
+      }
+    }
+  }
+
+_OVER:
+  if (removedNum != 2) return -1;
+
+  for (int32_t vn = 1; vn < TSDB_MAX_REPLICA; ++vn) {
+    SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
+    if (pVgid->dnodeId != 0) {
+      memcpy(&pVgroup->vnodeGid[0], pVgid, sizeof(SVnodeGid));
+      memset(pVgid, 0, sizeof(SVnodeGid));
+    }
+  }
+
+  mInfo("db:%s, vgId:%d, dnode:%d is keeped", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId);
+  return 0;
+}
+
 SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) {
   SEpSet epset = {0};
 

source/dnode/mnode/impl/test/db/db.cpp

@@ -73,14 +73,19 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
   {
     SAlterDbReq alterdbReq = {0};
     strcpy(alterdbReq.db, "1.d1");
+
     alterdbReq.buffer = 12;
-    alterdbReq.daysToKeep0 = 300;
+    alterdbReq.pageSize = -1;
-    alterdbReq.daysToKeep1 = 400;
+    alterdbReq.pages = -1;
-    alterdbReq.daysToKeep2 = 500;
+    alterdbReq.daysPerFile = -1;
+    alterdbReq.daysToKeep0 = -1;
+    alterdbReq.daysToKeep1 = -1;
+    alterdbReq.daysToKeep2 = -1;
     alterdbReq.fsyncPeriod = 4000;
     alterdbReq.walLevel = 2;
-    alterdbReq.strict = 2;
+    alterdbReq.strict = 1;
     alterdbReq.cacheLastRow = 1;
+    alterdbReq.replications = 1;
 
     int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
     void*   pReq = rpcMallocCont(contLen);

source/dnode/mnode/impl/test/dnode/mdnode.cpp

@@ -266,4 +266,96 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) {
   taosMsleep(1300);
   test.SendShowReq(TSDB_MGMT_TABLE_DNODE, "dnodes", "");
   EXPECT_EQ(test.GetShowRows(), 4);
+
+  // alter replica
+#if 0
+  {
+    SCreateDbReq createReq = {0};
+    strcpy(createReq.db, "1.d2");
+    createReq.numOfVgroups = 2;
+    createReq.buffer = -1;
+    createReq.pageSize = -1;
+    createReq.pages = -1;
+    createReq.daysPerFile = 1000;
+    createReq.daysToKeep0 = 3650;
+    createReq.daysToKeep1 = 3650;
+    createReq.daysToKeep2 = 3650;
+    createReq.minRows = 100;
+    createReq.maxRows = 4096;
+    createReq.fsyncPeriod = 3000;
+    createReq.walLevel = 1;
+    createReq.precision = 0;
+    createReq.compression = 2;
+    createReq.replications = 1;
+    createReq.strict = 1;
+    createReq.cacheLastRow = 0;
+    createReq.ignoreExist = 1;
+    createReq.numOfStables = 0;
+    createReq.numOfRetensions = 0;
+
+    int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
+    void*   pReq = rpcMallocCont(contLen);
+    tSerializeSCreateDbReq(pReq, contLen, &createReq);
+
+    SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
+    ASSERT_NE(pRsp, nullptr);
+    ASSERT_EQ(pRsp->code, 0);
+
+    test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", "");
+    EXPECT_EQ(test.GetShowRows(), 3);
+  }
+
+  {
+    SAlterDbReq alterdbReq = {0};
+    strcpy(alterdbReq.db, "1.d2");
+
+    alterdbReq.buffer = 12;
+    alterdbReq.pageSize = -1;
+    alterdbReq.pages = -1;
+    alterdbReq.daysPerFile = -1;
+    alterdbReq.daysToKeep0 = -1;
+    alterdbReq.daysToKeep1 = -1;
+    alterdbReq.daysToKeep2 = -1;
+    alterdbReq.fsyncPeriod = 4000;
+    alterdbReq.walLevel = 2;
+    alterdbReq.strict = 1;
+    alterdbReq.cacheLastRow = 1;
+    alterdbReq.replications = 3;
+
+    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
+    void*   pReq = rpcMallocCont(contLen);
+    tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
+
+    SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
+    ASSERT_NE(pRsp, nullptr);
+    ASSERT_EQ(pRsp->code, 0);
+  }
+
+  {
+    SAlterDbReq alterdbReq = {0};
+    strcpy(alterdbReq.db, "1.d2");
+
+    alterdbReq.buffer = 12;
+    alterdbReq.pageSize = -1;
+    alterdbReq.pages = -1;
+    alterdbReq.daysPerFile = -1;
+    alterdbReq.daysToKeep0 = -1;
+    alterdbReq.daysToKeep1 = -1;
+    alterdbReq.daysToKeep2 = -1;
+    alterdbReq.fsyncPeriod = 4000;
+    alterdbReq.walLevel = 2;
+    alterdbReq.strict = 1;
+    alterdbReq.cacheLastRow = 1;
+    alterdbReq.replications = 1;
+
+    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
+    void*   pReq = rpcMallocCont(contLen);
+    tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
+
+    SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
+    ASSERT_NE(pRsp, nullptr);
+    ASSERT_EQ(pRsp->code, 0);
+  }
+
+#endif
 }

source/dnode/vnode/inc/vnode.h

@@ -126,7 +126,7 @@ int     tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList
 int     tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
 int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReadHandle *pHandle);
-int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows,
+int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t* pUid, int32_t *pNumOfRows,
                             int16_t *pNumOfCols);
 
 // need to reposition

source/dnode/vnode/src/inc/tsdb.h

@@ -273,7 +273,8 @@ typedef enum {
 
 typedef struct {
   uint8_t  last : 1;
-  uint8_t  blkVer : 7;
+  uint8_t  hasDupKey : 1;  // 0: no dup TS key, 1: has dup TS key(since supporting Multi-Version)
+  uint8_t  blkVer : 6;
   uint8_t  numOfSubBlocks;
   col_id_t numOfCols;    // not including timestamp column
   uint32_t len;          // data block length
@@ -324,9 +325,8 @@ typedef struct {
 typedef struct {
   int16_t  colId;
   uint16_t type : 6;
-  uint16_t blen : 10;   // bitmap length(TODO: full UT for the bitmap compress of various data input)
+  uint16_t blen : 10;  // 0 no bitmap if all rows are NORM, > 0 bitmap length
-  uint32_t bitmap : 1;  // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
+  uint32_t len;        // data length + bitmap length
-  uint32_t len : 31;    // data length + bitmap length
   uint32_t offset;
 } SBlockColV0;
 

source/dnode/vnode/src/meta/metaTable.c

@@ -196,7 +196,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
   int8_t      type;
   int64_t     ctime;
   tb_uid_t    suid;
-  int         c, ret;
+  int         c = 0, ret;
 
   // search & delete the name idx
   tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);

source/dnode/vnode/src/tq/tq.c

@@ -161,7 +161,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
       tqReadHandleSetMsg(pReader, pReq, 0);
       while (tqNextDataBlock(pReader)) {
         SSDataBlock block = {0};
-        if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
+        if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
                                 &block.info.numOfCols) < 0) {
           ASSERT(0);
         }
@@ -540,7 +540,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
         tqReadHandleSetMsg(pReader, pCont, 0);
         while (tqNextDataBlock(pReader)) {
           SSDataBlock block = {0};
-          if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
+          if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
                                   &block.info.numOfCols) < 0) {
             ASSERT(0);
           }

source/dnode/vnode/src/tq/tqRead.c

@@ -84,10 +84,12 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
   return false;
 }
 
-int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows,
+int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, int32_t* pNumOfRows,
                             int16_t* pNumOfCols) {
   /*int32_t         sversion = pHandle->pBlock->sversion;*/
   // TODO set to real sversion
+  *pUid = 0;
+
   int32_t sversion = 0;
   if (pHandle->sver != sversion) {
     pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
@@ -169,7 +171,10 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
   tdSTSRowIterInit(&iter, pTschema);
   STSRow* row;
   int32_t curRow = 0;
+
   tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
+  *pUid = pHandle->msgIter.uid;  // set the uid of table for submit block
+
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
     tdSTSRowIterReset(&iter, row);
     // get all wanted col of that block

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -943,16 +943,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
                                                &(pAggrBlkCol->numOfNull));
 
       if (pAggrBlkCol->numOfNull == 0) {
-        TD_SET_COL_ROWS_NORM(pBlockCol);
+        pBlockCol->blen = 0;
       } else {
-        TD_SET_COL_ROWS_MISC(pBlockCol);
+        pBlockCol->blen = 1;
       }
       ++nColsOfBlockSma;
     } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) {
       // check if all rows normal
-      TD_SET_COL_ROWS_NORM(pBlockCol);
+      pBlockCol->blen = 0;
     } else {
-      TD_SET_COL_ROWS_MISC(pBlockCol);
+      pBlockCol->blen = 1;
     }
 
     ++nColsNotAllNull;
@@ -985,7 +985,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
 #ifdef TD_SUPPORT_BITMAP
     int32_t tBitmaps = 0;
     int32_t tBitmapsLen = 0;
-    if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
+    if ((ncol != 0) && (pBlockCol->blen > 0)) {
       tBitmaps = isSuper ? sBitmaps : nBitmaps;
     }
 #endif
@@ -1330,13 +1330,15 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
                                       TSKEY maxKey, int maxRows, int8_t update) {
   TSKEY     key1 = INT64_MAX;
   TSKEY     key2 = INT64_MAX;
+  TSKEY     lastKey = TSKEY_INITIAL_VAL;
   STSchema *pSchema = NULL;
 
   ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey);
   tdResetDataCols(pTarget);
 
   pTarget->bitmapMode = pDataCols->bitmapMode;
-
+  // TODO: filter Multi-Version
+  // TODO: support delete function
   while (true) {
     key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
     STSRow *row = tsdbNextIterRow(pCommitIter->pIter);
@@ -1349,6 +1351,9 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
     if (key1 == INT64_MAX && key2 == INT64_MAX) break;
 
     if (key1 < key2) {
+      if (lastKey != TSKEY_INITIAL_VAL) {
+        ++pTarget->numOfRows;
+      }
       for (int i = 0; i < pDataCols->numOfCols; ++i) {
         // TODO: dataColAppendVal may fail
         SCellVal sVal = {0};
@@ -1356,10 +1361,10 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
           TASSERT(0);
         }
         tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints,
-                             pTarget->bitmapMode);
+                             pTarget->bitmapMode, false);
       }
 
-      ++pTarget->numOfRows;
+      lastKey = key1;
       ++(*iter);
     } else if (key1 > key2) {
       if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
@@ -1367,7 +1372,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         ASSERT(pSchema != NULL);
       }
 
-      tdAppendSTSRowToDataCol(row, pSchema, pTarget);
+      if (key2 == lastKey) {
+        if (TD_SUPPORT_UPDATE(update)) {
+          tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
+        }
+      } else {
+        if (lastKey != TSKEY_INITIAL_VAL) {
+          ++pTarget->numOfRows;
+        }
+        tdAppendSTSRowToDataCol(row, pSchema, pTarget, false);
+        lastKey = key2;
+      }
 
       tSkipListIterNext(pCommitIter->pIter);
     } else {
@@ -1397,6 +1412,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
       ++(*iter);
       tSkipListIterNext(pCommitIter->pIter);
 #endif
+
+      if(lastKey != key1) {
+        lastKey = key1;
+        ++pTarget->numOfRows;
+      }
+
       // copy disk data
       for (int i = 0; i < pDataCols->numOfCols; ++i) {
         SCellVal sVal = {0};
@@ -1405,7 +1426,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         }
         // TODO: tdAppendValToDataCol may fail
         tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints,
-                             pTarget->bitmapMode);
+                             pTarget->bitmapMode, false);
       }
 
       if (TD_SUPPORT_UPDATE(update)) {
@@ -1416,26 +1437,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         }
 
         // TODO: merge with Multi-Version
-        STSRow *curRow = row;
+        tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
-
+      }
       ++(*iter);
       tSkipListIterNext(pCommitIter->pIter);
-        STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter);
-
-        if (key2 < TD_ROW_KEY(nextRow)) {
-          tdAppendSTSRowToDataCol(row, pSchema, pTarget);
-        } else {
-          tdAppendSTSRowToDataCol(row, pSchema, pTarget);
     }
-        // TODO: merge with Multi-Version
+
-      } else {
+    if (pTarget->numOfRows >= (maxRows - 1)) break;
+  }
+
+  if (lastKey != TSKEY_INITIAL_VAL) {
     ++pTarget->numOfRows;
-        ++(*iter);
-        tSkipListIterNext(pCommitIter->pIter);
-      }
-    }
-
-    if (pTarget->numOfRows >= maxRows) break;
   }
 }
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -20,7 +20,7 @@ static void     tsdbFreeTbData(STbData *pTbData);
 static char    *tsdbGetTsTupleKey(const void *data);
 static int      tsdbTbDataComp(const void *arg1, const void *arg2);
 static char    *tsdbTbDataGetUid(const void *arg);
-static int      tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row);
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge);
 
 int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) {
   STsdbMemTable *pMemTable;
@@ -85,11 +85,16 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   STSchema *pSchema = NULL;
   TSKEY     rowKey = 0;
   TSKEY     fKey = 0;
+  // only fetch lastKey from mem data as file data not used in this function actually
+  TSKEY      lastKey = TSKEY_INITIAL_VAL;
   bool       isRowDel = false;
   int        filterIter = 0;
   STSRow    *row = NULL;
   SMergeInfo mInfo;
 
+  // TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
+  // query handle)
+
   if (pMergeInfo == NULL) pMergeInfo = &mInfo;
 
   memset(pMergeInfo, 0, sizeof(*pMergeInfo));
@@ -111,7 +116,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   } else {
     fKey = tdGetKey(filterKeys[filterIter]);
   }
-
+  // 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
+  // 2. rowKey - would dup since Multi-Version supported
   while (true) {
     if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
 
@@ -125,12 +131,14 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
       } else {
         fKey = tdGetKey(filterKeys[filterIter]);
       }
+#if 0
     } else if (fKey > rowKey) {
       if (isRowDel) {
         pMergeInfo->rowsDeleteFailed++;
       } else {
         if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
         if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+
         pMergeInfo->rowsInserted++;
         pMergeInfo->nOperations++;
         pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
@@ -185,6 +193,94 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
         fKey = tdGetKey(filterKeys[filterIter]);
       }
     }
+#endif
+#if 1
+    } else if (fKey > rowKey) {
+      if (isRowDel) {
+        // TODO: support delete function
+        pMergeInfo->rowsDeleteFailed++;
+      } else {
+        if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+
+        if (lastKey != rowKey) {
+          pMergeInfo->rowsInserted++;
+          pMergeInfo->nOperations++;
+          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
+          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
+          if (pCols) {
+            if (lastKey != TSKEY_INITIAL_VAL) {
+              ++pCols->numOfRows;
+            }
+            tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false);
+          }
+          lastKey = rowKey;
+        } else {
+          if (keepDup) {
+            tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true);
+          } else {
+            // discard
+          }
+        }
+      }
+
+      tSkipListIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = TD_ROW_KEY(row);
+        isRowDel = TD_ROW_IS_DELETED(row);
+      }
+    } else {           // fkey == rowKey
+      if (isRowDel) {  // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
+        ASSERT(!keepDup);
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+        pMergeInfo->rowsDeleteSucceed++;
+        pMergeInfo->nOperations++;
+        tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false);
+      } else {
+        if (keepDup) {
+          if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+          if (lastKey != rowKey) {
+            pMergeInfo->rowsUpdated++;
+            pMergeInfo->nOperations++;
+            pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
+            pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
+            lastKey = rowKey;
+            ++pCols->numOfRows;
+            tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false);
+          } else {
+            tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true);
+          }
+        } else {
+          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
+          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
+        }
+      }
+
+      tSkipListIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = TD_ROW_KEY(row);
+        isRowDel = TD_ROW_IS_DELETED(row);
+      }
+
+      filterIter++;
+      if (filterIter >= nFilterKeys) {
+        fKey = INT64_MAX;
+      } else {
+        fKey = tdGetKey(filterKeys[filterIter]);
+      }
+    }
+#endif
+  }
+  if (lastKey != TSKEY_INITIAL_VAL) {
+    ++pCols->numOfRows;
   }
 
   return 0;
@@ -254,9 +350,12 @@ static STbData *tsdbNewTbData(tb_uid_t uid) {
   pTbData->keyMin = TSKEY_MAX;
   pTbData->keyMax = TSKEY_MIN;
   pTbData->nrows = 0;
-
+#if 0
   pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY,
                                    tsdbGetTsTupleKey);
+#endif
+  pTbData->pData =
+      tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, tsdbGetTsTupleKey);
   if (pTbData->pData == NULL) {
     taosMemoryFree(pTbData);
     return NULL;
@@ -291,7 +390,7 @@ static char *tsdbTbDataGetUid(const void *arg) {
   STbData *pTbData = (STbData *)arg;
   return (char *)(&(pTbData->uid));
 }
-static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row) {
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge) {
   if (pCols) {
     if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
       *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, TD_ROW_SVER(row));
@@ -301,7 +400,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema *
       }
     }
 
-    tdAppendSTSRowToDataCol(row, *ppSchema, pCols);
+    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
   }
 
   return 0;

source/dnode/vnode/src/tsdb/tsdbMemTable2.c

@@ -18,7 +18,8 @@
 typedef struct SMemTable          SMemTable;
 typedef struct SMemData           SMemData;
 typedef struct SMemSkipList       SMemSkipList;
-typedef struct SMemSkipListCfg SMemSkipListCfg;
+typedef struct SMemSkipListNode   SMemSkipListNode;
+typedef struct SMemSkipListCurosr SMemSkipListCurosr;
 
 struct SMemTable {
   STsdb     *pTsdb;
@@ -32,15 +33,17 @@ struct SMemTable {
   SMemData **pBuckets;
 };
 
-struct SMemSkipListCfg {
+struct SMemSkipListNode {
-  int8_t  maxLevel;
+  int8_t            level;
-  int32_t nKey;
+  SMemSkipListNode *forwards[];
-  int32_t nData;
 };
 
 struct SMemSkipList {
-  int8_t   level;
   uint32_t         seed;
+  int8_t           maxLevel;
+  int8_t           level;
+  int32_t          size;
+  SMemSkipListNode pHead[];
 };
 
 struct SMemData {
@@ -55,6 +58,20 @@ struct SMemData {
   SMemSkipList sl;
 };
 
+struct SMemSkipListCurosr {
+  SMemSkipList     *pSl;
+  SMemSkipListNode *pNodeC;
+};
+
+#define SL_NODE_SIZE(l)        (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
+#define SL_NODE_HALF_SIZE(l)   (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l))
+#define SL_NODE_FORWARD(n, l)  ((n)->forwards[l])
+#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
+#define SL_NODE_DATA(n)        (&SL_NODE_BACKWARD(n, (n)->level))
+
+#define SL_HEAD_NODE(sl) ((sl)->pHead)
+#define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel))
+
 // SMemTable
 int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
   SMemTable *pMemTb = NULL;
@@ -76,6 +93,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
   pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets));
   if (pMemTb->pBuckets == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pMemTb);
     return -1;
   }
 
@@ -83,8 +101,121 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
   return 0;
 }
 
-int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) {
+int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) {
-  // TODO
+  if (pMemTb) {
+    // loop to destroy the contents (todo)
+    taosMemoryFree(pMemTb->pBuckets);
+    taosMemoryFree(pMemTb);
+  }
+  return 0;
+}
+
+int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) {
+  SMemData          *pMemData;
+  STsdb             *pTsdb = pMemTb->pTsdb;
+  SVnode            *pVnode = pTsdb->pVnode;
+  SVBufPool         *pPool = pVnode->inUse;
+  int32_t            hash;
+  int32_t            tlen;
+  uint8_t            buf[16];
+  int32_t            rlen;
+  const uint8_t     *p;
+  SMemSkipListNode  *pSlNode;
+  const STSRow      *pTSRow;
+  SMemSkipListCurosr slc = {0};
+
+  // search hash
+  hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket;
+  for (pMemData = pMemTb->pBuckets[hash]; pMemData; pMemData = pMemData->pHashNext) {
+    if (pMemData->suid == pSubmitBlk->suid && pMemData->uid == pSubmitBlk->uid) break;
+  }
+
+  // create pMemData if need
+  if (pMemData == NULL) {
+    int8_t            maxLevel = pVnode->config.tsdbCfg.slLevel;
+    int32_t           tsize = sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2;
+    SMemSkipListNode *pHead, *pTail;
+
+    pMemData = vnodeBufPoolMalloc(pPool, tsize);
+    if (pMemData == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    pMemData->pHashNext = NULL;
+    pMemData->suid = pSubmitBlk->suid;
+    pMemData->uid = pSubmitBlk->uid;
+    pMemData->minKey = TSKEY_MAX;
+    pMemData->maxKey = TSKEY_MIN;
+    pMemData->minVer = -1;
+    pMemData->maxVer = -1;
+    pMemData->nRows = 0;
+    pMemData->sl.seed = taosRand();
+    pMemData->sl.maxLevel = maxLevel;
+    pMemData->sl.level = 0;
+    pMemData->sl.size = 0;
+    pHead = SL_HEAD_NODE(&pMemData->sl);
+    pTail = SL_TAIL_NODE(&pMemData->sl);
+    pHead->level = maxLevel;
+    pTail->level = maxLevel;
+    for (int iLevel = 0; iLevel < maxLevel; iLevel++) {
+      SL_NODE_FORWARD(pHead, iLevel) = pTail;
+      SL_NODE_FORWARD(pTail, iLevel) = pHead;
+    }
+
+    // add to MemTable
+    hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket;
+    pMemData->pHashNext = pMemTb->pBuckets[hash];
+    pMemTb->pBuckets[hash] = pMemData;
+    pMemTb->nHash++;
+  }
+
+// loop to insert data to skiplist
+#if 0
+  tsdbMemSkipListCursorOpen(&slc, &pMemData->sl);
+  p = pSubmitBlk->pData;
+  for (;;) {
+    if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break;
+
+    const uint8_t *pt = p;
+    p = tGetBinary(p, &pTSRow, &rlen);
+
+    // check the row (todo)
+
+    // move the cursor to position to write (todo)
+    int32_t c;
+    tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c);
+    ASSERT(c);
+
+    // encode row
+    int8_t  level = tsdbMemSkipListRandLevel(&pMemData->sl);
+    int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (p - pt);
+    pSlNode = vnodeBufPoolMalloc(pPool, tsize);
+    pSlNode->level = level;
+
+    uint8_t *pData = SL_NODE_DATA(pSlNode);
+    *(int64_t *)pData = version;
+    pData += sizeof(version);
+    memcpy(pData, pt, p - pt);
+
+    // insert row
+    tsdbMemSkipListCursorPut(&slc, pSlNode);
+
+    // update status
+    if (pTSRow->ts < pMemData->minKey) pMemData->minKey = pTSRow->ts;
+    if (pTSRow->ts > pMemData->maxKey) pMemData->maxKey = pTSRow->ts;
+  }
+  tsdbMemSkipListCursorClose(&slc);
+#endif
+
+  if (pMemData->minVer == -1) pMemData->minVer = version;
+  if (pMemData->maxVer == -1 || pMemData->maxVer < version) pMemData->maxVer = version;
+
+  if (pMemTb->minKey < pMemData->minKey) pMemTb->minKey = pMemData->minKey;
+  if (pMemTb->maxKey < pMemData->maxKey) pMemTb->maxKey = pMemData->maxKey;
+  if (pMemTb->minVer == -1) pMemTb->minVer = version;
+  if (pMemTb->maxVer == -1 || pMemTb->maxVer < version) pMemTb->maxVer = version;
+
   return 0;
 }
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -163,7 +163,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2);
 // static void*   destroyTableCheckInfo(SArray* pTableCheckInfo);
 static bool tsdbGetExternalRow(tsdbReaderT pHandle);
 
-static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions);
+static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions);
 
 static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
   pBlockLoadInfo->slot = -1;
@@ -351,36 +351,43 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData
               pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
   }
 }
-#if 0
+#if 1
 int nQUERY = 0;
 #endif
-static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) {
+static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) {
   if (vnodeIsRollup(pVnode)) {
     int level = 0;
-#if 1
+#if 0
     int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
     for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
       SRetention* pRetention = retentions + i;
       if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) {
         break;
       }
+      ++level;
     }
 #endif
-#if 0
+#if 1
-    ++nQUERY;
+    switch ((nQUERY++) % 3) {
-    if(nQUERY%3 == 0) {
+      case 0:
-      level = 2;
-    } else if(nQUERY%2 == 0) {
-      level = 1;
-    } else {
         level = 0;
+        break;
+      case 1:
+        level = 1;
+        break;
+      default:
+        level = 2;
+        break;
     }
 #endif
     if (level == TSDB_RETENTION_L0) {
+      tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
       return VND_RSMA0(pVnode);
     } else if (level == TSDB_RETENTION_L1) {
+      tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
       return VND_RSMA1(pVnode);
     } else {
+      tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
       return VND_RSMA2(pVnode);
     }
   }
@@ -393,7 +400,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
     goto _end;
   }
 
-  STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions);
+  STsdb* pTsdb = getTsdbByRetentions(pVnode, pReadHandle, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions);
 
   pReadHandle->order = pCond->order;
   pReadHandle->pTsdb = pTsdb;
@@ -803,12 +810,16 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
   tSkipListDestroyIter(pCheckInfo->iiter);
 }
 
-static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
+static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) {
   STSRow *rmem = NULL, *rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     if (node != NULL) {
       rmem = (STSRow*)SL_GET_NODE_DATA(node);
+      // TODO: filter max version
+      // if (TD_ROW_VER(rmem) > maxVer) {
+      //   rmem = NULL;
+      // }
     }
   }
 
@@ -816,6 +827,10 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     if (node != NULL) {
       rimem = (STSRow*)SL_GET_NODE_DATA(node);
+      // TODO: filter max version
+      // if (TD_ROW_VER(rimem) > maxVer) {
+      //   rimem = NULL;
+      // }
     }
   }
 
@@ -837,6 +852,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
   TSKEY r2 = TD_ROW_KEY(rimem);
 
   if (r1 == r2) {
+#if 0
     if (update == TD_ROW_DISCARD_UPDATE) {
       pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
       tSkipListIterNext(pCheckInfo->iter);
@@ -846,6 +862,13 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
     } else {
       pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
     }
+#endif
+    if (TD_SUPPORT_UPDATE(update)) {
+      pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
+    } else {
+      pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
+      tSkipListIterNext(pCheckInfo->iter);
+    }
     return r1;
   } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) {
     pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM;
@@ -856,12 +879,18 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
   }
 }
 
-static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow) {
+
+static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow, TDRowVerT maxVer) {
   STSRow *rmem = NULL, *rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     if (node != NULL) {
       rmem = (STSRow*)SL_GET_NODE_DATA(node);
+#if 0 // TODO: skiplist refactor
+      if (TD_ROW_VER(rmem) > maxVer) {
+        rmem = NULL;
+      }
+#endif
     }
   }
 
@@ -869,6 +898,11 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     if (node != NULL) {
       rimem = (STSRow*)SL_GET_NODE_DATA(node);
+#if 0 // TODO: skiplist refactor
+      if (TD_ROW_VER(rimem) > maxVer) {
+        rimem = NULL;
+      }
+#endif
     }
   }
 
@@ -890,6 +924,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
   TSKEY r2 = TD_ROW_KEY(rimem);
 
   if (r1 == r2) {
+#if 0
     if (update == TD_ROW_DISCARD_UPDATE) {
       tSkipListIterNext(pCheckInfo->iter);
       pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
@@ -903,6 +938,16 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
       *extraRow = rimem;
       return rmem;
     }
+#endif
+    if (TD_SUPPORT_UPDATE(update)) {
+      pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
+      *extraRow = rimem;
+      return rmem;
+    } else {
+      tSkipListIterNext(pCheckInfo->iter);
+      pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
+      return rimem;
+    }
   } else {
     if (ASCENDING_TRAVERSE(order)) {
       if (r1 < r2) {
@@ -973,7 +1018,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) {
     initTableMemIterator(pHandle, pCheckInfo);
   }
 
-  STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL);
+  STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL, TD_VER_MAX);
   if (row == NULL) {
     return false;
   }
@@ -1250,7 +1295,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
   /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo);
   assert(cur->pos >= 0 && cur->pos <= binfo.rows);
 
-  key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update);
+  key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update, TD_VER_MAX);
 
   if (key != TSKEY_INITIAL_VAL) {
     tsdbDebug("%p key in mem:%" PRId64 ", %s", pTsdbReadHandle, key, pTsdbReadHandle->idStr);
@@ -1497,10 +1542,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
             TASSERT(0);
           }
 
-          if (sVal.valType == TD_VTYPE_NULL) {
+          if (sVal.valType == TD_VTYPE_NORM) {
-            colDataAppendNULL(pColInfo, rowIndex);
-          } else {
             colDataAppend(pColInfo, rowIndex, sVal.val, false);
+          } else {
+            colDataAppendNULL(pColInfo, rowIndex);
           }
         }
       } else {  // handle the var-string
@@ -1513,10 +1558,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
             TASSERT(0);
           }
 
-          if (sVal.valType == TD_VTYPE_NULL) {
+          if (sVal.valType == TD_VTYPE_NORM) {
-            colDataAppendNULL(pColInfo, rowIndex);
-          } else {
             colDataAppend(pColInfo, rowIndex, sVal.val, false);
+          } else {
+            colDataAppendNULL(pColInfo, rowIndex);
           }
         }
       }
@@ -1541,11 +1586,26 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
   return numOfRows + num;
 }
 
-// TODO fix bug for reverse copy data problem
+/**
-// Note: row1 always has high priority
+ * @brief  // TODO fix bug for reverse copy data problem
-static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1,
+ *        Note: row1 always has high priority
+ *
+ * @param pTsdbReadHandle
+ * @param capacity
+ * @param curRow
+ * @param row1
+ * @param row2
+ * @param numOfCols
+ * @param uid
+ * @param pSchema1
+ * @param pSchema2
+ * @param update
+ * @param lastRowKey
+ * @return int32_t The quantity of rows appended
+ */
+static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t* curRow, STSRow* row1,
                                   STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2,
-                               bool forceSetNull) {
+                                  bool update, TSKEY* lastRowKey) {
 #if 1
   STSchema* pSchema;
   STSRow*   row;
@@ -1557,12 +1617,17 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
   bool     isChosenRowDataRow;
   int32_t  chosen_itr;
   SCellVal sVal = {0};
+  TSKEY    rowKey = TSKEY_INITIAL_VAL;
+  int32_t  nResult = 0;
+  int32_t  mergeOption = 0;  // 0 discard 1 overwrite 2 merge
 
   // the schema version info is embeded in STSRow
   int32_t numOfColsOfRow1 = 0;
 
   if (pSchema1 == NULL) {
-    pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
+    // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
+    // TODO: use the real schemaVersion
+    pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
   }
 
 #ifdef TD_DEBUG_PRINT_ROW
@@ -1579,7 +1644,9 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
   if (row2) {
     isRow2DataRow = TD_IS_TP_ROW(row2);
     if (pSchema2 == NULL) {
-      pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
+      // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
+      // TODO: use the real schemaVersion
+      pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
     }
     if (isRow2DataRow) {
       numOfColsOfRow2 = schemaNCols(pSchema2);
@@ -1610,19 +1677,19 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
       colIdOfRow2 = tdKvRowColIdAt(row2, k);
     }
 
-    if (colIdOfRow1 == colIdOfRow2) {
+    if (colIdOfRow1 < colIdOfRow2) { // the most probability
       if (colIdOfRow1 < pColInfo->info.colId) {
-        j++;
+        ++j;
-        k++;
         continue;
       }
       row = row1;
       pSchema = pSchema1;
       isChosenRowDataRow = isRow1DataRow;
       chosen_itr = j;
-    } else if (colIdOfRow1 < colIdOfRow2) {
+    } else if (colIdOfRow1 == colIdOfRow2) {
       if (colIdOfRow1 < pColInfo->info.colId) {
-        j++;
+        ++j;
+        ++k;
         continue;
       }
       row = row1;
@@ -1631,7 +1698,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
       chosen_itr = j;
     } else {
       if (colIdOfRow2 < pColInfo->info.colId) {
-        k++;
+        ++k;
         continue;
       }
       row = row2;
@@ -1639,16 +1706,48 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
       chosen_itr = k;
       isChosenRowDataRow = isRow2DataRow;
     }
+
     if (isChosenRowDataRow) {
       colId = pSchema->columns[chosen_itr].colId;
       offset = pSchema->columns[chosen_itr].offset;
       // TODO: use STSRowIter
       tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr - 1, &sVal);
+      if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        rowKey = *(TSKEY*)sVal.val;
+        if (rowKey != *lastRowKey) {
+          mergeOption = 1;
+          if (*lastRowKey != TSKEY_INITIAL_VAL) {
+            ++(*curRow);
+          }
+          ++nResult;
+        } else if (update){
+          mergeOption = 2;
+        } else {
+          mergeOption = 0;
+          break;
+        }
+
+        *lastRowKey = rowKey;
+      }
     } else {
       // TODO: use STSRowIter
       if (chosen_itr == 0) {
         colId = PRIMARYKEY_TIMESTAMP_COL_ID;
         tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal);
+        rowKey = *(TSKEY*)sVal.val;
+        if (rowKey != *lastRowKey) {
+          mergeOption = 1;
+          if (*lastRowKey != TSKEY_INITIAL_VAL) {
+            ++(*curRow);
+          }
+          ++nResult;
+        } else if(update) {
+          mergeOption = 2;
+        } else {
+          mergeOption = 0;
+          break;
+        }
+        *lastRowKey = rowKey;
       } else {
         SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1);
         colId = pColIdx->colId;
@@ -1657,35 +1756,46 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
       }
     }
 
+    ASSERT(rowKey != TSKEY_INITIAL_VAL);
+
     if (colId == pColInfo->info.colId) {
       if (tdValTypeIsNorm(sVal.valType)) {
-        colDataAppend(pColInfo, numOfRows, sVal.val, false);
+        colDataAppend(pColInfo, *curRow, sVal.val, false);
-      } else if (forceSetNull) {
+      } else if (tdValTypeIsNull(sVal.valType)) {
-        colDataAppend(pColInfo, numOfRows, NULL, true);
+        colDataAppend(pColInfo, *curRow, NULL, true);
+      } else if (tdValTypeIsNone(sVal.valType)) {
+        // TODO: Set null if nothing append for this row
+        if (mergeOption == 1) {
+          colDataAppend(pColInfo, *curRow, NULL, true);
+        }
+      } else {
+        ASSERT(0);
       }
 
-      i++;
+      ++i;
 
       if (row == row1) {
-        j++;
+        ++j;
       } else {
-        k++;
+        ++k;
       }
     } else {
-      if (forceSetNull) {
+      if (mergeOption == 1) {
-        colDataAppend(pColInfo, numOfRows, NULL, true);
+        colDataAppend(pColInfo, *curRow, NULL, true);
       }
-      i++;
+      ++i;
     }
   }
 
-  if (forceSetNull) {
+  if (mergeOption == 1) {
     while (i < numOfCols) {  // the remain columns are all null data
       SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
-      colDataAppend(pColInfo, numOfRows, NULL, true);
+      colDataAppend(pColInfo, *curRow, NULL, true);
-      i++;
+      ++i;
     }
   }
+
+  return nResult;
 #endif
 }
 
@@ -1859,6 +1969,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 
   // compared with the data from in-memory buffer, to generate the correct timestamp array list
   int32_t numOfRows = 0;
+  int32_t curRow = 0;
 
   int16_t   rv1 = -1;
   int16_t   rv2 = -1;
@@ -1874,9 +1985,11 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
     return;
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
+    TSKEY          lastRowKey = TSKEY_INITIAL_VAL;
+
     do {
       STSRow* row2 = NULL;
-      STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2);
+      STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2, TD_VER_MAX);
       if (row1 == NULL) {
         break;
       }
@@ -1905,9 +2018,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           rv2 = TD_ROW_SVER(row2);
         }
 
-        mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols,
+        numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
-                           pCheckInfo->tableId, pSchema1, pSchema2, true);
+                           pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey);
-        numOfRows += 1;
+        // numOfRows += 1;
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = key;
         }
@@ -1918,6 +2031,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 
         moveToNextRowInMem(pCheckInfo);
       } else if (key == tsArray[pos]) {  // data in buffer has the same timestamp of data in file block, ignore it
+#if 0
         if (pCfg->update) {
           if (pCfg->update == TD_ROW_PARTIAL_UPDATE) {
             doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos);
@@ -1933,7 +2047,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 
           bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE;
           mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols,
-                             pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull);
+                             pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull, &lastRowKey);
           numOfRows += 1;
           if (cur->win.skey == TSKEY_INITIAL_VAL) {
             cur->win.skey = key;
@@ -1943,6 +2057,35 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           cur->lastKey = key + step;
           cur->mixBlock = true;
 
+          moveToNextRowInMem(pCheckInfo);
+          pos += step;
+        } else {
+          moveToNextRowInMem(pCheckInfo);
+        }
+#endif
+        if (TD_SUPPORT_UPDATE(pCfg->update)) {
+          doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
+
+          if (rv1 != TD_ROW_SVER(row1)) {
+            //            pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
+            rv1 = TD_ROW_SVER(row1);
+          }
+          if (row2 && rv2 != TD_ROW_SVER(row2)) {
+            //            pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
+            rv2 = TD_ROW_SVER(row2);
+          }
+
+          numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+                             pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey);
+          // ++numOfRows;
+          if (cur->win.skey == TSKEY_INITIAL_VAL) {
+            cur->win.skey = key;
+          }
+
+          cur->win.ekey = key;
+          cur->lastKey = key + step;
+          cur->mixBlock = true;
+
           moveToNextRowInMem(pCheckInfo);
           pos += step;
         } else {
@@ -1958,11 +2101,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         assert(end != -1);
 
         if (tsArray[end] == key) {  // the value of key in cache equals to the end timestamp value, ignore it
+#if 0
           if (pCfg->update == TD_ROW_DISCARD_UPDATE) {
             moveToNextRowInMem(pCheckInfo);
           } else {
             end -= step;
           }
+#endif
+          if (!TD_SUPPORT_UPDATE(pCfg->update)) {
+            moveToNextRowInMem(pCheckInfo);
+          } else {
+            end -= step;
+          }
         }
 
         int32_t qstart = 0, qend = 0;
@@ -2572,6 +2722,7 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) {
 static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win,
                                  STsdbReadHandle* pTsdbReadHandle) {
   int       numOfRows = 0;
+  int       curRows = 0;
   int32_t   numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
   STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
   win->skey = TSKEY_INITIAL_VAL;
@@ -2579,9 +2730,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
   int64_t   st = taosGetTimestampUs();
   int16_t   rv = -1;
   STSchema* pSchema = NULL;
+  TSKEY     lastRowKey = TSKEY_INITIAL_VAL;
+
 
   do {
-    STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL);
+    STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL, TD_VER_MAX);
     if (row == NULL) {
       break;
     }
@@ -2604,16 +2757,18 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
       pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0);
       rv = TD_ROW_SVER(row);
     }
-    mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema,
+    numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema,
-                       NULL, true);
+                       NULL, pCfg->update, &lastRowKey);
 
-    if (++numOfRows >= maxRowsToRead) {
+    if (numOfRows >= maxRowsToRead) {
       moveToNextRowInMem(pCheckInfo);
       break;
     }
 
   } while (moveToNextRowInMem(pCheckInfo));
 
+  taosMemoryFreeClear(pSchema); // free the STSChema
+
   assert(numOfRows <= maxRowsToRead);
 
   // if the buffer is not full in case of descending order query, move the data in the front of the buffer
@@ -2731,6 +2886,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
   STSRow* pRow = NULL;
   TSKEY   key = TSKEY_INITIAL_VAL;
   int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
+  TSKEY   lastRowKey = TSKEY_INITIAL_VAL;
+  int32_t curRow = 0;
 
   if (++pTsdbReadHandle->activeIndex < numOfTables) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
@@ -2738,8 +2895,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
     //    if (ret != TSDB_CODE_SUCCESS) {
     //      return false;
     //    }
-    mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId,
+    mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, pRow, NULL, numOfCols, pCheckInfo->tableId,
-                       NULL, NULL, true);
+                       NULL, NULL, true, &lastRowKey);
     taosMemoryFreeClear(pRow);
 
     // update the last key value

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -246,6 +246,12 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
   return 0;
 }
 
+static FORCE_INLINE void tsdbSwapDataCols(SDataCols *pDest, SDataCols *pSrc) {
+  SDataCol *pCols = pDest->cols;
+  memcpy(pDest, pSrc, sizeof(SDataCols));
+  pSrc->cols = pCols;
+}
+
 int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   ASSERT(pBlock->numOfSubBlocks > 0);
   STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo);
@@ -266,17 +272,29 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
     if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
     // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
-                        update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
+                        TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
       return -1;
   }
+  // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line
+  if (pBlock->numOfSubBlocks == 1) {
+    tdResetDataCols(pReadh->pDCols[1]);
+    pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode;
+    if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL,
+                        TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) {
+      return -1;
+    }
+    tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]);
+    ASSERT(pReadh->pDCols[0]->bitmapMode != 0);
+  }
 
-  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
   ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
   ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
 
   return 0;
 }
 
+// TODO: filter by Multi-Version
 int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds,
                           bool mergeBitmap) {
   ASSERT(pBlock->numOfSubBlocks > 0);
@@ -297,9 +315,21 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
     if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
     // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
-                        update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
+                        TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
       return -1;
   }
+  // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line
+  if (pBlock->numOfSubBlocks == 1) {
+    tdResetDataCols(pReadh->pDCols[1]);
+    pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode;
+    if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL,
+                        TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) {
+      return -1;
+    }
+    tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]);
+    ASSERT(pReadh->pDCols[0]->bitmapMode != 0);
+  }
+
 
   if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) {
     for (int i = 0; i < numOfColsIds; ++i) {
@@ -312,7 +342,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
     }
   }
 
-  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
   ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
   ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
 
@@ -586,7 +616,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       tcolId = pBlockCol->colId;
       toffset = tsdbGetBlockColOffset(pBlockCol);
       tlen = pBlockCol->len;
-      pDataCol->bitmap = pBlockCol->bitmap;
+      pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0;
     } else {
       ASSERT(pDataCol->colId == tcolId);
       TD_SET_COL_ROWS_NORM(pDataCol);
@@ -594,17 +624,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
 
     // int32_t tBitmaps = 0;
     int32_t tLenBitmap = 0;
-    if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
+    if ((dcol != 0) && (pBlockCol->blen > 0)) {
       tLenBitmap = nBitmaps;
-#if 0
-      if (IS_VAR_DATA_TYPE(pDataCol->type)) {
-        tBitmaps = nBitmaps;
-        tLenBitmap = tBitmaps;
-      } else {
-        tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
-        tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
-      }
-#endif
     }
 
     if (tcolId == pDataCol->colId) {
@@ -623,15 +644,15 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       }
 
       if (dcol != 0) {
-        ccol++;
+        ++ccol;
       }
-      dcol++;
+      ++dcol;
     } else if (tcolId < pDataCol->colId) {
-      ccol++;
+      ++ccol;
     } else {
       // Set current column as NULL and forward
       dataColReset(pDataCol);
-      dcol++;
+      ++dcol;
     }
   }
 
@@ -754,8 +775,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
 
     if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {  // load the key row
       blockCol.colId = colId;
-      TD_SET_COL_ROWS_NORM(&blockCol);  // default is NORM for the primary key column
+      blockCol.blen = 0;  // default is NORM for the primary key column
-      blockCol.blen = 0;
       blockCol.len = pBlock->keyLen;
       blockCol.type = pDataCol->type;
       blockCol.offset = TSDB_KEY_COL_OFFSET;
@@ -785,7 +805,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
       ASSERT(pBlockCol->colId == pDataCol->colId);
     }
     // set the bitmap
-    pDataCol->bitmap = pBlockCol->bitmap;
+    pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0;
 
     if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1;
   }
@@ -803,17 +823,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   // int32_t tBitmaps = 0;
   int32_t tLenBitmap = 0;
 
-  if (!TD_COL_ROWS_NORM(pBlockCol)) {
+  if (pBlockCol->blen) {
     tLenBitmap = nBitmaps;
-#if 0
-    if (IS_VAR_DATA_TYPE(pDataCol->type)) {
-      tBitmaps = nBitmaps;
-      tLenBitmap = tBitmaps;
-    } else {
-      tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
-      tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
-    }
-#endif
   }
 
   int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES;

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -1635,17 +1635,20 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) {
   SSmaCfg vCreateSmaReq = {0};
   if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tsdbWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno));
+    tsdbWarn("vgId:%d tsma create msg received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno));
     return -1;
   }
-  tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName,
+  
-            vCreateSmaReq.tSma.indexUid);
+  tsdbDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", REPO_ID(pTsdb),
+            vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid);
 
   // record current timezone of server side
   vCreateSmaReq.tSma.timezoneInt = tsTimezone;
 
   if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) {
     // TODO: handle error
+    tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb),
+             vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno));
     tdDestroyTSma(&vCreateSmaReq.tSma);
     return -1;
   }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -339,7 +339,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
     goto _exit;
   }
 
-  rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs);
+  rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp));
   if (rsp.pArray == NULL) {
     rcode = -1;
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -360,7 +360,11 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
 
     // do create table
     if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) {
+      if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
+        cRsp.code = TSDB_CODE_SUCCESS;
+      } else {
         cRsp.code = terrno;
+      }
     } else {
       cRsp.code = TSDB_CODE_SUCCESS;
       tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
@@ -529,6 +533,13 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
         }
       }
 
+      msgIter.uid = createTbReq.uid;
+      if (createTbReq.type == TSDB_CHILD_TABLE) {
+        msgIter.suid = createTbReq.ctb.suid;
+      } else {
+        msgIter.suid = 0;
+      }
+
       tCoderClear(&coder);
     }
 

source/libs/executor/inc/executorimpl.h

@@ -87,9 +87,7 @@ typedef struct SResultInfo {  // TODO refactor
 typedef struct STableQueryInfo {
   TSKEY              lastKey;     // last check ts, todo remove it later
   SResultRowPosition pos;       // current active time window
-//  int32_t        groupIndex;  // group id in table list
 //  SVariant       tag;
-//  SResultRowInfo resInfo;     // result info
 } STableQueryInfo;
 
 typedef enum {
@@ -365,9 +363,10 @@ typedef struct STableScanInfo {
 typedef struct STagScanInfo {
   SColumnInfo     *pCols;
   SSDataBlock     *pRes;
-  int32_t      totalTables;
+  SArray          *pColMatchInfo;
   int32_t          curPos;
-  void        *pReader;
+  SReadHandle      readHandle;
+  STableGroupInfo *pTableGroups;
 } STagScanInfo;
 
 typedef struct SStreamBlockScanInfo {
@@ -579,9 +578,8 @@ typedef struct SSortOperatorInfo {
   uint32_t     sortBufSize;  // max buffer size for in-memory sort
   SArray*      pSortInfo;
   SSortHandle* pSortHandle;
-  SArray*      inputSlotMap;  // for index map from table scan output
+  SArray*      pColMatchInfo;  // for index map from table scan output
   int32_t      bufPageSize;
-//  int32_t      numOfRowsInRes;
 
   // TODO extact struct
   int64_t  startTs;       // sort start time
@@ -646,7 +644,7 @@ void    cleanupAggSup(SAggSupporter* pAggSup);
 void    destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
 void    appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity);
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo);
 SSDataBlock* loadNextDataBlock(void* param);
 
 void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
@@ -704,7 +702,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo*
                                            SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
 
 #if 0
 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
@@ -717,7 +715,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
 
 void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol);
 
-void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput);
 void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput);
 
 STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win);

source/libs/executor/inc/tsort.h

@@ -117,18 +117,25 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle);
 /**
  *
  * @param pHandle
- * @param colIndex
+ * @param colId
  * @return
  */
-bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex);
+bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId);
 
 /**
  *
  * @param pHandle
- * @param colIndex
+ * @param colId
  * @return
  */
-void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex);
+void* tsortGetValue(STupleHandle* pVHandle, int32_t colId);
+
+/**
+ *
+ * @param pSortHandle
+ * @return
+ */
+SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle);
 
 #ifdef __cplusplus
 }

source/libs/executor/src/executorimpl.c

@@ -3520,7 +3520,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
   SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
   SSortedMergeOperatorInfo* pInfo = pOperator->info;
   if (pOperator->status == OP_RES_TO_RETURN) {
-    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL);
   }
 
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
@@ -4701,7 +4701,7 @@ static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo);
 static SArray* extractColumnInfo(SNodeList* pNodeList);
 static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols);
 
-static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget);
+static SArray* createSortInfo(SNodeList* pNodeList);
 static SArray* createIndexMap(SNodeList* pNodeList);
 static SArray* extractPartitionColInfo(SNodeList* pNodeList);
 static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
@@ -4739,7 +4739,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
 
       SQueryTableDataCond cond = {0};
-
       int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
       if (code != TSDB_CODE_SUCCESS) {
         return NULL;
@@ -4783,6 +4782,25 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
           pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList,
           pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId);
       return pOperator;
+    } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
+      STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode;
+      SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
+
+      int32_t  code =
+          doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
+      if (code != TSDB_CODE_SUCCESS) {
+        return NULL;
+      }
+
+      int32_t num = 0;
+      SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num);
+
+      int32_t numOfOutputCols = 0;
+      SArray* colList =
+          extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols);
+
+      SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo);
+      return pOperator;
     } else {
       ASSERT(0);
     }
@@ -4852,16 +4870,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode;
 
     SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
-    SArray*      info = createSortInfo(pSortPhyNode->pSortKeys, pSortPhyNode->pTargets);
+    SArray*      info = createSortInfo(pSortPhyNode->pSortKeys);
-    SArray*      slotMap = createIndexMap(pSortPhyNode->pTargets);
 
     int32_t    numOfCols = 0;
-    SExprInfo* pExprInfo = NULL;
+    SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols);
-    if (pSortPhyNode->pExprs != NULL) {
-      pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols);
-    }
 
-    pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, slotMap, pTaskInfo);
+    int32_t numOfOutputCols = 0;
+    SArray* pColList =
+        extractColMatchInfo(pSortPhyNode->pTargets, pSortPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols);
+
+    pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) {
     SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
 
@@ -5019,7 +5037,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) {
   return pList;
 }
 
-SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) {
+SArray* createSortInfo(SNodeList* pNodeList) {
   size_t  numOfCols = LIST_LENGTH(pNodeList);
   SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo));
   if (pList == NULL) {
@@ -5034,22 +5052,7 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) {
     bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST);
 
     SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr;
-
+    bi.slotId = pColNode->slotId;
-    bool found = false;
-    for (int32_t j = 0; j < LIST_LENGTH(pNodeListTarget); ++j) {
-      STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j);
-
-      SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr;
-      if (pColNode->slotId == pColNodeT->slotId) {  // to find slotId in PhysiSort OutputDataBlockDesc
-        bi.slotId = pTarget->slotId;
-        found = true;
-        break;
-      }
-    }
-
-    if (!found) {
-      qError("sort slot id does not found");
-    }
     taosArrayPush(pList, &bi);
   }
 
@@ -5166,9 +5169,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle*
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
-#if 0
+
-  return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId);
-#endif
   return tsdbQueryTables(pHandle->vnode, &cond, pTableGroupInfo, queryId, taskId);
 
 _error:

source/libs/executor/src/indexoperator.c

@@ -252,8 +252,8 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu
   return TSDB_CODE_QRY_INVALID_INPUT;
 }
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
-  SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName,
+  SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName),
-                                   strlen(left->colName), right->condValue, strlen(right->condValue));
+                                   right->condValue, strlen(right->condValue));
   if (tm == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }

source/libs/executor/src/scanoperator.c

@@ -13,15 +13,16 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "ttime.h"
+#include <libs/function/function.h>
 #include "filter.h"
 #include "function.h"
 #include "functionMgt.h"
 #include "os.h"
 #include "querynodes.h"
+#include "systable.h"
 #include "tglobal.h"
 #include "tname.h"
-#include "systable.h"
+#include "ttime.h"
 
 #include "tdatablock.h"
 #include "tmsg.h"
@@ -538,10 +539,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
 
     while (tqNextDataBlock(pInfo->readerHandle)) {
       SArray*  pCols = NULL;
-      uint64_t groupId;
+      uint64_t groupId = 0;
-      int32_t  numOfRows;
+      uint64_t uid = 0;
-      int16_t  outputCol;
+      int32_t  numOfRows = 0;
-      int32_t  code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows, &outputCol);
+      int16_t  outputCol = 0;
+
+      int32_t  code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &uid, &numOfRows, &outputCol);
 
       if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
         pTaskInfo->code = code;
@@ -550,6 +553,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
 
       pInfo->pRes->info.groupId = groupId;
       pInfo->pRes->info.rows = numOfRows;
+      pInfo->pRes->info.uid = uid;
 
       int32_t numOfCols = pInfo->pRes->info.numOfCols;
       for (int32_t i = 0; i < numOfCols; ++i) {
@@ -605,10 +609,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
   SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
   SOperatorInfo*        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
-    taosMemoryFreeClear(pInfo);
-    taosMemoryFreeClear(pOperator);
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
-    return NULL;
+    goto _error;
   }
 
   int32_t numOfOutput = taosArrayGetSize(pColList);
@@ -625,16 +627,13 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
   tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColIds);
   int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList);
   if (code != 0) {
-    taosMemoryFreeClear(pInfo);
+    goto _error;
-    taosMemoryFreeClear(pOperator);
-    return NULL;
   }
 
   pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES);
   if (pInfo->pBlockLists == NULL) {
-    taosMemoryFreeClear(pInfo);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFreeClear(pOperator);
+    goto _error;
-    return NULL;
   }
 
   pInfo->readerHandle = streamReadHandle;
@@ -655,6 +654,11 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
   pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL);
 
   return pOperator;
+
+  _error:
+  taosMemoryFreeClear(pInfo);
+  taosMemoryFreeClear(pOperator);
+  return NULL;
 }
 
 static void destroySysScanOperator(void* param, int32_t numOfOutput) {
@@ -1159,16 +1163,17 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe
 }
 
 static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
-#if 0
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
   }
 
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+
+#if 0
   int32_t maxNumOfTables = (int32_t)pResultInfo->capacity;
 
   STagScanInfo *pInfo = pOperator->info;
   SSDataBlock  *pRes = pInfo->pRes;
-  *newgroup = false;
 
   int32_t count = 0;
   SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);
@@ -1237,55 +1242,54 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
     pOperator->status = OP_EXEC_DONE;
     //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
   } else {  // return only the tags|table name etc.
-    SExprInfo* pExprInfo = &pOperator->pExpr[0];  // todo use the column list instead of exprinfo
+#endif
 
-    count = 0;
+  STagScanInfo* pInfo = pOperator->info;
-    while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
+  SExprInfo*    pExprInfo = &pOperator->pExpr[0];
-      int32_t i = pInfo->curPos++;
+  SSDataBlock*  pRes = pInfo->pRes;
 
-      STableQueryInfo* item = taosArrayGetP(pa, i);
+  SArray* pa = taosArrayGetP(pInfo->pTableGroups->pGroupList, 0);
+
+  char        str[512] = {0};
+  int32_t     count = 0;
+  SMetaReader mr = {0};
+
+  while (pInfo->curPos < pInfo->pTableGroups->numOfTables && count < pOperator->resultInfo.capacity) {
+    STableKeyInfo* item = taosArrayGet(pa, pInfo->curPos);
 
-      char *data = NULL, *dst = NULL;
-      int16_t type = 0, bytes = 0;
     for (int32_t j = 0; j < pOperator->numOfExprs; ++j) {
-        // not assign value in case of user defined constant output column
+      SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
-        if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) {
-          continue;
-        }
 
-        SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j);
+      // refactor later
-        type  = pExprInfo[j].base.resSchema.type;
+      if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) {
-        bytes = pExprInfo[j].base.resSchema.bytes;
+        metaReaderInit(&mr, pInfo->readHandle.meta, 0);
+        metaGetTableEntryByUid(&mr, item->uid);
 
-        if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
+        STR_TO_VARSTR(str, mr.me.name);
-          data = tsdbGetTableName(item->pTable);
+        metaReaderClear(&mr);
-        } else {
-          data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
-        }
 
-        dst  = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
+        colDataAppend(pDst, count, str, false);
-        doSetTagValueToResultBuf(dst, data, type, bytes);
+
+        //        data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
+        //        dst  = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
+        //        doSetTagValueToResultBuf(dst, data, type, bytes);
       }
 
       count += 1;
     }
 
-    if (pInfo->curPos >= pInfo->totalTables) {
+    if (++pInfo->curPos >= pInfo->pTableGroups->numOfTables) {
       pOperator->status = OP_EXEC_DONE;
     }
+  }
 
   // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
-  }
-
   if (pOperator->status == OP_EXEC_DONE) {
-    setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
+    setTaskStatus(pTaskInfo, TASK_COMPLETED);
   }
 
   pRes->info.rows = count;
   return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
-
-#endif
-  return TSDB_CODE_SUCCESS;
 }
 
 static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
@@ -1293,14 +1297,18 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
   pInfo->pRes = blockDataDestroy(pInfo->pRes);
 }
 
-SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput,
+                                         SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) {
   STagScanInfo*  pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
   SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
     goto _error;
   }
 
-  pInfo->pReader          = readHandle;
+  pInfo->pTableGroups     = pTableGroupInfo;
+  pInfo->pColMatchInfo    = pColMatchInfo;
+  pInfo->pRes             = pResBlock;
+  pInfo->readHandle       = *pReadHandle;
   pInfo->curPos           = 0;
   pOperator->name         = "TagScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
@@ -1311,6 +1319,9 @@ SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int
   pOperator->numOfExprs   = numOfOutput;
   pOperator->pTaskInfo    = pTaskInfo;
 
+  initResultSizeInfo(pOperator, 4096);
+  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+
   pOperator->fpSet =
       createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL);
 

source/libs/executor/src/sortoperator.c

@@ -5,7 +5,7 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator);
 static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput);
 
 SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols,
-                                      SArray* pIndexMap, SExecTaskInfo* pTaskInfo) {
+                                      SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
   SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
   SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   int32_t            rowSize = pResBlock->info.rowSize;
@@ -20,17 +20,19 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
   pInfo->binfo.pRes   = pResBlock;
 
   initResultSizeInfo(pOperator, 1024);
-  pInfo->bufPageSize  = rowSize < 1024 ? 1024 * 2 : rowSize * 2;  // there are headers, so pageSize = rowSize + header
 
-  pInfo->sortBufSize  = pInfo->bufPageSize * 16;  // TODO dynamic set the available sort buffer
   pInfo->pSortInfo    = pSortInfo;
-  pInfo->inputSlotMap = pIndexMap;
+  pInfo->pColMatchInfo= pColMatchColInfo;
   pOperator->name     = "SortOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
   pOperator->blocking = true;
   pOperator->status   = OP_NOT_OPENED;
   pOperator->info     = pInfo;
 
+  // lazy evaluation for the following parameter since the input datablock is not known till now.
+//  pInfo->bufPageSize  = rowSize < 1024 ? 1024 * 2 : rowSize * 2;  // there are headers, so pageSize = rowSize + header
+//  pInfo->sortBufSize  = pInfo->bufPageSize * 16;  // TODO dynamic set the available sort buffer
+
   pOperator->pTaskInfo = pTaskInfo;
   pOperator->fpSet =
       createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, NULL);
@@ -45,14 +47,12 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
   return NULL;
 }
 
-// TODO merge aggregate super table
 void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-
     bool isNull = tsortIsNullVal(pTupleHandle, i);
     if (isNull) {
-      colDataAppend(pColInfo, pBlock->info.rows, NULL, true);
+      colDataAppendNULL(pColInfo, pBlock->info.rows);
     } else {
       char* pData = tsortGetValue(pTupleHandle, i);
       colDataAppend(pColInfo, pBlock->info.rows, pData, false);
@@ -62,11 +62,16 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   pBlock->info.rows += 1;
 }
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) {
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) {
   blockDataCleanup(pDataBlock);
-  blockDataEnsureCapacity(pDataBlock, capacity);
+  ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols);
 
-  blockDataEnsureCapacity(pDataBlock, capacity);
+  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
+  if (p == NULL) {
+    return NULL;
+  }
+
+  blockDataEnsureCapacity(p, capacity);
 
   while (1) {
     STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
@@ -74,12 +79,32 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
       break;
     }
 
-    appendOneRowToDataBlock(pDataBlock, pTupleHandle);
+    appendOneRowToDataBlock(p, pTupleHandle);
-    if (pDataBlock->info.rows >= capacity) {
+    if (p->info.rows >= capacity) {
       return pDataBlock;
     }
   }
 
+  if (p->info.rows > 0) {
+    int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
+    for(int32_t i = 0; i < numOfCols; ++i) {
+      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
+
+      for(int32_t j = 0; j < p->info.numOfCols; ++j) {
+        SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, j);
+        if (pSrc->info.colId == pmInfo->colId) {
+          SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+          colDataAssign(pDst, pSrc, p->info.rows);
+          break;
+        }
+      }
+    }
+
+    pDataBlock->info.rows = p->info.rows;
+    pDataBlock->info.capacity = p->info.rows;
+  }
+
+  blockDataDestroy(p);
   return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
 }
 
@@ -106,16 +131,16 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
   SSortOperatorInfo* pInfo = pOperator->info;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
   }
 
-  int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
+//  pInfo->binfo.pRes is not equalled to the input datablock.
-  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT,
+//  int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
-                                             pInfo->bufPageSize, numOfBufPage, pInfo->binfo.pRes, pTaskInfo->id.str);
+  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT,
+                                             -1, -1, NULL, pTaskInfo->id.str);
 
   tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator);
 
-
   SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
   ps->param = pOperator->pDownstream[0];
   tsortAddSource(pInfo->pSortHandle, ps);
@@ -127,7 +152,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
   }
 
   pOperator->status = OP_RES_TO_RETURN;
-  return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+  return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
 }
 
 void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
@@ -135,5 +160,5 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
   pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
 
   taosArrayDestroy(pInfo->pSortInfo);
-  taosArrayDestroy(pInfo->inputSlotMap);
+  taosArrayDestroy(pInfo->pColMatchInfo);
 }

source/libs/executor/src/tsort.c

@@ -64,25 +64,8 @@ struct SSortHandle {
 
 static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param);
 
-static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) {
+SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) {
-  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  return createOneDataBlock(pSortHandle->pDataBlock, false);
-  pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
-  pBlock->info.numOfCols = numOfCols;
-
-  for(int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData  colInfo = {0};
-
-    colInfo.info.type  = pSchema[i].type;
-    colInfo.info.bytes = pSchema[i].bytes;
-    colInfo.info.colId = pSchema[i].colId;
-    taosArrayPush(pBlock->pDataBlock, &colInfo);
-
-    if (IS_VAR_DATA_TYPE(colInfo.info.type)) {
-      pBlock->info.hasVarCol = true;
-    }
-  }
-
-  return pBlock;
 }
 
 /**
@@ -98,7 +81,10 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, SArray* pIndexMap, int32_t
   pSortHandle->numOfPages = numOfPages;
   pSortHandle->pSortInfo  = pSortInfo;
   pSortHandle->pIndexMap  = pIndexMap;
+
+  if (pBlock != NULL) {
     pSortHandle->pDataBlock = createOneDataBlock(pBlock, false);
+  }
 
   pSortHandle->pOrderedSource     = taosArrayInit(4, POINTER_BYTES);
   pSortHandle->cmpParam.orderInfo = pSortInfo;
@@ -530,6 +516,17 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
 
       if (pHandle->pDataBlock == NULL) {
         pHandle->pDataBlock = createOneDataBlock(pBlock, false);
+
+        // calculate the buffer pages according to the total available buffers.
+        int32_t rowSize = blockDataGetRowSize(pBlock);
+        if (rowSize * 4 > 4096) {
+          pHandle->pageSize = rowSize * 4;
+        } else {
+          pHandle->pageSize = 4096;
+        }
+        // todo!!
+        pHandle->numOfPages = 1024;
+        sortBufSize = pHandle->numOfPages * pHandle->pageSize;
       }
 
       // perform the scalar function calculation before apply the sort
@@ -538,7 +535,6 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
       }
 
       // todo relocate the columns
-
       int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock, pHandle->pIndexMap);
       if (code != 0) {
         return code;
@@ -557,7 +553,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
       }
     }
 
-    if (pHandle->pDataBlock->info.rows > 0) {
+    if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) {
       size_t size = blockDataGetSize(pHandle->pDataBlock);
 
       // Perform the in-memory sort and then flush data in the buffer into disk.
@@ -689,7 +685,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) {
 
 bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) {
   SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex);
-  return colDataIsNull(pColInfoSrc, 0, pVHandle->rowIndex, NULL);
+  return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex);
 }
 
 void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {

source/libs/function/inc/builtinsimpl.h

@@ -73,6 +73,11 @@ bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo)
 int32_t spreadFunction(SqlFunctionCtx* pCtx);
 int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
+bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t histogramFunction(SqlFunctionCtx* pCtx);
+int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/function/src/builtins.c

@@ -225,6 +225,26 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (4 != LIST_LENGTH(pFunc->pParameterList)) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (!IS_NUMERIC_TYPE(colType)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
+      ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
+      ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE };
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // todo
   return TSDB_CODE_SUCCESS;
@@ -242,8 +262,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l
                            "The parameters of first/last can only be columns");
   }
 
-  uint8_t paraType = ((SExprNode*)pPara)->resType.type;
+  pFunc->node.resType = ((SExprNode*)pPara)->resType;
-  pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType};
   return TSDB_CODE_SUCCESS;
 }
 
@@ -600,6 +619,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = diffFunction,
     .finalizeFunc = functionFinalize
   },
+  {
+    .name = "histogram",
+    .type = FUNCTION_TYPE_HISTOGRAM,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHistogram,
+    .getEnvFunc   = getHistogramFuncEnv,
+    .initFunc     = histogramFunctionSetup,
+    .processFunc  = histogramFunction,
+    .finalizeFunc = histogramFinalize
+  },
   {
     .name = "abs",
     .type = FUNCTION_TYPE_ABS,
@@ -917,7 +946,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .translateFunc = translateTbnameColumn,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,
-    .sprocessFunc = NULL,
+    .sprocessFunc = qTbnameFunction,
     .finalizeFunc = NULL
   },
   {

source/libs/function/src/builtinsimpl.c

@@ -20,6 +20,8 @@
 #include "tdatablock.h"
 #include "tpercentile.h"
 
+#define HISTOGRAM_MAX_BINS_NUM 100
+
 typedef struct SSumRes {
   union {
     int64_t  isum;
@@ -89,6 +91,22 @@ typedef struct SSpreadInfo {
   double max;
 } SSpreadInfo;
 
+typedef struct SHistoFuncBin {
+  double lower;
+  double upper;
+  union {
+    int64_t count;
+    double  percentage;
+  };
+} SHistoFuncBin;
+
+typedef struct SHistoFuncInfo {
+  int32_t numOfBins;
+  bool    normalized;
+  SHistoFuncBin bins[];
+} SHistoFuncInfo;
+
+
 #define SET_VAL(_info, numOfElem, res) \
   do {                                 \
     if ((numOfElem) <= 0) {            \
@@ -1777,3 +1795,34 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   }
   return functionFinalize(pCtx, pBlock);
 }
+
+bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
+  pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
+  return true;
+}
+
+bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
+  if (!functionSetup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+  char* binType = pCtx->param[1].param.pz;
+  char* binDesc = pCtx->param[2].param.pz;
+  int64_t nornalized = pCtx->param[3].param.i;
+
+
+  return true;
+}
+
+int32_t histogramFunction(SqlFunctionCtx *pCtx) {
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  //if (pInfo->hasResult == true) {
+  //  SET_DOUBLE_VAL(&pInfo->result, pInfo->max - pInfo->min);
+  //}
+  return functionFinalize(pCtx, pBlock);
+}

source/libs/function/src/tudf.c

@@ -594,7 +594,9 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS
 
   //TODO: free the array output->pDataBlock
   output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
-  taosArrayPush(output->pDataBlock, input->columnData);
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    taosArrayPush(output->pDataBlock, (input + i)->columnData);
+  }
   return 0;
 }
 

source/libs/function/src/udfd.c

@@ -81,6 +81,9 @@ typedef struct SUdf {
   TUdfAggStartFunc      aggStartFunc;
   TUdfAggProcessFunc    aggProcFunc;
   TUdfAggFinishFunc     aggFinishFunc;
+
+  TUdfInitFunc          initFunc;
+  TUdfDestroyFunc       destroyFunc;
 } SUdf;
 
 // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
@@ -101,7 +104,19 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
     fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
     return UDFC_CODE_LOAD_UDF_FAILURE;
   }
-  //TODO: init and destroy function
+
+  char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0};
+  char *initSuffix = "_init";
+  strcpy(initFuncName, udfName);
+  strncat(initFuncName, initSuffix, strlen(initSuffix));
+  uv_dlsym(&udf->lib, initFuncName, (void**)(&udf->initFunc));
+
+  char destroyFuncName[TSDB_FUNC_NAME_LEN+5] = {0};
+  char *destroySuffix = "_destroy";
+  strcpy(destroyFuncName, udfName);
+  strncat(destroyFuncName, destroySuffix, strlen(destroySuffix));
+  uv_dlsym(&udf->lib, destroyFuncName, (void**)(&udf->destroyFunc));
+
   if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) {
     char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
     strcpy(processFuncName, udfName);
@@ -159,6 +174,9 @@ void udfdProcessRequest(uv_work_t *req) {
       if (udf->state == UDF_STATE_INIT) {
         udf->state = UDF_STATE_LOADING;
         udfdLoadUdf(setup->udfName, udf);
+        if (udf->initFunc) {
+          udf->initFunc();
+        }
         udf->state = UDF_STATE_READY;
         uv_cond_broadcast(&udf->condReady);
         uv_mutex_unlock(&udf->lock);
@@ -170,7 +188,6 @@ void udfdProcessRequest(uv_work_t *req) {
       }
       SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle));
       handle->udf = udf;
-      // TODO: allocate private structure and call init function and set it to handle
       SUdfResponse rsp;
       rsp.seqNum = request.seqNum;
       rsp.type = request.type;
@@ -275,10 +292,12 @@ void udfdProcessRequest(uv_work_t *req) {
       if (unloadUdf) {
         uv_cond_destroy(&udf->condReady);
         uv_mutex_destroy(&udf->lock);
+        if (udf->destroyFunc) {
+          (udf->destroyFunc)();
+        }
         uv_dlclose(&udf->lib);
         taosMemoryFree(udf);
       }
-      // TODO: call destroy and free udf private
       taosMemoryFree(handle);
 
       SUdfResponse  response;

source/libs/index/inc/indexCache.h

@@ -31,6 +31,7 @@ extern "C" {
 typedef struct MemTable {
   T_REF_DECLARE()
   SSkipList* mem;
+  void*      pCache;
 } MemTable;
 typedef struct IndexCache {
   T_REF_DECLARE()
@@ -47,6 +48,7 @@ typedef struct IndexCache {
 } IndexCache;
 
 #define CACHE_VERSION(cache) atomic_load_32(&cache->version)
+
 typedef struct CacheTerm {
   // key
   char*   colVal;

source/libs/index/inc/indexComm.h

@@ -24,7 +24,7 @@ extern char JSON_COLUMN[];
 extern char JSON_VALUE_DELIM;
 
 char* indexPackJsonData(SIndexTerm* itm);
-
+char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
 #ifdef __cplusplus
 }
 #endif

source/libs/index/inc/indexInt.h

@@ -166,7 +166,9 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
   } while (0)
 
 #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
+
 #define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F)
+
 #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \
   do {                                       \
     uint8_t oldTy = ty;                      \

source/libs/index/src/index.c

@@ -244,8 +244,8 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde
   return 0;
 }
 
-SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType,
+SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
-                            const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) {
+                            int32_t nColName, const char* colVal, int32_t nColVal) {
   SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm)));
   if (tm == NULL) {
     return NULL;

source/libs/index/src/indexCache.c

@@ -34,44 +34,96 @@ static char*   indexCacheTermGet(const void* pData);
 
 static MemTable* indexInternalCacheCreate(int8_t type);
 
-static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchSuffix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRegex(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
 /*comm func of compare, used in (LE/LT/GE/GT compare)*/
-static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s,
+                                      RangeType type);
+static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+
+static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
                                            RangeType type);
 
 typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
 typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
 
-static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) {
-static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; }
+  int32_t ret = func(a, b);
-static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; }
+  switch (comType) {
-static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; }
+    case QUERY_LESS_THAN: {
+      if (ret < 0) return MATCH;
+    } break;
+    case QUERY_LESS_EQUAL: {
+      if (ret <= 0) return MATCH;
+      break;
+    }
+    case QUERY_GREATER_THAN: {
+      if (ret > 0) return MATCH;
+      break;
+    }
+    case QUERY_GREATER_EQUAL: {
+      if (ret >= 0) return MATCH;
+    }
+  }
+  return CONTINUE;
+}
+static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
+  __compar_fn_t func = getComparFunc(type, 0);
+  return tDoCommpare(func, QUERY_LESS_THAN, a, b);
+}
+static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
+  __compar_fn_t func = getComparFunc(type, 0);
+  return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
+}
+static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
+  __compar_fn_t func = getComparFunc(type, 0);
+  return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
+}
+static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
+  __compar_fn_t func = getComparFunc(type, 0);
+  return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
+}
 
 static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual,
                                                                     tCompareGreaterThan, tCompareGreaterEqual};
 
-static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
+static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
-    cacheSearchTerm,      cacheSearchPrefix,      cacheSearchSuffix,       cacheSearchRegex, cacheSearchLessThan,
+    {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual,
-    cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange};
+     cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange},
+    {cacheSearchTerm_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON,
+     cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON,
+     cacheSearchRange_JSON}};
 
 static void doMergeWork(SSchedMsg* msg);
 static bool indexCacheIteratorNext(Iterate* itera);
 
-static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
   if (cache == NULL) {
     return 0;
   }
-
   MemTable*   mem = cache;
-  char*     key = indexCacheTermGet(ct);
+  IndexCache* pCache = mem->pCache;
+
+  CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+  pCt->colVal = term->colVal;
+  pCt->version = atomic_load_32(&pCache->version);
+
+  char* key = indexCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
   while (tSkipListIterNext(iter)) {
@@ -80,7 +132,7 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S
       break;
     }
     CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
-    if (0 == strcmp(c->colVal, ct->colVal)) {
+    if (0 == strcmp(c->colVal, pCt->colVal)) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
         // taosArrayPush(result, &c->uid);
@@ -92,30 +144,39 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S
       break;
     }
   }
+
+  taosMemoryFree(pCt);
   tSkipListDestroyIter(iter);
   return 0;
 }
-static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
   // impl later
   return 0;
 }
-static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchSuffix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
   // impl later
   return 0;
 }
-static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchRegex(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
   // impl later
   return 0;
 }
-static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
                                       RangeType type) {
   if (cache == NULL) {
     return 0;
   }
+
   _cache_range_compare cmpFn = rangeCompare[type];
 
   MemTable*   mem = cache;
-  char*     key = indexCacheTermGet(ct);
+  IndexCache* pCache = mem->pCache;
+
+  CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+  pCt->colVal = term->colVal;
+  pCt->version = atomic_load_32(&pCache->version);
+
+  char* key = indexCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
   while (tSkipListIterNext(iter)) {
@@ -124,7 +185,7 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult
       break;
     }
     CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
-    TExeCond   cond = cmpFn(c->colVal, ct->colVal, ct->colType);
+    TExeCond   cond = cmpFn(c->colVal, pCt->colVal, pCt->colType);
     if (cond == MATCH) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
@@ -134,26 +195,153 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult
         INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
       }
     } else if (cond == CONTINUE) {
+      continue;
     } else if (cond == BREAK) {
       break;
     }
   }
+  taosMemoryFree(pCt);
   tSkipListDestroyIter(iter);
   return TSDB_CODE_SUCCESS;
 }
-static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
-  return cacheSearchCompareFunc(cache, ct, tr, s, LT);
+  return cacheSearchCompareFunc(cache, term, tr, s, LT);
 }
-static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
-  return cacheSearchCompareFunc(cache, ct, tr, s, LE);
+  return cacheSearchCompareFunc(cache, term, tr, s, LE);
 }
-static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
-  return cacheSearchCompareFunc(cache, ct, tr, s, GT);
+  return cacheSearchCompareFunc(cache, term, tr, s, GT);
 }
-static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
-  return cacheSearchCompareFunc(cache, ct, tr, s, GE);
+  return cacheSearchCompareFunc(cache, term, tr, s, GE);
 }
-static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+
+static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  if (cache == NULL) {
+    return 0;
+  }
+  MemTable*   mem = cache;
+  IndexCache* pCache = mem->pCache;
+
+  CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+  pCt->colVal = term->colVal;
+  pCt->version = atomic_load_32(&pCache->version);
+
+  char* exBuf = NULL;
+  if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+    exBuf = indexPackJsonData(term);
+    pCt->colVal = exBuf;
+  }
+  char* key = indexCacheTermGet(pCt);
+
+  SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
+  while (tSkipListIterNext(iter)) {
+    SSkipListNode* node = tSkipListIterGet(iter);
+    if (node == NULL) {
+      break;
+    }
+    CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
+    if (0 == strcmp(c->colVal, pCt->colVal)) {
+      if (c->operaType == ADD_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        // taosArrayPush(result, &c->uid);
+        *s = kTypeValue;
+      } else if (c->operaType == DEL_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+      }
+    } else {
+      break;
+    }
+  }
+
+  taosMemoryFree(pCt);
+  taosMemoryFree(exBuf);
+  tSkipListDestroyIter(iter);
+  return 0;
+
+  return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT);
+}
+static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc_JSON(cache, term, tr, s, LE);
+}
+static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc_JSON(cache, term, tr, s, GT);
+}
+static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE);
+}
+static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
+                                           RangeType type) {
+  if (cache == NULL) {
+    return 0;
+  }
+  _cache_range_compare cmpFn = rangeCompare[type];
+
+  MemTable*   mem = cache;
+  IndexCache* pCache = mem->pCache;
+
+  CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+  pCt->colVal = term->colVal;
+  pCt->version = atomic_load_32(&pCache->version);
+
+  int8_t dType = INDEX_TYPE_GET_TYPE(term->colType);
+  int    skip = 0;
+  char*  exBuf = NULL;
+
+  if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+    exBuf = indexPackJsonDataPrefix(term, &skip);
+    pCt->colVal = exBuf;
+  }
+  char* key = indexCacheTermGet(pCt);
+
+  SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
+  while (tSkipListIterNext(iter)) {
+    SSkipListNode* node = tSkipListIterGet(iter);
+    if (node == NULL) {
+      break;
+    }
+    CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
+
+    TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType);
+    if (cond == MATCH) {
+      if (c->operaType == ADD_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        // taosArrayPush(result, &c->uid);
+        *s = kTypeValue;
+      } else if (c->operaType == DEL_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+      }
+    } else if (cond == CONTINUE) {
+      continue;
+    } else if (cond == BREAK) {
+      break;
+    }
+  }
+
+  taosMemoryFree(pCt);
+  taosMemoryFree(exBuf);
+  tSkipListDestroyIter(iter);
+
+  return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
   // impl later
   return 0;
 }
@@ -167,6 +355,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
   };
 
   cache->mem = indexInternalCacheCreate(type);
+  cache->mem->pCache = cache;
   cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName);
   cache->type = type;
   cache->index = idx;
@@ -325,6 +514,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
       indexCacheRef(cache);
       cache->imm = cache->mem;
       cache->mem = indexInternalCacheCreate(cache->type);
+      cache->mem->pCache = cache;
       cache->occupiedMem = 0;
       // sched to merge
       // unref cache in bgwork
@@ -379,11 +569,19 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u
   return 0;
 }
 
-static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SIdxTempResult* tr, STermValueType* s) {
+static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTempResult* tr, STermValueType* s) {
   if (mem == NULL) {
     return 0;
   }
-  return cacheSearch[qtype](mem, ct, tr, s);
+
+  SIndexTerm*     term = query->term;
+  EIndexQueryType qtype = query->qType;
+
+  if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+    return cacheSearch[1][qtype](mem, term, tr, s);
+  } else {
+    return cacheSearch[0][qtype](mem, term, tr, s);
+  }
 }
 int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) {
   int64_t st = taosGetTimestampUs();
@@ -400,25 +598,24 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result
   indexMemRef(imm);
   taosThreadMutexUnlock(&pCache->mtx);
 
-  SIndexTerm*     term = query->term;
+  // SIndexTerm*     term = query->term;
-  EIndexQueryType qtype = query->qType;
+  // EIndexQueryType qtype = query->qType;
 
-  bool  hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
+  // bool  isJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
-  char* p = term->colVal;
+  // char* p = term->colVal;
-  if (hasJson) {
+  // if (isJson) {
-    p = indexPackJsonData(term);
+  //  p = indexPackJsonData(term);
-  }
+  //}
-  CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)};
+  // CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)};
 
-  int ret = indexQueryMem(mem, &ct, qtype, result, s);
+  int ret = indexQueryMem(mem, query, result, s);
   if (ret == 0 && *s != kTypeDeletion) {
     // continue search in imm
-    ret = indexQueryMem(imm, &ct, qtype, result, s);
+    ret = indexQueryMem(imm, query, result, s);
-  }
-
-  if (hasJson) {
-    taosMemoryFreeClear(p);
   }
+  // if (isJson) {
+  //  taosMemoryFreeClear(p);
+  //}
 
   indexMemUnRef(mem);
   indexMemUnRef(imm);

source/libs/index/src/indexComm.c

@@ -46,3 +46,30 @@ char* indexPackJsonData(SIndexTerm* itm) {
 
   return buf;
 }
+char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
+  /*
+   * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
+   * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
+   */
+  uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType);
+
+  int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1;
+  char*   buf = (char*)taosMemoryCalloc(1, sz);
+  char*   p = buf;
+
+  memcpy(p, itm->colName, itm->nColName);
+  p += itm->nColName;
+
+  memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM));
+  p += sizeof(JSON_VALUE_DELIM);
+
+  memcpy(p, &ty, sizeof(ty));
+  p += sizeof(ty);
+
+  memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM));
+  p += sizeof(JSON_VALUE_DELIM);
+
+  *skip = p - buf;
+
+  return buf;
+}

source/libs/index/src/indexTfile.c

@@ -311,17 +311,17 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult
   bool  hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
   int   ret = 0;
   char* p = tem->colVal;
-  uint64_t sz = tem->nColVal;
+  int   skip = 0;
+
   if (hasJson) {
-    p = indexPackJsonData(tem);
+    p = indexPackJsonDataPrefix(tem, &skip);
-    sz = strlen(p);
   }
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
   AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
   FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
 
-  FstSlice h = fstSliceCreate((uint8_t*)p, sz);
+  FstSlice h = fstSliceCreate((uint8_t*)p, skip);
   fstStreamBuilderSetRange(sb, &h, type);
   fstSliceDestroy(&h);
 
@@ -606,16 +606,16 @@ static bool tfileIteratorNext(Iterate* iiter) {
 static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
 
 static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
-  TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter));
+  TFileFstIter* iter = taosMemoryCalloc(1, sizeof(TFileFstIter));
-  if (tIter == NULL) {
+  if (iter == NULL) {
     return NULL;
   }
 
-  tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
+  iter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
-  tIter->fb = fstSearch(reader->fst, tIter->ctx);
+  iter->fb = fstSearch(reader->fst, iter->ctx);
-  tIter->st = streamBuilderIntoStream(tIter->fb);
+  iter->st = streamBuilderIntoStream(iter->fb);
-  tIter->rdr = reader;
+  iter->rdr = reader;
-  return tIter;
+  return iter;
 }
 
 Iterate* tfileIteratorCreate(TFileReader* reader) {

source/libs/index/test/indexTests.cc

@@ -483,7 +483,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
 
   std::string     colName("voltage");
   std::string     colVal("ab");
-  SIndexTerm*     term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+  SIndexTerm*     term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                      colVal.c_str(), colVal.size());
   SIndexTermQuery query = {term, QUERY_TERM};
 
@@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::string colName("voltage");
   {
     std::string colVal("v1");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
@@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) {
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v2");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
@@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::cout << "--------first----------" << std::endl;
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v4");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
@@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) {
     std::string colVal("v4");
     for (size_t i = 0; i < 10; i++) {
       colVal[colVal.size() - 1] = 'a' + i;
-      SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          colVal.c_str(), colVal.size());
       coj->Put(term, colId, version++, suid++);
       indexTermDestroy(term);
@@ -623,7 +623,7 @@ TEST_F(IndexCacheEnv, cache_test) {
   // begin query
   {
     std::string     colVal("v3");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
@@ -638,7 +638,7 @@ TEST_F(IndexCacheEnv, cache_test) {
   }
   {
     std::string     colVal("v2");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
@@ -670,7 +670,7 @@ class IndexObj {
     return ret;
   }
   void Del(const std::string& colName, const std::string& colVal, uint64_t uid) {
-    SIndexTerm*      term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -679,7 +679,7 @@ class IndexObj {
   }
   int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                       size_t numOfTable = 100 * 10000) {
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -701,7 +701,7 @@ class IndexObj {
         // opt
         tColVal[taosRand() % colValSize] = 'a' + k % 26;
       }
-      SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          tColVal.c_str(), tColVal.size());
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
@@ -737,7 +737,7 @@ class IndexObj {
 
   int SearchOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
@@ -759,7 +759,7 @@ class IndexObj {
   }
   int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
@@ -784,7 +784,7 @@ class IndexObj {
 
   void PutOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, 10);
@@ -792,7 +792,7 @@ class IndexObj {
   }
   void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, val);
@@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string           colName("tag1"), colVal("Hello");
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 

source/libs/index/test/jsonUT.cc

@@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("ab1");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("123");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                     colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("abxxxxxxxxxxxx");
     for (int i = 0; i < 1000; i++) {
       colVal[i % colVal.size()] = '0' + i % 128;
-      SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          colVal.c_str(), colVal.size());
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("voltagefdadfa");
     std::string colVal("abxxxxxxxxxxxx");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                     colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));

source/libs/parser/src/parTranslater.c

@@ -2174,17 +2174,6 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
-  SNode* pNode;
-  FOREACH(pNode, pStmt->pTags) {
-    SColumnDefNode* pCol = (SColumnDefNode*)pNode;
-    if (pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1) {
-      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
   if (NULL == pFuncs) {
     return TSDB_CODE_SUCCESS;
@@ -2196,6 +2185,113 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pTags) {
+  int32_t ntags = LIST_LENGTH(pTags);
+  if (0 == ntags) {
+    return TSDB_CODE_SUCCESS;
+  } else if (ntags > TSDB_MAX_TAGS) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM);
+  }
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t tagsSize = 0;
+  SNode*  pNode = NULL;
+  FOREACH(pNode, pTags) {
+    SColumnDefNode* pTag = (SColumnDefNode*)pNode;
+    int32_t         len = strlen(pTag->colName);
+    if (NULL != taosHashGet(pHash, pTag->colName, len)) {
+      code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN);
+    }
+    if (TSDB_CODE_SUCCESS == code && pTag->dataType.type == TSDB_DATA_TYPE_JSON && ntags > 1) {
+      code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_BINARY_LEN) ||
+          (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_NCHAR_LEN)) {
+        code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
+      }
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      code = taosHashPut(pHash, pTag->colName, len, &pTag, POINTER_BYTES);
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      tagsSize += pTag->dataType.bytes;
+    } else {
+      break;
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code && tagsSize > TSDB_MAX_TAGS_LEN) {
+    code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_LENGTH, TSDB_MAX_TAGS_LEN);
+  }
+
+  return code;
+}
+
+static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pCols) {
+  int32_t ncols = LIST_LENGTH(pCols);
+  if (ncols < TSDB_MIN_COLUMNS) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
+  } else if (ncols > TSDB_MAX_COLUMNS) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TOO_MANY_COLUMNS);
+  }
+
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  bool    first = true;
+  int32_t rowSize = 0;
+  SNode*  pNode = NULL;
+  FOREACH(pNode, pCols) {
+    SColumnDefNode* pCol = (SColumnDefNode*)pNode;
+    if (first) {
+      first = false;
+      if (TSDB_DATA_TYPE_TIMESTAMP != pCol->dataType.type) {
+        code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN);
+      }
+    }
+    int32_t len = strlen(pCol->colName);
+    if (TSDB_CODE_SUCCESS == code && NULL != taosHashGet(pHash, pCol->colName, len)) {
+      code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN);
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_BINARY_LEN) ||
+          (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_NCHAR_LEN)) {
+        code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
+      }
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES);
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      rowSize += pCol->dataType.bytes;
+    } else {
+      break;
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code && rowSize > TSDB_MAX_BYTES_PER_ROW) {
+    code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW);
+  }
+
+  return code;
+}
+
+static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+  SHashObj* pHash = taosHashInit(LIST_LENGTH(pStmt->pTags) + LIST_LENGTH(pStmt->pCols),
+                                 taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  if (NULL == pHash) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = checkTableTagsSchema(pCxt, pHash, pStmt->pTags);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = checkTableColsSchema(pCxt, pHash, pStmt->pCols);
+  }
+
+  taosHashCleanup(pHash);
+  return code;
+}
+
 static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
   int32_t code = checkRangeOption(pCxt, "delay", pStmt->pOptions->delay, TSDB_MIN_ROLLUP_DELAY, TSDB_MAX_ROLLUP_DELAY);
   if (TSDB_CODE_SUCCESS == code) {
@@ -2211,7 +2307,7 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
     code = checkTableSmaOption(pCxt, pStmt);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = checkTableTags(pCxt, pStmt);
+    code = checkTableSchema(pCxt, pStmt);
   }
   return code;
 }
@@ -3838,6 +3934,10 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl
     goto over;
   }
 
+  if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code && pClause->ignoreNotExists) {
+    code = TSDB_CODE_SUCCESS;
+  }
+
   *pIsSuperTable = false;
 
   SVgroupInfo info = {0};

source/libs/parser/src/parUtil.c

@@ -129,7 +129,23 @@ static char* getSyntaxErrFormat(int32_t errCode) {
     case TSDB_CODE_PAR_INVALID_DROP_STABLE:
       return "Cannot drop super table in batch";
     case TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE:
-      return "start(end) time of query range required or time range too large";
+      return "Start(end) time of query range required or time range too large";
+    case TSDB_CODE_PAR_DUPLICATED_COLUMN:
+      return "Duplicated column names";
+    case TSDB_CODE_PAR_INVALID_TAGS_LENGTH:
+      return "Tags length exceeds max length %d";
+    case TSDB_CODE_PAR_INVALID_ROW_LENGTH:
+      return "Row length exceeds max length %d";
+    case TSDB_CODE_PAR_INVALID_COLUMNS_NUM:
+      return "Illegal number of columns";
+    case TSDB_CODE_PAR_TOO_MANY_COLUMNS:
+      return "Too many columns";
+    case TSDB_CODE_PAR_INVALID_FIRST_COLUMN:
+      return "First column must be timestamp";
+    case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN:
+      return "Invalid binary/nchar column length";
+    case TSDB_CODE_PAR_INVALID_TAGS_NUM:
+      return "Invalid number of tag columns";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:

source/libs/planner/src/planLogicCreater.c

@@ -277,6 +277,11 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
     code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols);
   }
 
+  // rewrite the expression in subsequent clauses
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM);
+  }
+
   pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta);
 
   if (TSDB_CODE_SUCCESS == code) {

source/libs/planner/src/planOptimizer.c

@@ -123,14 +123,40 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) {
   return NULL;
 }
 
+static bool needOptimizeDataRequire(const SFunctionNode* pFunc) {
+  if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) {
+    return false;
+  }
+  SNode* pPara = NULL;
+  FOREACH(pPara, pFunc->pParameterList) {
+    if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) {
+  if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) {
+    return false;
+  }
+  SNode* pPara = NULL;
+  FOREACH(pPara, pFunc->pParameterList) {
+    if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) {
   SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent);
   SNode*     pFunc = NULL;
   FOREACH(pFunc, pAllFuncs) {
     int32_t code = TSDB_CODE_SUCCESS;
-    if (fmIsSpecialDataRequiredFunc(((SFunctionNode*)pFunc)->funcId)) {
+    if (needOptimizeDataRequire((SFunctionNode*)pFunc)) {
       code = nodesListMakeStrictAppend(pSdrFuncs, nodesCloneNode(pFunc));
-    } else if (fmIsDynamicScanOptimizedFunc(((SFunctionNode*)pFunc)->funcId)) {
+    } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) {
       code = nodesListMakeStrictAppend(pDsoFuncs, nodesCloneNode(pFunc));
     }
     if (TSDB_CODE_SUCCESS != code) {
@@ -541,9 +567,14 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
   if (QUERY_NODE_OPERATOR != nodeType(pCond)) {
     return false;
   }
+
+  SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions;
+  if (OP_TYPE_EQUAL != pOper->opType) {
+    return false;
+  }
+
   SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
   SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
-  SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions;
   if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) {
     return cpdIsPrimaryKey(pOper->pRight, pRightCols);
   } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) {

source/libs/planner/test/planOptimizeTest.cpp

@@ -20,11 +20,21 @@ using namespace std;
 
 class PlanOptimizeTest : public PlannerTestBase {};
 
+TEST_F(PlanOptimizeTest, optimizeScanData) {
+  useDb("root", "test");
+
+  run("SELECT COUNT(*) FROM t1");
+
+  run("SELECT COUNT(c1) FROM t1");
+
+  run("SELECT COUNT(CAST(c1 AS BIGINT)) FROM t1");
+}
+
 TEST_F(PlanOptimizeTest, orderByPrimaryKey) {
   useDb("root", "test");
 
-  run("select * from t1 order by ts");
+  run("SELECT * FROM t1 ORDER BY ts");
-  run("select * from t1 order by ts desc");
+  run("SELECT * FROM t1 ORDER BY ts DESC");
-  run("select c1 from t1 order by ts");
+  run("SELECT c1 FROM t1 ORDER BY ts");
-  run("select c1 from t1 order by ts desc");
+  run("SELECT c1 FROM t1 ORDER BY ts DESC");
 }

source/libs/scalar/src/sclfunc.c

@@ -1513,3 +1513,9 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
   colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t*) colDataGetData(pInput->columnData, 4));
   return TSDB_CODE_SUCCESS;
 }
+
+int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  ASSERT(inputNum == 1);
+  colDataAppend(pOutput->columnData, pOutput->numOfRows, colDataGetData(pInput->columnData, 0), false);
+  return TSDB_CODE_SUCCESS;
+}

source/libs/scalar/src/sclvector.c

@@ -1023,8 +1023,7 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
-      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0);
-                * getVectorDoubleValueFnRight(RIGHT_COL, 0);
     }
   }
 }
@@ -1050,8 +1049,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
-      *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
+      *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i);
-                * getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
   } else if (pLeft->numOfRows == 1) {
     vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);

source/libs/scheduler/src/scheduler.c

@@ -1085,19 +1085,22 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
     case TDMT_VND_CREATE_TABLE_RSP: {
       SVCreateTbBatchRsp batchRsp = {0};
       if (msg) {
-        SCH_ERR_JRET(tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp));
+        SCoder coder = {0};
-        if (batchRsp.pArray) {
+        tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER);
-          int32_t num = taosArrayGetSize(batchRsp.pArray);
+        code = tDecodeSVCreateTbBatchRsp(&coder, &batchRsp);
-          for (int32_t i = 0; i < num; ++i) {
+        if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) {
-            SVCreateTbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
+          for (int32_t i = 0; i < batchRsp.nRsps; ++i) {
+            SVCreateTbRsp *rsp = batchRsp.pRsps + i;
             if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) {
-              taosArrayDestroy(batchRsp.pArray);
+              tCoderClear(&coder);
               SCH_ERR_JRET(rsp->code);
+            } else if (TSDB_CODE_SUCCESS != rsp->code) {
+              code = rsp->code;
             }
           }
-
-          taosArrayDestroy(batchRsp.pArray);
         }
+        tCoderClear(&coder);
+        SCH_ERR_JRET(code);
       }
 
       SCH_ERR_JRET(rspCode);
@@ -1110,13 +1113,14 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
         SCoder coder = {0};
         tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER);
         code = tDecodeSVDropTbBatchRsp(&coder, &batchRsp);
-        if (TSDB_CODE_SUCCESS == code && batchRsp.pArray) {
+        if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) {
-          int32_t num = taosArrayGetSize(batchRsp.pArray);
+          for (int32_t i = 0; i < batchRsp.nRsps; ++i) {
-          for (int32_t i = 0; i < num; ++i) {
+            SVDropTbRsp *rsp = batchRsp.pRsps + i;
-            SVDropTbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
             if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) {
               tCoderClear(&coder);
               SCH_ERR_JRET(rsp->code);
+            } else if (TSDB_CODE_SUCCESS != rsp->code) {
+              code = rsp->code;
             }
           }
         }

source/libs/sync/src/syncRaftLog.c

@@ -57,7 +57,13 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   syncMeta.seqNum = pEntry->seqNum;
   syncMeta.term = pEntry->term;
   code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
-  assert(code == 0);
+  if (code != 0) {
+    int32_t err = terrno;
+    const char *errStr = tstrerror(err);
+    sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr);
+    ASSERT(0);
+  }    
+  //assert(code == 0);
 
   walFsync(pWal, true);
   return code;
@@ -69,7 +75,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
 
   if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
     SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
-    assert(walReadWithHandle(pWalHandle, index) == 0);
+    int32_t code = walReadWithHandle(pWalHandle, index);
+    if (code != 0) {
+      int32_t err = terrno;
+      const char *errStr = tstrerror(err);
+      sError("walReadWithHandle error, err:%d, msg:%s", err, errStr);
+      ASSERT(0);
+    }    
+    //assert(walReadWithHandle(pWalHandle, index) == 0);
 
     SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
     assert(pEntry != NULL);

source/libs/transport/src/transSrv.c

@@ -784,6 +784,11 @@ static void uvDestroyConn(uv_handle_t* handle) {
   tDebug("server conn %p destroy", conn);
   // uv_timer_stop(&conn->pTimer);
   transQueueDestroy(&conn->srvMsgs);
+
+  if (conn->regArg.init == 1) {
+    transFreeMsg(conn->regArg.msg.pCont);
+    conn->regArg.init = 0;
+  }
   QUEUE_REMOVE(&conn->queue);
   taosMemoryFree(conn->pTcp);
   taosMemoryFree(conn);

source/libs/wal/src/walRead.c

@@ -49,7 +49,10 @@ void walCloseReadHandle(SWalReadHandle *pRead) {
   taosMemoryFree(pRead);
 }
 
-int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
+int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) {
+  // TODO
+  return 0;
+}
 
 static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
   int code = 0;

source/libs/wal/src/walWrite.c

@@ -198,12 +198,14 @@ int walRoll(SWal *pWal) {
   if (pWal->pWriteIdxTFile != NULL) {
     code = taosCloseFile(&pWal->pWriteIdxTFile);
     if (code != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
     }
   }
   if (pWal->pWriteLogTFile != NULL) {
     code = taosCloseFile(&pWal->pWriteLogTFile);
     if (code != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
     }
   }
@@ -263,14 +265,19 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog
   if (index == pWal->vers.lastVer + 1) {
     if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
       pWal->vers.firstVer = index;
-      code = walRoll(pWal);
+      if (walRoll(pWal) < 0) {
-      ASSERT(code == 0);
+        return -1;
+      }
     } else {
       int64_t passed = walGetSeq() - pWal->lastRollSeq;
       if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
-        walRoll(pWal);
+        if (walRoll(pWal) < 0) {
+          return -1;
+        }
       } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
-        walRoll(pWal);
+        if (walRoll(pWal) < 0) {
+          return -1;
+        }
       }
     }
   } else {

source/util/src/terror.c

@@ -443,6 +443,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR,             "scheduler status erro
 TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR,           "scheduler internal error")
 TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR,                 "Invalid msg order")
 
+// parser
+TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TABLE_NOT_EXIST,          "Table does not exist")
+
 //planner
 TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR,          "planner internal error")
 

tests/requirements.txt

@@ -0,0 +1,5 @@
+taospy
+numpy
+fabric2
+psutil
+pandas

tests/script/tsim/query/explain.sim

@@ -25,15 +25,15 @@ sql insert into tb3 values (now, 3, "Hash  (cost=229.20..229.20 rows=101 width=2
 sql create table tb4 using st1 tags(4);
 sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1  (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)");
 
-sql create table tb1 using st2 tags(1);
+#sql create table tb1 using st2 tags(1);
-sql insert into tb1 values (now, 1, "Hash Join  (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)");
+#sql insert into tb1 values (now, 1, "Hash Join  (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)");
 
-sql create table tb2 using st2 tags(2);
+#sql create table tb2 using st2 tags(2);
-sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2  (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)");
+#sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2  (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)");
-sql create table tb3 using st2 tags(3);
+#sql create table tb3 using st2 tags(3);
-sql insert into tb3 values (now, 3, "Hash  (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)");
+#sql insert into tb3 values (now, 3, "Hash  (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)");
-sql create table tb4 using st2 tags(4);
+#sql create table tb4 using st2 tags(4);
-sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1  (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)");
+#sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1  (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)");
 
 
 print ======== step2 

tests/script/tsim/query/scalarFunction.sim

@@ -102,20 +102,26 @@ print ====> $data60 $data61 $data62 $data63 $data64 $data65
 print ====> $data70 $data71 $data72 $data73 $data74 $data75
 print ====> $data80 $data81 $data82 $data83 $data84 $data85
 print ====> $data90 $data91 $data92 $data93 $data94 $data95
+print ====> rows = $rows and rowNum = $rowNum for ct1
 if $rows != $rowNum then
   return -1
 endi
+
 print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb
 sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb
+print ====> rows = $rows and totalRows = $totalRows for stb
 if $rows != $totalRows then
   return -1
 endi
+
 print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb
 sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb
+print ====> rows = $rows and rowNum = $rowNum for ntb
 if $rows != $rowNum then
   return -1
 endi
 
+
 print ====> log
 sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1
 print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1

tests/script/tsim/query/session.sim

@@ -65,6 +65,15 @@ sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.031', 5)
 sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.036', 6)
 sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.51', 7)
 
+# vnode does not return the precision of the table
+print ====> create database d1 precision 'us' 
+sql create database d1 precision 'us'
+sql use d1
+sql create table dev_001 (ts timestamp ,i timestamp ,j int)
+sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
+sql create table secondts(ts timestamp,t2 timestamp,i int)
+sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
+
 $loop_test = 0  
 loop_test_pos:
 
@@ -288,15 +297,6 @@ sql_error sql select count(*) from dev_001 session(ts,0s)
 sql_error select count(*) from dev_001 session(i,1y)
 sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0'
 
-# vnode does not return the precision of the table
-print ====> create database d1 precision 'us' 
-sql create database d1 precision 'us'
-sql use d1
-sql create table dev_001 (ts timestamp ,i timestamp ,j int)
-sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
-sql create table secondts(ts timestamp,t2 timestamp,i int)
-sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
-
 #print ====> select count(*) from dev_001 session(ts,1u)
 #sql select _wstartts, count(*) from dev_001 session(ts,1u)
 #print rows: $rows

tests/script/tsim/query/udf.sim

@@ -43,6 +43,27 @@ if $data00 != 2.236067977 then
   return -1
 endi
 
+sql create table t2 (ts timestamp, f1 int, f2 int);
+sql insert into t2 values(now, 0, 0)(now+1s, 1, 1);
+sql select udf1(f1, f2) from t2;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 88 then
+  return -1
+endi
+if $data10 != 88 then
+  return -1
+endi
+
+sql select udf2(f1, f2) from t2;
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 1.414213562 then
+  return -1
+endi
+
 #sql drop function udf1;
 #sql drop function udf2;
 system sh/exec.sh -n dnode1 -s stop -x SIGKILL

tests/script/tsim/table/basic1.sim

@@ -91,9 +91,9 @@ print =============== create normal table
 sql create database ndb
 sql use ndb
 sql create table nt0 (ts timestamp, i int)
-sql create table if not exists nt0 (ts timestamp, i int)
+# sql create table if not exists nt0 (ts timestamp, i int)
 sql create table nt1 (ts timestamp, i int)
-sql create table if not exists nt1 (ts timestamp, i int)
+# sql create table if not exists nt1 (ts timestamp, i int)
 sql create table if not exists nt3 (ts timestamp, i int)
 
 sql show tables

tests/system-test/2-query/join.py

@@ -5,6 +5,7 @@ from util.sql import *
 from util.cases import *
 from util.dnodes import *
 
+PRIMARY_COL = "ts"
 
 INT_COL     = "c1"
 BINT_COL    = "c2"
@@ -18,9 +19,10 @@ BINARY_COL  = "c8"
 NCHAR_COL   = "c9"
 TS_COL      = "c10"
 
-UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ]
+NUM_COL     = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
 CHAR_COL    = [ BINARY_COL, NCHAR_COL, ]
-TS_TYPE_COL = [TS_COL]
+BOOLEAN_COL = [ BOOL_COL, ]
+TS_TYPE_COL = [ TS_COL, ]
 
 class TDTestCase:
 
@@ -28,88 +30,171 @@ class TDTestCase:
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 
-    def __length_condition(self):
+    def __query_condition(self,tbname):
-        length_condition = []
+        query_condition = []
         for char_col in CHAR_COL:
-            length_condition.extend(
+            query_condition.extend(
                 (
-                    char_col,
+                    f"{tbname}.{char_col}",
-                    f"upper( {char_col} )",
+                    f"upper( {tbname}.{char_col} )",
                 )
             )
-            length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL)
+            query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
-            length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
-            length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL )
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
+        for num_col in NUM_COL:
+            query_condition.extend(
+                (
+                    f"{tbname}.{num_col}",
+                    f"sin( {tbname}.{num_col} )"
+                )
+            )
+            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL )
 
-        length_condition.append('''"test1234!@#$%^&*():'><?/.,][}{"''')
+        query_condition.append(''' "test1234!@#$%^&*():'><?/.,][}{" ''')
 
-        return length_condition
+        return query_condition
 
-    def __where_condition(self, col):
+    def __join_condition(self, tb_list, filter=PRIMARY_COL):
-        # return f" where count({col}) > 0 "
+        # sourcery skip: flip-comparison
+        if 1 == len(tb_list):
+            join_filter = f"{tb_list[0]}.{filter} = {tb_list[0]}.{filter} "
+        elif 2 == len(tb_list):
+            join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} "
+        else:
+            join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} "
+            for i in range(1, len(tb_list)-1 ):
+                join_filter += f"and {tb_list[i]}.{filter} = {tb_list[i+1]}.{filter}"
+
+        return join_filter
+
+    def __where_condition(self, col, tbname):
+        if col in NUM_COL:
+            return f" abs( {tbname}.{col} ) >= 0"
+        elif col in CHAR_COL:
+            return f" lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
+        elif col in BOOLEAN_COL:
+            return f" {tbname}.{col} in (false, true)  "
+        elif col in TS_TYPE_COL or col in PRIMARY_COL:
+            return f" cast( {tbname}.{col} as binary(16) ) is not null "
+        else:
             return ""
 
-    def __group_condition(self, col, having = ""):
+    def __group_condition(self, tbname, col, having = ""):
         return f" group by {col} having {having}" if having else f" group by {col} "
 
-    def __length_current_check(self, tbname):
+    def __join_check(self, tblist, checkrows, join_flag=True):
-        length_condition = self.__length_condition()
+        query_conditions = self.__query_condition(tblist[0])
-        for condition in length_condition:
+        join_condition = self.__join_condition(tb_list=tblist) if join_flag else " "
-            where_condition = self.__where_condition(condition)
+        for condition in query_conditions:
-            group_having = self.__group_condition(condition, having=f"{condition} is not null " )
+            where_condition =  self.__where_condition(col=condition, tbname=tblist[0])
-            group_no_having= self.__group_condition(condition )
+            group_having = self.__group_condition(tbname=tblist[0], col=condition, having=f"{condition} is not null " )
+            group_no_having= self.__group_condition(tbname=tblist[0], col=condition )
             groups = ["", group_having, group_no_having]
-
             for group_condition in groups:
-                tdSql.query(f"select {condition} from {tbname} {where_condition}  {group_condition} ")
+                if where_condition:
-                datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
+                    sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition} "
-                length_data = [ len(str(data))  if data else None for data in datas ]
+                else:
-                tdSql.query(f"select length( {condition} ) from {tbname} {where_condition}  {group_condition}")
+                    sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition}  {group_condition} "
-                for i in range(len(length_data)):
-                    tdSql.checkData(i, 0, length_data[i] ) if length_data[i] else tdSql.checkData(i, 0, None)
 
-    def __length_err_check(self,tbname):
+                if not join_flag :
-        sqls = []
+                    tdSql.error(sql=sql)
+                if len(tblist) == 2:
+                    if "ct1" in tblist or "t1" in tblist:
+                        self.__join_current(sql, checkrows)
+                    elif where_condition or "not null" in group_condition:
+                        self.__join_current(sql, checkrows + 2 )
+                    elif group_condition:
+                        self.__join_current(sql, checkrows + 3 )
+                    else:
+                        self.__join_current(sql, checkrows + 5 )
+                if len(tblist) > 2 or len(tblist) < 1:
+                    tdSql.error(sql=sql)
 
-        for un_char_col in UN_CHAR_COL:
+    # def __join_err_check(self,tbname):
-            sqls.extend(
+    #     sqls = []
-                (
-                    f"select length( {un_char_col} ) from {tbname} ",
-                    f"select length(ceil( {un_char_col} )) from {tbname} ",
-                    f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ",
-                )
-            )
 
-            sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL )
+    #     for un_char_col in NUM_COL:
-            sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
+    #         sqls.extend(
+    #             (
+    #                 f"select length( {un_char_col} ) from {tbname} ",
+    #                 f"select length(ceil( {un_char_col} )) from {tbname} ",
+    #                 f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ",
+    #             )
+    #         )
 
-        sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL)
+    #         sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL )
-        sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
+    #         sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
-        sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL)
+
-        sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL )
+    #     sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL)
-        sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL )
+    #     sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
-        sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL )
+    #     sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL)
-        sqls.extend(
+    #     sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL )
-            (
+    #     sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL )
-                f"select length() from {tbname} ",
+    #     sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL )
-                f"select length(*) from {tbname} ",
+    #     sqls.extend(
-                f"select length(ccccccc) from {tbname} ",
+    #         (
-                f"select length(111) from {tbname} ",
+    #             f"select length() from {tbname} ",
-                f"select length(c8, 11) from {tbname} ",
+    #             f"select length(*) from {tbname} ",
-            )
+    #             f"select length(ccccccc) from {tbname} ",
-        )
+    #             f"select length(111) from {tbname} ",
+    #             f"select length(c8, 11) from {tbname} ",
+    #         )
+    #     )
+
+    #     return sqls
+
+    def __join_current(self, sql, checkrows):
+        tdSql.query(sql=sql)
+        tdSql.checkRows(checkrows)
 
-        return sqls
 
     def __test_current(self):
+        # sourcery skip: extract-duplicate-method, inline-immediately-returned-variable
         tdLog.printNoPrefix("==========current sql condition check , must return query ok==========")
-        tbname = ["ct1", "ct2", "ct4", "t1"]
+        tblist_1 = ["ct1", "ct2"]
-        for tb in tbname:
+        self.__join_check(tblist_1, 1)
-            self.__length_current_check(tb)
+        tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========")
-            tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========")
+        tblist_2 = ["ct2", "ct4"]
+        self.__join_check(tblist_2, self.rows)
+        tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========")
+        tblist_3 = ["t1", "ct4"]
+        self.__join_check(tblist_3, 1)
+        tdLog.printNoPrefix(f"==========current sql condition check in {tblist_3} over==========")
+        tblist_4 = ["t1", "ct1"]
+        self.__join_check(tblist_4, 1)
+        tdLog.printNoPrefix(f"==========current sql condition check in {tblist_4} over==========")
 
     def __test_error(self):
+        # sourcery skip: extract-duplicate-method, move-assign-in-block
         tdLog.printNoPrefix("==========err sql condition check , must return error==========")
+        err_list_1 = ["ct1","ct2", "ct4"]
+        err_list_2 = ["ct1","ct2", "t1"]
+        err_list_3 = ["ct1","ct4", "t1"]
+        err_list_4 = ["ct2","ct4", "t1"]
+        err_list_5 = ["ct1", "ct2","ct4", "t1"]
+        self.__join_check(err_list_1, -1)
+        tdLog.printNoPrefix(f"==========err sql condition check in {err_list_1} over==========")
+        self.__join_check(err_list_2, -1)
+        tdLog.printNoPrefix(f"==========err sql condition check in {err_list_2} over==========")
+        self.__join_check(err_list_3, -1)
+        tdLog.printNoPrefix(f"==========err sql condition check in {err_list_3} over==========")
+        self.__join_check(err_list_4, -1)
+        tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========")
+        self.__join_check(err_list_5, -1)
+        tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========")
+        self.__join_check(["ct2", "ct4"], -1, join_flag=False)
+        tdLog.printNoPrefix("==========err sql condition check in has no join condition over==========")
+
+        tdSql.error( f"select c1, c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" )
+        tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{INT_COL}=ct4.{INT_COL}" )
+        tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{TS_COL}=ct4.{TS_COL}" )
+        tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{TS_COL}" )
+        tdSql.error( f"select ct2.c1, ct1.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" )
+        tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and c1 is not null " )
+        tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and ct1.c1 is not null " )
+
+
         tbname = ["ct1", "ct2", "ct4", "t1"]
 
         for tb in tbname:
@@ -150,18 +235,18 @@ class TDTestCase:
         now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
         for i in range(rows):
             tdSql.execute(
-                f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )"
+                f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
             )
             tdSql.execute(
-                f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )"
+                f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
             )
             tdSql.execute(
-                f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i},  {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )"
+                f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i},  {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
             )
         tdSql.execute(
             f'''insert into ct1 values
-            ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } )
+            ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
-            ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } )
+            ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
             '''
         )
 
@@ -172,11 +257,11 @@ class TDTestCase:
             ( { now_time +  7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
             (
                 { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
-                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000}
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
                 )
             (
                 { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
-                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000}
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
                 )
             '''
         )
@@ -188,11 +273,11 @@ class TDTestCase:
             ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
             (
                 { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
-                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 }
+                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
                 )
             (
                 { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
-                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 }
+                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
                 )
             '''
         )
@@ -200,7 +285,7 @@ class TDTestCase:
         for i in range(rows):
             insert_data = f'''insert into t1 values
                 ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
-                "binary_{i}", "nchar_{i}", { now_time - 1000 * i } )
+                "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
                 '''
             tdSql.execute(insert_data)
         tdSql.execute(
@@ -210,12 +295,12 @@ class TDTestCase:
             ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
             ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
                 { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
-                "binary_limit-1", "nchar_limit-1", { now_time - 86400000 }
+                "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
                 )
             (
                 { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
                 { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
-                "binary_limit-2", "nchar_limit-2", { now_time - 172800000 }
+                "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
                 )
             '''
         )
@@ -228,7 +313,8 @@ class TDTestCase:
         self.__create_tb()
 
         tdLog.printNoPrefix("==========step2:insert data")
-        self.__insert_data(10)
+        self.rows = 10
+        self.__insert_data(self.rows)
 
         tdLog.printNoPrefix("==========step3:all check")
         self.all_test()

tests/system-test/2-query/sqrt.py

@@ -0,0 +1,551 @@
+import taos
+import sys
+import datetime
+import inspect
+import math
+from util.log import *
+from util.sql import *
+from util.cases import *
+
+
+class TDTestCase:
+    updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , 
+    "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143,
+    "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143}
+    def init(self, conn, powSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+    
+    def prepare_datas(self):
+        tdSql.execute(
+            '''create table stb1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            tags (t1 int)
+            '''
+        )
+        
+        tdSql.execute(
+            '''
+            create table t1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            '''
+        )
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+        for i in range(9):
+            tdSql.execute(
+                f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+        tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
+        tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+        tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
+        tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+
+        tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+        tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+        tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL  ) ")
+
+        tdSql.execute(
+            f'''insert into t1 values
+            ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
+            ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
+            ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
+            ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
+            ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
+            ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
+            ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
+            ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
+            ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
+            ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            '''
+        )
+    
+    def check_result_auto_sqrt(self ,origin_query , pow_query):
+
+        pow_result = tdSql.getResult(pow_query)
+        origin_result = tdSql.getResult(origin_query)
+
+        auto_result =[]
+        
+        for row in origin_result:
+            row_check = []
+            for elem in row:
+                if elem == None:
+                    elem = None
+                elif elem < 0:
+                    elem = None
+                else:
+                    elem = math.sqrt(elem)
+                row_check.append(elem)
+            auto_result.append(row_check)
+
+        check_status = True
+
+        for row_index , row in enumerate(pow_result):
+            for col_index , elem in enumerate(row):
+                if auto_result[row_index][col_index] == None  and not (auto_result[row_index][col_index] == None and elem == None):
+                    check_status = False
+                elif auto_result[row_index][col_index] != None  and (auto_result[row_index][col_index] - elem > 0.00000001):
+                    check_status = False 
+                else:
+                    pass
+        if not check_status:
+            tdLog.notice("sqrt function value has not as expected , sql is \"%s\" "%pow_query )
+            sys.exit(1)
+        else:
+            tdLog.info("sqrt value check pass , it work as expected ,sql is \"%s\"   "%pow_query )
+        
+    def test_errors(self):
+        error_sql_lists = [
+            "select sqrt from t1",
+            # "select sqrt(-+--+c1 ) from t1",
+            # "select +-sqrt(c1) from t1",
+            # "select ++-sqrt(c1) from t1",
+            # "select ++--sqrt(c1) from t1",
+            # "select - -sqrt(c1)*0 from t1",
+            # "select sqrt(tbname+1) from t1 ",
+            "select sqrt(123--123)==1 from t1",
+            "select sqrt(c1) as 'd1' from t1",
+            "select sqrt(c1 ,c2) from t1",
+            "select sqrt(c1 ,NULL ) from t1",
+            "select sqrt(,) from t1;",
+            "select sqrt(sqrt(c1) ab from t1)",
+            "select sqrt(c1 ) as int from t1",
+            "select sqrt from stb1",
+            # "select sqrt(-+--+c1) from stb1",
+            # "select +-sqrt(c1) from stb1",
+            # "select ++-sqrt(c1) from stb1",
+            # "select ++--sqrt(c1) from stb1",
+            # "select - -sqrt(c1)*0 from stb1",
+            # "select sqrt(tbname+1) from stb1 ",
+            "select sqrt(123--123)==1 from stb1",
+            "select sqrt(c1) as 'd1' from stb1",
+            "select sqrt(c1 ,c2 ) from stb1",
+            "select sqrt(c1 ,NULL) from stb1",
+            "select sqrt(,) from stb1;",
+            "select sqrt(sqrt(c1) ab from stb1)",
+            "select sqrt(c1) as int from stb1"
+        ]
+        for error_sql in error_sql_lists:
+            tdSql.error(error_sql)
+    
+    def support_types(self):
+        type_error_sql_lists = [
+            "select sqrt(ts) from t1" , 
+            "select sqrt(c7) from t1",
+            "select sqrt(c8) from t1",
+            "select sqrt(c9) from t1",
+            "select sqrt(ts) from ct1" , 
+            "select sqrt(c7) from ct1",
+            "select sqrt(c8) from ct1",
+            "select sqrt(c9) from ct1",
+            "select sqrt(ts) from ct3" , 
+            "select sqrt(c7) from ct3",
+            "select sqrt(c8) from ct3",
+            "select sqrt(c9) from ct3",
+            "select sqrt(ts) from ct4" , 
+            "select sqrt(c7) from ct4",
+            "select sqrt(c8) from ct4",
+            "select sqrt(c9) from ct4",
+            "select sqrt(ts) from stb1" , 
+            "select sqrt(c7) from stb1",
+            "select sqrt(c8) from stb1",
+            "select sqrt(c9) from stb1" ,
+
+            "select sqrt(ts) from stbbb1" , 
+            "select sqrt(c7) from stbbb1",
+
+            "select sqrt(ts) from tbname",
+            "select sqrt(c9) from tbname"
+
+        ]
+        
+        for type_sql in type_error_sql_lists:
+            tdSql.error(type_sql)
+        
+        
+        type_sql_lists = [
+            "select sqrt(c1) from t1",
+            "select sqrt(c2) from t1",
+            "select sqrt(c3) from t1",
+            "select sqrt(c4) from t1",
+            "select sqrt(c5) from t1",
+            "select sqrt(c6) from t1",
+
+            "select sqrt(c1) from ct1",
+            "select sqrt(c2) from ct1",
+            "select sqrt(c3) from ct1",
+            "select sqrt(c4) from ct1",
+            "select sqrt(c5) from ct1",
+            "select sqrt(c6) from ct1",
+
+            "select sqrt(c1) from ct3",
+            "select sqrt(c2) from ct3",
+            "select sqrt(c3) from ct3",
+            "select sqrt(c4) from ct3",
+            "select sqrt(c5) from ct3",
+            "select sqrt(c6) from ct3",
+
+            "select sqrt(c1) from stb1",
+            "select sqrt(c2) from stb1",
+            "select sqrt(c3) from stb1",
+            "select sqrt(c4) from stb1",
+            "select sqrt(c5) from stb1",
+            "select sqrt(c6) from stb1",
+
+            "select sqrt(c6) as alisb from stb1", 
+            "select sqrt(c6) alisb from stb1", 
+        ]
+
+        for type_sql in type_sql_lists:
+            tdSql.query(type_sql)
+    
+    def basic_sqrt_function(self):
+
+        # basic query 
+        tdSql.query("select c1 from ct3")
+        tdSql.checkRows(0)
+        tdSql.query("select c1 from t1")
+        tdSql.checkRows(12)
+        tdSql.query("select c1 from stb1")
+        tdSql.checkRows(25)
+
+        # used for empty table  , ct3 is empty
+        tdSql.query("select sqrt(c1) from ct3")
+        tdSql.checkRows(0)
+        tdSql.query("select sqrt(c2) from ct3")
+        tdSql.checkRows(0)
+        tdSql.query("select sqrt(c3) from ct3")
+        tdSql.checkRows(0)
+        tdSql.query("select sqrt(c4) from ct3")
+        tdSql.checkRows(0)
+        tdSql.query("select sqrt(c5) from ct3")
+        tdSql.checkRows(0)
+        tdSql.query("select sqrt(c6) from ct3")
+        tdSql.checkRows(0)
+
+
+        # # used for regular table
+        tdSql.query("select sqrt(c1) from t1")
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(1 , 0, 1.000000000)
+        tdSql.checkData(3 , 0, 1.732050808)
+        tdSql.checkData(5 , 0, None)
+
+        tdSql.query("select c1, c2, c3 , c4, c5 from t1")
+        tdSql.checkData(1, 4, 1.11000)
+        tdSql.checkData(3, 3, 33)
+        tdSql.checkData(5, 4, None)
+
+        tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1")
+        tdSql.checkData(1, 5, 1.11000)
+        tdSql.checkData(3, 4, 33)
+        tdSql.checkData(5, 5, None)
+
+        self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from t1")
+        
+        # used for sub table
+        tdSql.query("select c2 ,sqrt(c2) from ct1")
+        tdSql.checkData(0, 1, 298.140906284)
+        tdSql.checkData(1 , 1, 278.885281074)
+        tdSql.checkData(3 , 1, 235.701081881)
+        tdSql.checkData(4 , 1, 0.000000000)
+
+        tdSql.query("select c1, c5 ,sqrt(c5) from ct4")
+        tdSql.checkData(0 , 2, None)
+        tdSql.checkData(1 , 2, 2.979932904)
+        tdSql.checkData(2 , 2, 2.787471970)
+        tdSql.checkData(3 , 2, 2.580697551)
+        tdSql.checkData(5 , 2, None)
+
+        self.check_result_auto_sqrt( "select c1, c2, c3 , c4, c5 from ct1", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c4), sqrt(c5) from ct1")
+       
+        # nest query for sqrt functions
+        tdSql.query("select c4  , sqrt(c4) ,sqrt(sqrt(c4)) , sqrt(sqrt(sqrt(c4))) from ct1;")
+        tdSql.checkData(0 , 0 , 88)
+        tdSql.checkData(0 , 1 , 9.380831520)
+        tdSql.checkData(0 , 2 , 3.062814314)
+        tdSql.checkData(0 , 3 , 1.750089802)
+
+        tdSql.checkData(1 , 0 , 77)
+        tdSql.checkData(1 , 1 , 8.774964387)
+        tdSql.checkData(1 , 2 , 2.962256638)
+        tdSql.checkData(1 , 3 , 1.721120750)
+
+        tdSql.checkData(11 , 0 , -99)
+        tdSql.checkData(11 , 1 , None)
+        tdSql.checkData(11 , 2 , None)
+        tdSql.checkData(11 , 3 , None)
+
+        # used for stable table 
+        
+        tdSql.query("select sqrt(c1) from stb1")
+        tdSql.checkRows(25)
+      
+
+        # used for not exists table
+        tdSql.error("select sqrt(c1) from stbbb1")
+        tdSql.error("select sqrt(c1) from tbname")
+        tdSql.error("select sqrt(c1) from ct5")
+
+        # mix with common col 
+        tdSql.query("select c1, sqrt(c1) from ct1")
+        tdSql.checkData(0 , 0 ,8)
+        tdSql.checkData(0 , 1 ,2.828427125)
+        tdSql.checkData(4 , 0 ,0)
+        tdSql.checkData(4 , 1 ,0.000000000)
+        tdSql.query("select c2, sqrt(c2) from ct4")
+        tdSql.checkData(0 , 0 , None)
+        tdSql.checkData(0 , 1 ,None)
+        tdSql.checkData(4 , 0 ,55555)
+        tdSql.checkData(4 , 1 ,235.701081881)
+        tdSql.checkData(5 , 0 ,None)
+        tdSql.checkData(5 , 1 ,None)
+
+        # mix with common functions
+        tdSql.query("select c1, sqrt(c1),sqrt(c1), sqrt(sqrt(c1)) from ct4 ")
+        tdSql.checkData(0 , 0 ,None)
+        tdSql.checkData(0 , 1 ,None)
+        tdSql.checkData(0 , 2 ,None)
+        tdSql.checkData(0 , 3 ,None)
+        
+        tdSql.checkData(3 , 0 , 6)
+        tdSql.checkData(3 , 1 ,2.449489743)
+        tdSql.checkData(3 , 2 ,2.449489743)
+        tdSql.checkData(3 , 3 ,1.565084580)
+
+        tdSql.query("select c1, sqrt(c1),c5, floor(c5) from stb1 ")
+
+        # # mix with agg functions , not support
+        tdSql.error("select c1, sqrt(c1),c5, count(c5) from stb1 ")
+        tdSql.error("select c1, sqrt(c1),c5, count(c5) from ct1 ")
+        tdSql.error("select sqrt(c1), count(c5) from stb1 ")
+        tdSql.error("select sqrt(c1), count(c5) from ct1 ")
+        tdSql.error("select c1, count(c5) from ct1 ")
+        tdSql.error("select c1, count(c5) from stb1 ")
+
+        # agg functions mix with agg functions
+
+        tdSql.query("select max(c5), count(c5) from stb1")
+        tdSql.query("select max(c5), count(c5) from ct1")
+
+        
+        # bug fix for count
+        tdSql.query("select count(c1) from ct4 ")
+        tdSql.checkData(0,0,9)
+        tdSql.query("select count(*) from ct4 ")
+        tdSql.checkData(0,0,12)
+        tdSql.query("select count(c1) from stb1 ")
+        tdSql.checkData(0,0,22)
+        tdSql.query("select count(*) from stb1 ")
+        tdSql.checkData(0,0,25)
+
+        # # bug fix for compute 
+        tdSql.query("select c1, sqrt(c1) -0 ,sqrt(c1-4)-0 from ct4 ")
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(0, 2, None)
+        tdSql.checkData(1, 0, 8)
+        tdSql.checkData(1, 1, 2.828427125)
+        tdSql.checkData(1, 2, 2.000000000)
+
+        tdSql.query(" select c1, sqrt(c1) -0 ,sqrt(c1-0.1)-0.1 from ct4")
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(0, 2, None)
+        tdSql.checkData(1, 0, 8)
+        tdSql.checkData(1, 1, 2.828427125)
+        tdSql.checkData(1, 2, 2.710693865)
+
+        tdSql.query("select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1")
+
+    def test_big_number(self):
+
+        tdSql.query("select c1, sqrt(100000000) from ct1")  # bigint to double data overflow
+        tdSql.checkData(4, 1, 10000.000000000)
+
+
+        tdSql.query("select c1, sqrt(10000000000000) from ct1")  # bigint to double data overflow
+        tdSql.checkData(4, 1, 3162277.660168380)
+
+        tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000) from ct1")  # bigint to double data overflow
+        tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value
+        tdSql.checkData(1, 1, 3162277660171.025390625)
+
+        tdSql.query("select c1, sqrt(10000000000000000000000000000000000) from ct1")  # bigint to double data overflow
+        tdSql.query("select c1, sqrt(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value
+        tdSql.checkData(4, 1, 100000000000000000.000000000)
+
+        tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000) from ct1")  # bigint to double data overflow
+        tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value
+
+        tdSql.checkData(4, 1, 100000000000000000000.000000000)
+
+        tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1")  # bigint to double data overflow
+
+    def pow_base_test(self):
+
+        # base is an regular number ,int or double
+        tdSql.query("select c1, sqrt(c1) from ct1")
+        tdSql.checkData(0, 1,2.828427125)
+        tdSql.checkRows(13)
+
+        # # bug for compute in functions
+        # tdSql.query("select c1, abs(1/0) from ct1") 
+        # tdSql.checkData(0, 0, 8)
+        # tdSql.checkData(0, 1, 1)
+
+        tdSql.query("select c1, sqrt(1) from ct1")  
+        tdSql.checkData(0, 1, 1.000000000)
+        tdSql.checkRows(13)
+
+        # two cols start sqrt(x,y)
+        tdSql.query("select c1,c2, sqrt(c2) from ct1") 
+        tdSql.checkData(0, 2, 298.140906284)
+        tdSql.checkData(1, 2, 278.885281074)
+        tdSql.checkData(4, 2, 0.000000000)
+
+    def abs_func_filter(self):
+        tdSql.execute("use db")
+        tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1>5 ")
+        tdSql.checkRows(3)
+        tdSql.checkData(0,0,8)
+        tdSql.checkData(0,1,8.000000000)
+        tdSql.checkData(0,2,8.000000000)
+        tdSql.checkData(0,3,7.900000000)
+        tdSql.checkData(0,4,3.000000000)
+
+        tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0,0,5)
+        tdSql.checkData(0,1,5.000000000)
+        tdSql.checkData(0,2,5.000000000)
+        tdSql.checkData(0,3,4.900000000)
+        tdSql.checkData(0,4,2.000000000)
+
+        tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0,0,5)
+        tdSql.checkData(0,1,5.000000000)
+        tdSql.checkData(0,2,5.000000000)
+        tdSql.checkData(0,3,4.900000000)
+        tdSql.checkData(0,4,2.000000000)
+
+        tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=sqrt(c1) limit 1 ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0,0,1)
+        tdSql.checkData(0,1,11111)
+        tdSql.checkData(0,2,1.000000000)
+        tdSql.checkData(0,3,1.000000000)
+        tdSql.checkData(0,4,0.900000000)
+        tdSql.checkData(0,5,1.000000000)
+        
+    def pow_Arithmetic(self):
+        pass
+    
+    def check_boundary_values(self):
+
+        tdSql.execute("drop database if exists bound_test")
+        tdSql.execute("create database if not exists bound_test")
+        time.sleep(3)
+        tdSql.execute("use bound_test")
+        tdSql.execute(
+            "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);"
+        )
+        tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )')
+        tdSql.execute(
+                f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+            )
+        tdSql.execute(
+                f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+            )
+        tdSql.execute(
+                f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+            )
+        tdSql.execute(
+                f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+            )
+        tdSql.error(
+                f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+            )
+        self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from sub1_bound")
+       
+        self.check_result_auto_sqrt( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c3), sqrt(c2) ,sqrt(c1) from sub1_bound")
+
+        self.check_result_auto_sqrt("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))  nest_col_func from sub1_bound" , "select sqrt(abs(c1)) from sub1_bound" )
+        
+        # check basic elem for table per row
+        tdSql.query("select sqrt(abs(c1)) ,sqrt(abs(c2)) , sqrt(abs(c3)) , sqrt(abs(c4)), sqrt(abs(c5)), sqrt(abs(c6)) from sub1_bound ")
+        tdSql.checkData(0,0,math.sqrt(2147483647))
+        tdSql.checkData(0,1,math.sqrt(9223372036854775807))
+        tdSql.checkData(0,2,math.sqrt(32767))
+        tdSql.checkData(0,3,math.sqrt(127))
+        tdSql.checkData(0,4,math.sqrt(339999995214436424907732413799364296704.00000))
+        tdSql.checkData(1,0,math.sqrt(2147483647))
+        tdSql.checkData(1,1,math.sqrt(9223372036854775807))
+        tdSql.checkData(1,2,math.sqrt(32767))
+        tdSql.checkData(1,3,math.sqrt(127))
+        tdSql.checkData(1,4,math.sqrt(339999995214436424907732413799364296704.00000))
+        tdSql.checkData(3,0,math.sqrt(2147483646))
+        tdSql.checkData(3,1,math.sqrt(9223372036854775806))
+        tdSql.checkData(3,2,math.sqrt(32766))
+        tdSql.checkData(3,3,math.sqrt(126))
+        tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000))
+
+        # check  + - * / in functions
+        tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ")
+        tdSql.checkData(0,0,math.sqrt(2147483648.000000000))
+        tdSql.checkData(0,1,math.sqrt(9223372036854775807))
+        tdSql.checkData(0,2,math.sqrt(32767.000000000))
+        tdSql.checkData(0,3,math.sqrt(63.500000000))
+      
+    
+    def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table ==============")
+        
+        self.prepare_datas()
+
+        tdLog.printNoPrefix("==========step2:test errors ==============")    
+
+        self.test_errors()
+        
+        tdLog.printNoPrefix("==========step3:support types ============") 
+
+        self.support_types()
+
+        tdLog.printNoPrefix("==========step4: sqrt basic query ============") 
+
+        self.basic_sqrt_function()
+
+        tdLog.printNoPrefix("==========step5: big number sqrt query ============") 
+
+        self.test_big_number()
+
+        tdLog.printNoPrefix("==========step6: base  number for sqrt query ============") 
+
+        self.pow_base_test()
+
+        tdLog.printNoPrefix("==========step7: sqrt boundary query ============") 
+
+        self.check_boundary_values()
+
+        tdLog.printNoPrefix("==========step8: sqrt filter query ============") 
+
+        self.abs_func_filter()
+
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -24,3 +24,4 @@ python3 ./test.py -f 2-query/floor.py
 python3 ./test.py -f 2-query/round.py
 python3 ./test.py -f 2-query/log.py
 python3 ./test.py -f 2-query/pow.py
+python3 ./test.py -f 2-query/sqrt.py