Merge branch '3.0' into feature/qnode

cmake/cmake.options

@@ -146,6 +146,6 @@ option(
 option(
    BUILD_WITH_INVERTEDINDEX
    "If use invertedIndex" 
-   OFF 
+   ON 
 )
 

include/client/taos.h

@@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
 typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
-  void     *buffer;
+  void *    buffer;
   uintptr_t buffer_length;
-  int32_t  *length;
+  int32_t * length;
-  char     *is_null;
+  char *    is_null;
   int       num;
 } TAOS_MULTI_BIND;
 
@@ -130,7 +130,7 @@ DLL_EXPORT void       taos_cleanup(void);
 DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT setConfRet taos_set_config(const char *config);
 DLL_EXPORT int        taos_init(void);
-DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
+DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
@@ -147,17 +147,17 @@ DLL_EXPORT int        taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name
 DLL_EXPORT int        taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
 DLL_EXPORT int        taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
 
-DLL_EXPORT int       taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
+DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
-DLL_EXPORT int       taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
+DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
-DLL_EXPORT int       taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
+DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
+DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
-DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
+DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt);
-DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
+DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
-DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT char *    taos_stmt_errstr(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
@@ -179,11 +179,11 @@ DLL_EXPORT bool        taos_is_update_query(TAOS_RES *res);
 DLL_EXPORT int         taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
-DLL_EXPORT int        *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
+DLL_EXPORT int *       taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
 DLL_EXPORT int         taos_validate_sql(TAOS *taos, const char *sql);
 DLL_EXPORT void        taos_reset_current_db(TAOS *taos);
 
-DLL_EXPORT int      *taos_fetch_lengths(TAOS_RES *res);
+DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
 DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
 
 DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
@@ -204,7 +204,7 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
 #endif
 
-DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
+DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
 
 /* --------------------------TMQ INTERFACE------------------------------- */
@@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
-DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
+DLL_EXPORT char **     tmq_list_to_c_array(const tmq_list_t *);
 
 DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
@@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
+DLL_EXPORT TAOS_RES *     tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
 DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
@@ -260,7 +260,7 @@ enum tmq_conf_res_t {
 
 typedef enum tmq_conf_res_t tmq_conf_res_t;
 
-DLL_EXPORT tmq_conf_t    *tmq_conf_new();
+DLL_EXPORT tmq_conf_t *   tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);

include/common/tdatablock.h

@@ -71,20 +71,14 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
 #define colDataGetData(p1_, r_) \
   ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
 
-static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
+#define IS_JSON_NULL(type,data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL)
-  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
-    if (colDataIsNull_var(pColumnInfoData, row)) {
-      return true;
-    }
-    char* data = colDataGetVarData(pColumnInfoData, row);
-    return (*data == TSDB_DATA_TYPE_NULL);
-  }
 
+static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
   if (!pColumnInfoData->hasNull) {
     return false;
   }
 
-  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
@@ -232,7 +226,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
 void  blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress);
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
 
-void blockDebugShowData(const SArray* dataBlocks);
+void blockDebugShowData(const SArray* dataBlocks, const char* flag);
 
 int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
                                     tb_uid_t suid);

include/common/tdataformat.h

@@ -34,6 +34,7 @@ typedef struct SValue        SValue;
 typedef struct SColVal       SColVal;
 typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
+typedef struct SColData      SColData;
 typedef struct STagVal       STagVal;
 typedef struct STag          STag;
 
@@ -41,6 +42,9 @@ typedef struct STag          STag;
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
 void    tTSchemaDestroy(STSchema *pTSchema);
 
+// SValue
+int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type);
+
 // STSRow2
 #define COL_VAL_NONE(CID)     ((SColVal){.cid = (CID), .isNone = 1})
 #define COL_VAL_NULL(CID)     ((SColVal){.cid = (CID), .isNull = 1})
@@ -166,6 +170,12 @@ struct STag {
 };
 #pragma pack(pop)
 
+struct SColData {
+  int16_t  cid;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
 #if 1  //================================================================================================================================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP

include/libs/index/index.h

@@ -193,8 +193,9 @@ void indexInit();
 
 /* index filter */
 typedef struct SIndexMetaArg {
-  void*    metaHandle;
   void*    metaEx;
+  void*    idx;
+  void*    ivtIdx;
   uint64_t suid;
 } SIndexMetaArg;
 

include/libs/nodes/nodes.h

@@ -188,7 +188,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN_JOIN,
   QUERY_NODE_LOGIC_PLAN_AGG,
   QUERY_NODE_LOGIC_PLAN_PROJECT,
-  QUERY_NODE_LOGIC_PLAN_VNODE_MODIF,
+  QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY,
   QUERY_NODE_LOGIC_PLAN_EXCHANGE,
   QUERY_NODE_LOGIC_PLAN_MERGE,
   QUERY_NODE_LOGIC_PLAN_WINDOW,
@@ -210,7 +210,8 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
   QUERY_NODE_PHYSICAL_PLAN_MERGE,
   QUERY_NODE_PHYSICAL_PLAN_SORT,
-  QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL,
@@ -223,6 +224,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_PARTITION,
   QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
   QUERY_NODE_PHYSICAL_PLAN_INSERT,
+  QUERY_NODE_PHYSICAL_PLAN_DELETE,
   QUERY_NODE_PHYSICAL_SUBPLAN,
   QUERY_NODE_PHYSICAL_PLAN
 } ENodeType;

include/libs/nodes/plannodes.h

@@ -37,29 +37,30 @@ typedef struct SLogicNode {
 typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType;
 
 typedef struct SScanLogicNode {
-  SLogicNode         node;
+  SLogicNode    node;
-  SNodeList*         pScanCols;
+  SNodeList*    pScanCols;
-  SNodeList*         pScanPseudoCols;
+  SNodeList*    pScanPseudoCols;
-  struct STableMeta* pMeta;
+  int8_t        tableType;
-  SVgroupsInfo*      pVgroupList;
+  uint64_t      tableId;
-  EScanType          scanType;
+  SVgroupsInfo* pVgroupList;
-  uint8_t            scanSeq[2];  // first is scan count, and second is reverse scan count
+  EScanType     scanType;
-  STimeWindow        scanRange;
+  uint8_t       scanSeq[2];  // first is scan count, and second is reverse scan count
-  SName              tableName;
+  STimeWindow   scanRange;
-  bool               showRewrite;
+  SName         tableName;
-  double             ratio;
+  bool          showRewrite;
-  SNodeList*         pDynamicScanFuncs;
+  double        ratio;
-  int32_t            dataRequired;
+  SNodeList*    pDynamicScanFuncs;
-  int64_t            interval;
+  int32_t       dataRequired;
-  int64_t            offset;
+  int64_t       interval;
-  int64_t            sliding;
+  int64_t       offset;
-  int8_t             intervalUnit;
+  int64_t       sliding;
-  int8_t             slidingUnit;
+  int8_t        intervalUnit;
-  SNode*             pTagCond;
+  int8_t        slidingUnit;
-  int8_t             triggerType;
+  SNode*        pTagCond;
-  int64_t            watermark;
+  int8_t        triggerType;
-  int16_t            tsColId;
+  int64_t       watermark;
-  double             filesFactor;
+  int16_t       tsColId;
+  double        filesFactor;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -85,12 +86,20 @@ typedef struct SProjectLogicNode {
   int64_t    soffset;
 } SProjectLogicNode;
 
-typedef struct SVnodeModifLogicNode {
+typedef enum EModifyTableType { MODIFY_TABLE_TYPE_INSERT = 1, MODIFY_TABLE_TYPE_DELETE } EModifyTableType;
-  SLogicNode     node;
+
-  int32_t        msgType;
+typedef struct SVnodeModifyLogicNode {
-  SArray*        pDataBlocks;
+  SLogicNode       node;
-  SVgDataBlocks* pVgDataBlocks;
+  EModifyTableType modifyType;
-} SVnodeModifLogicNode;
+  int32_t          msgType;
+  SArray*          pDataBlocks;
+  SVgDataBlocks*   pVgDataBlocks;
+  SNode*           pModifyRows;  // SColumnNode
+  uint64_t         tableId;
+  int8_t           tableType;  // table type
+  char             tableFName[TSDB_TABLE_FNAME_LEN];
+  STimeWindow      deleteTimeRange;
+} SVnodeModifyLogicNode;
 
 typedef struct SExchangeLogicNode {
   SLogicNode node;
@@ -106,28 +115,30 @@ typedef struct SMergeLogicNode {
 
 typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
 
-typedef enum EStreamIntervalAlgorithm {
+typedef enum EIntervalAlgorithm {
-  STREAM_INTERVAL_ALGO_FINAL = 1,
+  INTERVAL_ALGO_HASH = 1,
-  STREAM_INTERVAL_ALGO_SEMI,
+  INTERVAL_ALGO_SORT_MERGE,
-  STREAM_INTERVAL_ALGO_SINGLE
+  INTERVAL_ALGO_STREAM_FINAL,
-} EStreamIntervalAlgorithm;
+  INTERVAL_ALGO_STREAM_SEMI,
+  INTERVAL_ALGO_STREAM_SINGLE,
+} EIntervalAlgorithm;
 
 typedef struct SWindowLogicNode {
-  SLogicNode               node;
+  SLogicNode         node;
-  EWindowType              winType;
+  EWindowType        winType;
-  SNodeList*               pFuncs;
+  SNodeList*         pFuncs;
-  int64_t                  interval;
+  int64_t            interval;
-  int64_t                  offset;
+  int64_t            offset;
-  int64_t                  sliding;
+  int64_t            sliding;
-  int8_t                   intervalUnit;
+  int8_t             intervalUnit;
-  int8_t                   slidingUnit;
+  int8_t             slidingUnit;
-  int64_t                  sessionGap;
+  int64_t            sessionGap;
-  SNode*                   pTspk;
+  SNode*             pTspk;
-  SNode*                   pStateExpr;
+  SNode*             pStateExpr;
-  int8_t                   triggerType;
+  int8_t             triggerType;
-  int64_t                  watermark;
+  int64_t            watermark;
-  double                   filesFactor;
+  double             filesFactor;
-  EStreamIntervalAlgorithm stmInterAlgo;
+  EIntervalAlgorithm intervalAlgo;
 } SWindowLogicNode;
 
 typedef struct SFillLogicNode {
@@ -308,6 +319,7 @@ typedef struct SIntervalPhysiNode {
   int8_t           slidingUnit;
 } SIntervalPhysiNode;
 
+typedef SIntervalPhysiNode SSortMergeIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode;
@@ -370,6 +382,14 @@ typedef struct SDataInserterNode {
   char*         pData;
 } SDataInserterNode;
 
+typedef struct SDataDeleterNode {
+  SDataSinkNode sink;
+  uint64_t      tableId;
+  int8_t        tableType;  // table type
+  char          tableFName[TSDB_TABLE_FNAME_LEN];
+  STimeWindow   deleteTimeRange;
+} SDataDeleterNode;
+
 typedef struct SSubplan {
   ENodeType      type;
   SSubplanId     id;  // unique id of the subplan

include/util/tdef.h

@@ -129,7 +129,7 @@ typedef enum EOperatorType {
   OP_TYPE_SUB,
   OP_TYPE_MULTI,
   OP_TYPE_DIV,
-  OP_TYPE_MOD,
+  OP_TYPE_REM,
   // unary arithmetic operator
   OP_TYPE_MINUS,
   OP_TYPE_ASSIGN,

source/common/src/tdatablock.c

@@ -109,7 +109,7 @@ int32_t getJsonValueLen(const char *data) {
     dataLen = DOUBLE_BYTES + CHAR_BYTES;
   } else if (*data == TSDB_DATA_TYPE_BOOL) {
     dataLen = CHAR_BYTES + CHAR_BYTES;
-  } else if (*data == TD_TAG_JSON) {   // json string
+  } else if (*data & TD_TAG_JSON) {   // json string
     dataLen = ((STag*)(data))->len;
   } else {
     ASSERT(0);
@@ -1488,7 +1488,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
   return buf;
 }
 
-void blockDebugShowData(const SArray* dataBlocks) {
+void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
   char    pBuf[128] = {0};
   int32_t sz = taosArrayGetSize(dataBlocks);
   for (int32_t i = 0; i < sz; i++) {
@@ -1496,7 +1496,7 @@ void blockDebugShowData(const SArray* dataBlocks) {
     int32_t      colNum = pDataBlock->info.numOfCols;
     int32_t      rows = pDataBlock->info.rows;
     for (int32_t j = 0; j < rows; j++) {
-      printf("|");
+      printf("%s |", flag);
       for (int32_t k = 0; k < colNum; k++) {
         SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
         void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
@@ -1521,8 +1521,11 @@ void blockDebugShowData(const SArray* dataBlocks) {
           case TSDB_DATA_TYPE_UBIGINT:
             printf(" %15lu |", *(uint64_t*)var);
             break;
+          case TSDB_DATA_TYPE_FLOAT:
+            printf(" %15f |", *(float*)var);
+            break;
           case TSDB_DATA_TYPE_DOUBLE:
-            printf(" %15f |", *(double*)var);
+            printf(" %15lf |", *(double*)var);
             break;
         }
       }
@@ -1550,8 +1553,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
     bufSize += sizeof(SSubmitBlk);
   }
 
-  ASSERT(bufSize < 3 * 1024 * 1024);
-
   *pReq = taosMemoryCalloc(1, bufSize);
   if (!(*pReq)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -1562,7 +1563,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
   int32_t     msgLen = sizeof(SSubmitReq);
   int32_t     numOfBlks = 0;
   SRowBuilder rb = {0};
-  tdSRowInit(&rb, pTSchema->version);  // TODO: use the latest version
+  tdSRowInit(&rb, pTSchema->version);
 
   for (int32_t i = 0; i < sz; ++i) {
     SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
@@ -1580,18 +1581,17 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
     pSubmitBlk->uid = pDataBlock->info.groupId;
     pSubmitBlk->numOfRows = rows;
 
-    ++numOfBlks;
-
     msgLen += sizeof(SSubmitBlk);
     int32_t dataLen = 0;
     for (int32_t j = 0; j < rows; ++j) {                     // iterate by row
       tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen));  // set row buf
-      printf("|");
       bool isStartKey = false;
       int32_t offset = 0;
       for (int32_t k = 0; k < colNum; ++k) {  // iterate by column
         SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
-        void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+        STColumn*         pCol = &pTSchema->columns[k];
+        ASSERT(pCol->type == pColInfoData->info.type);
+        void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
         switch (pColInfoData->info.type) {
           case TSDB_DATA_TYPE_TIMESTAMP:
             if (!isStartKey) {
@@ -1600,29 +1600,29 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
                                   offset, k);
 
             } else {
-              tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k);
+              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k);
             }
             break;
           case TSDB_DATA_TYPE_NCHAR: {
-            tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, offset, k);
+            tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, offset, k);
             break;
           }
           case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-            tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, offset, k);
+            tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, offset, k);
             break;
           }
           case TSDB_DATA_TYPE_VARBINARY:
           case TSDB_DATA_TYPE_DECIMAL:
           case TSDB_DATA_TYPE_BLOB:
           case TSDB_DATA_TYPE_MEDIUMBLOB:
-            printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type);
+            uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type);
             TASSERT(0);
             break;
           default:
             if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) {
-              tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k);
+              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k);
             } else {
-              printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
+              uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
               TASSERT(0);
             }
             break;
@@ -1630,7 +1630,13 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
         offset += TYPE_BYTES[pColInfoData->info.type];
       }
       dataLen += TD_ROW_LEN(rb.pBuf);
+#ifdef TD_DEBUG_PRINT_ROW
+      tdSRowPrint(rb.pBuf, pTSchema, __func__);
+#endif
     }
+
+    ++numOfBlks;
+
     pSubmitBlk->dataLen = dataLen;
     msgLen += pSubmitBlk->dataLen;
   }

source/common/src/tdataformat.c

@@ -36,8 +36,6 @@ typedef struct {
 #define GET_BIT1(p, i)     (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
 #define GET_BIT2(p, i)     (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3))
 
-static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2);
-
 // SValue
 static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) {
   int32_t n = 0;
@@ -141,6 +139,11 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) {
   return n;
 }
 
+int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) {
+  // TODO
+  return 0;
+}
+
 // STSRow2 ========================================================================
 static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) {
   if (pb) {

source/dnode/vnode/CMakeLists.txt

@@ -36,6 +36,7 @@ target_sources(
 
     # tsdb
     "src/tsdb/tsdbCommit.c"
+    "src/tsdb/tsdbCommit2.c"
     "src/tsdb/tsdbFile.c"
     "src/tsdb/tsdbFS.c"
     "src/tsdb/tsdbOpen.c"

source/dnode/vnode/inc/vnode.h

@@ -56,7 +56,7 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp);
 int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
 int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
-int32_t vnodePreprocessQueryMsg(SVnode * pVnode, SRpcMsg * pMsg);
+int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo);
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
@@ -86,7 +86,7 @@ typedef struct SMetaFltParam {
   tb_uid_t suid;
   int16_t  cid;
   int16_t  type;
-  char    *val;
+  char *   val;
   bool     reverse;
   int (*filterFunc)(void *a, void *b, int16_t type);
 
@@ -121,7 +121,8 @@ int32_t      tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo
 bool         isTsdbCacheLastRow(tsdbReaderT *pReader);
 int32_t      tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
 int32_t      tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list);
-void        *tsdbGetIdx(SMeta *pMeta);
+void *       tsdbGetIdx(SMeta *pMeta);
+void *       tsdbGetIvtIdx(SMeta *pMeta);
 int64_t      tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
 
 bool    tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
@@ -197,7 +198,7 @@ struct SMetaEntry {
   int64_t  version;
   int8_t   type;
   tb_uid_t uid;
-  char    *name;
+  char *   name;
   union {
     struct {
       SSchemaWrapper schemaRow;
@@ -225,17 +226,17 @@ struct SMetaEntry {
 
 struct SMetaReader {
   int32_t    flags;
-  SMeta     *pMeta;
+  SMeta *    pMeta;
   SDecoder   coder;
   SMetaEntry me;
-  void      *pBuf;
+  void *     pBuf;
   int32_t    szBuf;
 };
 
 struct SMTbCursor {
-  TBC        *pDbc;
+  TBC *       pDbc;
-  void       *pKey;
+  void *      pKey;
-  void       *pVal;
+  void *      pVal;
   int32_t     kLen;
   int32_t     vLen;
   SMetaReader mr;

source/dnode/vnode/src/inc/meta.h

@@ -69,12 +69,11 @@ struct SMeta {
   TTB*    pUidIdx;
   TTB*    pNameIdx;
   TTB*    pCtbIdx;
-#ifdef USE_INVERTED_INDEX
+  // ivt idx and idx
   void* pTagIvtIdx;
-#else
+  TTB*  pTagIdx;
-  TTB* pTagIdx;
+  TTB*  pTtlIdx;
-#endif
+
-  TTB*      pTtlIdx;
   TTB*      pSmaIdx;
   SMetaIdx* pIdx;
 };
@@ -117,7 +116,7 @@ typedef struct {
 } SSmaIdxKey;
 
 // metaTable ==================
-int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData,  int8_t type, tb_uid_t uid,
+int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,
                         STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey);
 
 #ifndef META_REFACT

source/dnode/vnode/src/inc/tsdb.h

@@ -34,18 +34,97 @@ extern "C" {
 
 typedef struct TSDBROW TSDBROW;
 typedef struct TSDBKEY TSDBKEY;
+typedef struct TABLEID TABLEID;
 typedef struct SDelOp  SDelOp;
 
 static int tsdbKeyCmprFn(const void *p1, const void *p2);
 
 // tsdbMemTable2.c ==============================================================================================
-typedef struct SMemTable SMemTable;
+typedef struct SMemTable    SMemTable;
+typedef struct SMemData     SMemData;
+typedef struct SMemDataIter SMemDataIter;
 
 int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable);
 void    tsdbMemTableDestroy2(SMemTable *pMemTable);
+int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
+int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
+
+/* SMemDataIter */
+void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
+bool tsdbMemDataIterNext(SMemDataIter *pIter);
+void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
+
+// tsdbCommit2.c ==============================================================================================
+int32_t tsdbBegin2(STsdb *pTsdb);
+int32_t tsdbCommit2(STsdb *pTsdb);
+
+// tsdbFile.c ==============================================================================================
+typedef int32_t          TSDB_FILE_T;
+typedef struct SDFInfo   SDFInfo;
+typedef struct SDFile    SDFile;
+typedef struct SDFileSet SDFileSet;
+
+// SDFile
+void    tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
+void    tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile);
+int     tsdbOpenDFile(SDFile *pDFile, int flags);
+void    tsdbCloseDFile(SDFile *pDFile);
+int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence);
+int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte);
+void    tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm);
+int     tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset);
+int     tsdbRemoveDFile(SDFile *pDFile);
+int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte);
+int     tsdbCopyDFile(SDFile *pSrc, SDFile *pDest);
+int     tsdbEncodeSDFile(void **buf, SDFile *pDFile);
+void   *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile);
+int     tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType);
+int     tsdbUpdateDFileHeader(SDFile *pDFile);
+int     tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo);
+int     tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version);
+
+// SDFileSet
+void  tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver);
+void  tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet);
+int   tsdbEncodeDFileSet(void **buf, SDFileSet *pSet);
+void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet);
+int   tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet);
+void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet);
+int   tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to);
+int   tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader);
+int   tsdbUpdateDFileSetHeader(SDFileSet *pSet);
+int   tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet);
+void  tsdbCloseDFileSet(SDFileSet *pSet);
+int   tsdbOpenDFileSet(SDFileSet *pSet, int flags);
+void  tsdbRemoveDFileSet(SDFileSet *pSet);
+int   tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest);
+void  tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey);
+
+// tsdbFS.c ==============================================================================================
+typedef struct STsdbFS     STsdbFS;
+typedef struct SFSIter     SFSIter;
+typedef struct STsdbFSMeta STsdbFSMeta;
+
+STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg);
+void    *tsdbFreeFS(STsdbFS *pfs);
+int      tsdbOpenFS(STsdb *pRepo);
+void     tsdbCloseFS(STsdb *pRepo);
+void     tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
+int      tsdbEndFSTxn(STsdb *pRepo);
+int      tsdbEndFSTxnWithError(STsdbFS *pfs);
+void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
+// void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
+
+void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
+void       tsdbFSIterSeek(SFSIter *pIter, int fid);
+SDFileSet *tsdbFSIterNext(SFSIter *pIter);
+int        tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
+int        tsdbRLockFS(STsdbFS *pFs);
+int        tsdbWLockFS(STsdbFS *pFs);
+int        tsdbUnLockFS(STsdbFS *pFs);
 
 // tsdbMemTable ================
-typedef struct STsdbRow      STsdbRow;
 typedef struct STbData       STbData;
 typedef struct STsdbMemTable STsdbMemTable;
 typedef struct SMergeInfo    SMergeInfo;
@@ -56,15 +135,6 @@ void tsdbMemTableDestroy(STsdbMemTable *pMemTable);
 int  tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead,
                            SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
 
-// tsdbCommit ================
-
-// tsdbFS ================
-typedef struct STsdbFS STsdbFS;
-
-// tsdbSma ================
-typedef struct SSmaEnv  SSmaEnv;
-typedef struct SSmaEnvs SSmaEnvs;
-
 // structs
 typedef struct {
   int   minFid;
@@ -99,7 +169,7 @@ struct STable {
 #define TABLE_TID(t) (t)->tid
 #define TABLE_UID(t) (t)->uid
 
-int tsdbPrepareCommit(STsdb *pTsdb);
+// int tsdbPrepareCommit(STsdb *pTsdb);
 typedef enum {
   TSDB_FILE_HEAD = 0,  // .head
   TSDB_FILE_DATA,      // .data
@@ -110,7 +180,7 @@ typedef enum {
   TSDB_FILE_META,      // meta
 } E_TSDB_FILE_T;
 
-typedef struct {
+struct SDFInfo {
   uint32_t magic;
   uint32_t fver;
   uint32_t len;
@@ -119,22 +189,22 @@ typedef struct {
   uint32_t offset;
   uint64_t size;
   uint64_t tombSize;
-} SDFInfo;
+};
 
-typedef struct {
+struct SDFile {
   SDFInfo   info;
   STfsFile  f;
   TdFilePtr pFile;
   uint8_t   state;
-} SDFile;
+};
 
-typedef struct {
+struct SDFileSet {
   int      fid;
   int8_t   state;  // -128~127
   uint8_t  ver;    // 0~255, DFileSet version
   uint16_t reserve;
   SDFile   files[TSDB_FILE_MAX];
-} SDFileSet;
+};
 
 struct STbData {
   tb_uid_t   uid;
@@ -159,11 +229,11 @@ struct STsdbMemTable {
   SHashObj  *pHashIdx;
 };
 
-typedef struct {
+struct STsdbFSMeta {
   uint32_t version;       // Commit version from 0 to increase
   int64_t  totalPoints;   // total points
   int64_t  totalStorage;  // Uncompressed total storage
-} STsdbFSMeta;
+};
 
 // ==================
 typedef struct {
@@ -175,9 +245,7 @@ typedef struct {
 struct STsdbFS {
   TdThreadRwlock lock;
 
-  SFSStatus *cstatus;        // current status
+  SFSStatus *cstatus;  // current status
-  SHashObj  *metaCache;      // meta cache
-  SHashObj  *metaCacheComp;  // meta cache for compact
   bool       intxn;
   SFSStatus *nstatus;  // new status
 };
@@ -243,31 +311,21 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
 // tsdbReadImpl
 typedef struct SReadH SReadH;
 
+struct TSDBKEY {
+  int64_t version;
+  TSKEY   ts;
+};
+
 typedef struct {
+  uint64_t suid;
+  uint64_t uid;
   uint32_t len;
   uint32_t offset;
   uint32_t hasLast : 2;
   uint32_t numOfBlocks : 30;
-  uint64_t uid;
+  TSDBKEY  maxKey;
-  TSKEY    maxKey;
 } SBlockIdx;
 
-#ifdef TD_REFACTOR_3
-typedef struct {
-  int64_t last : 1;
-  int64_t offset : 63;
-  int32_t algorithm : 8;
-  int32_t numOfRows : 24;
-  int32_t len;
-  int32_t keyLen;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
-  int16_t numOfSubBlocks;
-  int16_t numOfCols;  // not including timestamp column
-  TSKEY   keyFirst;
-  TSKEY   keyLast;
-} SBlock;
-
-#else
-
 typedef enum {
   TSDB_SBLK_VER_0 = 0,
   TSDB_SBLK_VER_MAX,
@@ -290,16 +348,9 @@ typedef struct {
   int64_t  offset;
   uint64_t aggrStat : 1;
   uint64_t aggrOffset : 63;
-  TSKEY    keyFirst;
+  TSDBKEY  minKey;
-  TSKEY    keyLast;
+  TSDBKEY  maxKey;
-} SBlockV0;
+} SBlock;
-
-#define SBlock SBlockV0  // latest SBlock definition
-
-static FORCE_INLINE bool tsdbIsSupBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 1; }
-static FORCE_INLINE bool tsdbIsSubBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 0; }
-
-#endif
 
 typedef struct {
   int32_t  delimiter;  // For recovery usage
@@ -308,33 +359,13 @@ typedef struct {
   SBlock   blocks[];
 } SBlockInfo;
 
-#ifdef TD_REFACTOR_3
-typedef struct {
-  int16_t  colId;
-  uint16_t bitmap : 1;  // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
-  uint16_t reserve : 15;
-  int32_t  len;
-  uint32_t type : 8;
-  uint32_t offset : 24;
-  int64_t  sum;
-  int64_t  max;
-  int64_t  min;
-  int16_t  maxIndex;
-  int16_t  minIndex;
-  int16_t  numOfNull;
-  uint8_t  offsetH;
-  char     padding[1];
-} SBlockCol;
-#else
 typedef struct {
   int16_t  colId;
   uint16_t type : 6;
   uint16_t blen : 10;  // 0 no bitmap if all rows are NORM, > 0 bitmap length
   uint32_t len;        // data length + bitmap length
   uint32_t offset;
-} SBlockColV0;
+} SBlockCol;
-
-#define SBlockCol SBlockColV0  // latest SBlockCol definition
 
 typedef struct {
   int16_t colId;
@@ -344,31 +375,7 @@ typedef struct {
   int64_t sum;
   int64_t max;
   int64_t min;
-} SAggrBlkColV0;
+} SAggrBlkCol;
-
-#define SAggrBlkCol SAggrBlkColV0  // latest SAggrBlkCol definition
-
-#endif
-
-// Code here just for back-ward compatibility
-static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
-#ifdef TD_REFACTOR_3
-  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
-  pBlockCol->offsetH = (uint8_t)(offset >> 24);
-#else
-  pBlockCol->offset = offset;
-#endif
-}
-
-static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
-#ifdef TD_REFACTOR_3
-  uint32_t offset1 = pBlockCol->offset;
-  uint32_t offset2 = pBlockCol->offsetH;
-  return (offset1 | (offset2 << 24));
-#else
-  return pBlockCol->offset;
-#endif
-}
 
 typedef struct {
   int32_t   delimiter;  // For recovery usage
@@ -409,8 +416,7 @@ struct SReadH {
 #define TSDB_READ_COMP_BUF(rh)  ((rh)->pCBuf)
 #define TSDB_READ_EXBUF(rh)     ((rh)->pExBuf)
 
-#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
+#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM))
-  (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
 
 static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
   switch (blkVer) {
@@ -420,7 +426,7 @@ static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
   }
 }
 
-#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
+#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkCol) * (ncols) + sizeof(TSCKSUM))
 
 static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
   switch (blkVer) {
@@ -518,11 +524,11 @@ static FORCE_INLINE void *taosTZfree(void *ptr) {
 
 void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
 
-static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
+static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t minutes, int8_t precision) {
   if (key < 0) {
-    return (int)((key + 1) / tsTickPerMin[precision] / days - 1);
+    return (int)((key + 1) / tsTickPerMin[precision] / minutes - 1);
   } else {
-    return (int)((key / tsTickPerMin[precision] / days));
+    return (int)((key / tsTickPerMin[precision] / minutes));
   }
 }
 
@@ -546,7 +552,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
 #define TSDB_FILE_STATE_OK   0
 #define TSDB_FILE_STATE_BAD  1
 
-#define TSDB_FILE_INFO(tf)         (&((tf)->info))
 #define TSDB_FILE_F(tf)            (&((tf)->f))
 #define TSDB_FILE_PFILE(tf)        ((tf)->pFile)
 #define TSDB_FILE_FULL_NAME(tf)    (TSDB_FILE_F(tf)->aname)
@@ -564,7 +569,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
 #define TSDB_FILE_IS_OK(tf)        (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
 #define TSDB_FILE_IS_BAD(tf)       (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
 
-typedef int32_t TSDB_FILE_T;
 typedef enum {
   TSDB_FS_VER_0 = 0,
   TSDB_FS_VER_MAX,
@@ -585,129 +589,9 @@ static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) {  // latest v
   }
 }
 
-void  tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
-void  tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile);
-int   tsdbEncodeSDFile(void **buf, SDFile *pDFile);
-void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile);
-int   tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType);
-int   tsdbUpdateDFileHeader(SDFile *pDFile);
-int   tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo);
-int   tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version);
-
-static FORCE_INLINE void tsdbSetDFileInfo(SDFile *pDFile, SDFInfo *pInfo) { pDFile->info = *pInfo; }
-
-static FORCE_INLINE int tsdbOpenDFile(SDFile *pDFile, int flags) {
-  ASSERT(!TSDB_FILE_OPENED(pDFile));
-
-  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
-  if (pDFile->pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return 0;
-}
-
-static FORCE_INLINE void tsdbCloseDFile(SDFile *pDFile) {
-  if (TSDB_FILE_OPENED(pDFile)) {
-    taosCloseFile(&pDFile->pFile);
-    TSDB_FILE_SET_CLOSED(pDFile);
-  }
-}
-
-static FORCE_INLINE int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence) {
-  // ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
-  if (loffset < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return loffset;
-}
-
-static FORCE_INLINE int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte) {
-  ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
-  if (nwrite < nbyte) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return nwrite;
-}
-
-static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm) {
-  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM));
-}
-
-static FORCE_INLINE int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset) {
-  ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t toffset;
-
-  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
-    return -1;
-  }
-
-  ASSERT(pDFile->info.size == toffset);
-
-  if (offset) {
-    *offset = toffset;
-  }
-
-  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
-    return -1;
-  }
-
-  pDFile->info.size += nbyte;
-
-  return (int)nbyte;
-}
-
-static FORCE_INLINE int tsdbRemoveDFile(SDFile *pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
-
-static FORCE_INLINE int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte) {
-  ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
-  if (nread < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return nread;
-}
-
-static FORCE_INLINE int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) {
-  if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
-  return 0;
-}
-
 // =============== SDFileSet
 
-typedef struct {
+#define TSDB_LATEST_FSET_VER    0
-  int      fid;
-  int8_t   state;
-  uint8_t  ver;
-  uint16_t reserve;
-#if 0
-  SDFInfo   info;
-#endif
-  STfsFile  f;
-  TdFilePtr pFile;
-
-} SSFile;  // files split by days with fid
-
-#define TSDB_LATEST_FSET_VER 0
-
 #define TSDB_FSET_FID(s)        ((s)->fid)
 #define TSDB_FSET_STATE(s)      ((s)->state)
 #define TSDB_FSET_VER(s)        ((s)->ver)
@@ -727,55 +611,6 @@ typedef struct {
     }                                                                          \
   } while (0);
 
-void  tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver);
-void  tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet);
-int   tsdbEncodeDFileSet(void **buf, SDFileSet *pSet);
-void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet);
-int   tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet);
-void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet);
-int   tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to);
-int   tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader);
-int   tsdbUpdateDFileSetHeader(SDFileSet *pSet);
-int   tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet);
-
-static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet *pSet) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
-  }
-}
-
-static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet *pSet, int flags) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
-      tsdbCloseDFileSet(pSet);
-      return -1;
-    }
-  }
-  return 0;
-}
-
-static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet *pSet) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
-  }
-}
-
-static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
-      tsdbRemoveDFileSet(pDest);
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) {
-  *minKey = fid * days * tsTickPerMin[precision];
-  *maxKey = *minKey + days * tsTickPerMin[precision] - 1;
-}
-
 static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) {
   for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
     if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
@@ -803,69 +638,26 @@ typedef struct {
 #define FS_VERSION(pfs)        ((pfs)->cstatus->meta.version)
 #define FS_TXN_VERSION(pfs)    ((pfs)->nstatus->meta.version)
 
-typedef struct {
+struct SFSIter {
   int        direction;
   uint64_t   version;  // current FS version
   STsdbFS   *pfs;
   int        index;  // used to position next fset when version the same
   int        fid;    // used to seek when version is changed
   SDFileSet *pSet;
-} SFSIter;
+};
 
 #define TSDB_FS_ITER_FORWARD  TSDB_ORDER_ASC
 #define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
 
-STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg);
-void    *tsdbFreeFS(STsdbFS *pfs);
-int      tsdbOpenFS(STsdb *pRepo);
-void     tsdbCloseFS(STsdb *pRepo);
-void     tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
-int      tsdbEndFSTxn(STsdb *pRepo);
-int      tsdbEndFSTxnWithError(STsdbFS *pfs);
-void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
-// void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
-int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
-
-void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
-void       tsdbFSIterSeek(SFSIter *pIter, int fid);
-SDFileSet *tsdbFSIterNext(SFSIter *pIter);
-int        tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
-
-static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) {
-  int code = taosThreadRwlockRdlock(&(pFs->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) {
-  int code = taosThreadRwlockWrlock(&(pFs->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
-  int code = taosThreadRwlockUnlock(&(pFs->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
 struct TSDBROW {
   int64_t version;
   STSRow2 tsRow;
 };
 
-struct TSDBKEY {
+struct TABLEID {
-  int64_t version;
+  tb_uid_t suid;
-  TSKEY   ts;
+  tb_uid_t uid;
 };
 
 struct SDelOp {
@@ -875,6 +667,25 @@ struct SDelOp {
   SDelOp *pNext;
 };
 
+typedef struct {
+  tb_uid_t suid;
+  tb_uid_t uid;
+  int64_t  version;
+  TSKEY    sKey;
+  TSKEY    eKey;
+} SDelInfo;
+
+struct SMemTable {
+  STsdb  *pTsdb;
+  int32_t nRef;
+  TSDBKEY minKey;
+  TSDBKEY maxKey;
+  int64_t nRows;
+  int64_t nDelOp;
+  SArray *aSkmInfo;
+  SArray *aMemData;
+};
+
 static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
   TSDBKEY *pKey1 = (TSDBKEY *)p1;
   TSDBKEY *pKey2 = (TSDBKEY *)p2;
@@ -894,6 +705,34 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
   return 0;
 }
 
+typedef struct SMemSkipListNode SMemSkipListNode;
+typedef struct SMemSkipList {
+  uint32_t          seed;
+  int32_t           size;
+  int8_t            maxLevel;
+  int8_t            level;
+  SMemSkipListNode *pHead;
+  SMemSkipListNode *pTail;
+} SMemSkipList;
+
+struct SMemData {
+  tb_uid_t     suid;
+  tb_uid_t     uid;
+  TSDBKEY      minKey;
+  TSDBKEY      maxKey;
+  SDelOp      *delOpHead;
+  SDelOp      *delOpTail;
+  SMemSkipList sl;
+};
+
+struct SMemDataIter {
+  SMemData         *pMemData;
+  int8_t            backward;
+  TSDBROW          *pRow;
+  SMemSkipListNode *pNode;  // current node
+  TSDBROW           row;
+};
+
 #endif
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/vnodeInt.h

@@ -87,7 +87,7 @@ int             metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p
 int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
 int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids);
-int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp *pMetaRsp);
+int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
 int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
@@ -104,6 +104,7 @@ int32_t         metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppRea
 int32_t         metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
 int32_t         metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
+void*           metaGetIvtIdx(SMeta* pMeta);
 
 int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
 int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
@@ -112,7 +113,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
 int          tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg);
 int          tsdbClose(STsdb** pTsdb);
 int          tsdbBegin(STsdb* pTsdb);
-int          tsdbCommit(STsdb* pTsdb);
+int32_t      tsdbCommit(STsdb* pTsdb);
 int          tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
 int          tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
 int          tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);

source/dnode/vnode/src/meta/metaOpen.c

@@ -93,7 +93,6 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
   }
 
   // open pTagIdx
-#ifdef USE_INVERTED_INDEX
   // TODO(yihaoDeng), refactor later
   char indexFullPath[128] = {0};
   sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
@@ -104,13 +103,11 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
     goto _err;
   }
 
-#else
   ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
   if (ret < 0) {
     metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
-#endif
 
   // open pTtlIdx
   ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
@@ -141,11 +138,8 @@ _err:
   if (pMeta->pIdx) metaCloseIdx(pMeta);
   if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
   if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
-#ifdef USE_INVERTED_INDEX
   if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
-#else
   if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
-#endif
   if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
   if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
   if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);

source/dnode/vnode/src/meta/metaSma.c

@@ -28,9 +28,9 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
   int         vLen = 0;
   const void *pKey = NULL;
   const void *pVal = NULL;
-  void       *pBuf = NULL;
+  void *      pBuf = NULL;
   int32_t     szBuf = 0;
-  void       *p = NULL;
+  void *      p = NULL;
   SMetaReader mr = {0};
 
   // validate req
@@ -83,8 +83,8 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
 
 static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
-  void    *pKey = NULL;
+  void *   pKey = NULL;
-  void    *pVal = NULL;
+  void *   pVal = NULL;
   int      kLen = 0;
   int      vLen = 0;
   SEncoder coder = {0};

source/dnode/vnode/src/meta/metaTable.c

@@ -15,6 +15,7 @@
 
 #include "meta.h"
 
+static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
 static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
@@ -25,7 +26,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type);
 
-static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
+static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
   pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema));
   if (NULL == pMetaRsp->pSchemas) {
     terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
@@ -43,6 +44,68 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema
   return 0;
 }
 
+static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) {
+#ifdef USE_INVERTED_INDEX
+  if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
+    return -1;
+  }
+  void *      data = pCtbEntry->ctbEntry.pTags;
+  const char *tagName = pSchema->name;
+
+  tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
+  tb_uid_t    tuid = pCtbEntry->uid;
+  const void *pTagData = pCtbEntry->ctbEntry.pTags;
+  int32_t     nTagData = 0;
+
+  SArray *pTagVals = NULL;
+  if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
+    return -1;
+  }
+  char key[512] = {0};
+
+  SIndexMultiTerm *terms = indexMultiTermCreate();
+  int16_t          nCols = taosArrayGetSize(pTagVals);
+  for (int i = 0; i < nCols; i++) {
+    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
+    char     type = pTagVal->type;
+    sprintf(key, "%s_%s", tagName, pTagVal->pKey);
+    int32_t nKey = strlen(key);
+
+    SIndexTerm *term = NULL;
+    if (type == TSDB_DATA_TYPE_NULL) {
+      // handle null value
+    } else if (type == TSDB_DATA_TYPE_NCHAR) {
+      if (pTagVal->nData > 0) {
+        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
+        memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
+        type = TSDB_DATA_TYPE_VARCHAR;
+        term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len);
+      } else if (pTagVal->nData == 0) {
+        char *  val = NULL;
+        int32_t len = 0;
+        // handle NULL key
+      }
+    } else if (type == TSDB_DATA_TYPE_DOUBLE) {
+      double val = *(double *)(&pTagVal->i64);
+      int    len = 0;
+      term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
+    } else if (type == TSDB_DATA_TYPE_BOOL) {
+      int val = *(int *)(&pTagVal->i64);
+      int len = 0;
+      term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
+    }
+    if (term != NULL) {
+      indexMultiTermAdd(terms, term);
+    }
+    memset(key, 0, sizeof(key));
+  }
+  tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
+  indexMultiTermDestroy(terms);
+#endif
+  return -1;
+}
+
 int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   SMetaEntry  me = {0};
   int         kLen = 0;
@@ -341,7 +404,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   return 0;
 }
 
-
 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
   void *          pVal = NULL;
   int             nVal = 0;
@@ -824,28 +886,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   } else {
     // pTagData = pCtbEntry->ctbEntry.pTags;
     // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
+    pTagData = pCtbEntry->ctbEntry.pTags;
+    nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
+    return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn);
   }
-
-  // update tag index
-#ifdef USE_INVERTED_INDEX
-  tb_uid_t suid = pCtbEntry->ctbEntry.suid;
-  tb_uid_t tuid = pCtbEntry->uid;
-
-  SIndexMultiTerm *tmGroup = indexMultiTermCreate();
-
-  SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name),
-                                   pTagData, pTagData == NULL ? 0 : strlen(pTagData));
-  indexMultiTermAdd(tmGroup, tm);
-  int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid);
-  indexMultiTermDestroy(tmGroup);
-#else
   if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
                           pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
     return -1;
   }
   tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
   metaDestroyTagIdxKey(pTagIdxKey);
-#endif
   tDecoderClear(&dc);
   tdbFree(pData);
   return 0;
@@ -930,10 +980,5 @@ _err:
   return -1;
 }
 // refactor later
-void *metaGetIdx(SMeta *pMeta) {
+void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; }
-#ifdef USE_INVERTED_INDEX
+void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; }
-  return pMeta->pTagIvtIdx;
-#else
-  return pMeta->pTagIdx;
-#endif
-}

source/dnode/vnode/src/sma/smaRollup.c

@@ -400,7 +400,11 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3
   }
 
   if (taosArrayGetSize(pResult) > 0) {
-    blockDebugShowData(pResult);
+#if 1
+    char flag[10] = {0};
+    snprintf(flag, 10, "level %" PRIi8, level);
+    blockDebugShowData(pResult, flag);
+#endif
     STsdb      *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
     SSubmitReq *pReq = NULL;
     if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) != 0) {
@@ -444,7 +448,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
   }
 
   if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
-    // TODO: use the proper schema instead of 0, and cache STSchema in cache
+    // TODO: cache STSchema
     STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1);
     if (!pTSchema) {
       terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -86,7 +86,8 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError);
 static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo);
 static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter,
                                       SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update);
-int         tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
+static int  tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
+static int  tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn);
 
 int tsdbBegin(STsdb *pTsdb) {
   if (!pTsdb) return 0;
@@ -100,7 +101,81 @@ int tsdbBegin(STsdb *pTsdb) {
   return 0;
 }
 
-int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
+int32_t tsdbCommit(STsdb *pTsdb) {
+  int32_t    code = 0;
+  SCommitH   commith = {0};
+  SDFileSet *pSet = NULL;
+  int        fid;
+
+  ASSERT(pTsdb->imem == NULL && pTsdb->mem);
+  pTsdb->imem = pTsdb->mem;
+  pTsdb->mem = NULL;
+
+  // start commit
+  tsdbStartCommit(pTsdb);
+  if (tsdbInitCommitH(&commith, pTsdb) < 0) {
+    return -1;
+  }
+
+  // Skip expired memory data and expired FSET
+  tsdbSeekCommitIter(&commith, commith.rtn.minKey);
+  while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) {
+    if (pSet->fid < commith.rtn.minFid) {
+      tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid,
+               TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
+    } else {
+      break;
+    }
+  }
+
+  // commit
+  fid = tsdbNextCommitFid(&(commith));
+  while (true) {
+    // Loop over both on disk and memory
+    if (pSet == NULL && fid == TSDB_IVLD_FID) break;
+
+    if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) {
+      // Only has existing FSET but no memory data to commit in this
+      // existing FSET, only check if file in correct retention
+      if (tsdbApplyRtnOnFSet(pTsdb, pSet, &(commith.rtn)) < 0) {
+        tsdbDestroyCommitH(&commith);
+        return -1;
+      }
+
+      pSet = tsdbFSIterNext(&(commith.fsIter));
+    } else {
+      // Has memory data to commit
+      SDFileSet *pCSet;
+      int        cfid;
+
+      if (pSet == NULL || pSet->fid > fid) {
+        // Commit to a new FSET with fid: fid
+        pCSet = NULL;
+        cfid = fid;
+      } else {
+        // Commit to an existing FSET
+        pCSet = pSet;
+        cfid = pSet->fid;
+        pSet = tsdbFSIterNext(&(commith.fsIter));
+      }
+
+      if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) {
+        tsdbDestroyCommitH(&commith);
+        return -1;
+      }
+
+      fid = tsdbNextCommitFid(&commith);
+    }
+  }
+
+  // end commit
+  tsdbDestroyCommitH(&commith);
+  tsdbEndCommit(pTsdb, TSDB_CODE_SUCCESS);
+
+  return code;
+}
+
+static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
   SDiskID   did;
   SDFileSet nSet = {0};
   STsdbFS  *pfs = REPO_FS(pRepo);
@@ -141,87 +216,15 @@ int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
   return 0;
 }
 
-int tsdbPrepareCommit(STsdb *pTsdb) {
+// int tsdbPrepareCommit(STsdb *pTsdb) {
-  if (pTsdb->mem == NULL) return 0;
+//   if (pTsdb->mem == NULL) return 0;
 
-  ASSERT(pTsdb->imem == NULL);
+//   ASSERT(pTsdb->imem == NULL);
 
-  pTsdb->imem = pTsdb->mem;
+//   pTsdb->imem = pTsdb->mem;
-  pTsdb->mem = NULL;
+//   pTsdb->mem = NULL;
-  return 0;
+//   return 0;
-}
+// }
-
-int tsdbCommit(STsdb *pRepo) {
-  SCommitH   commith = {0};
-  SDFileSet *pSet = NULL;
-  int        fid;
-
-  // if (pRepo->imem == NULL) return 0;
-  pRepo->imem = pRepo->mem;
-  pRepo->mem = NULL;
-
-  tsdbStartCommit(pRepo);
-  // Resource initialization
-  if (tsdbInitCommitH(&commith, pRepo) < 0) {
-    return -1;
-  }
-
-  // Skip expired memory data and expired FSET
-  tsdbSeekCommitIter(&commith, commith.rtn.minKey);
-  while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) {
-    if (pSet->fid < commith.rtn.minFid) {
-      tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
-               TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
-    } else {
-      break;
-    }
-  }
-
-  // Loop to commit to each file
-  fid = tsdbNextCommitFid(&(commith));
-  while (true) {
-    // Loop over both on disk and memory
-    if (pSet == NULL && fid == TSDB_IVLD_FID) break;
-
-    if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) {
-      // Only has existing FSET but no memory data to commit in this
-      // existing FSET, only check if file in correct retention
-      if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) {
-        tsdbDestroyCommitH(&commith);
-        return -1;
-      }
-
-      pSet = tsdbFSIterNext(&(commith.fsIter));
-    } else {
-      // Has memory data to commit
-      SDFileSet *pCSet;
-      int        cfid;
-
-      if (pSet == NULL || pSet->fid > fid) {
-        // Commit to a new FSET with fid: fid
-        pCSet = NULL;
-        cfid = fid;
-      } else {
-        // Commit to an existing FSET
-        pCSet = pSet;
-        cfid = pSet->fid;
-        pSet = tsdbFSIterNext(&(commith.fsIter));
-      }
-
-      if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) {
-        tsdbDestroyCommitH(&commith);
-        return -1;
-      }
-
-      fid = tsdbNextCommitFid(&commith);
-    }
-  }
-
-  tsdbDestroyCommitH(&commith);
-  tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS);
-
-  return 0;
-}
 
 void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
   STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
@@ -543,8 +546,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
       return -1;
     }
 
-    tsdbDebug("vgId:%d, FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet),
+    tsdbDebug("vgId:%d, FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo),
-              TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
+              TSDB_FSET_FID(pSet), TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
   } else {
     pCommith->isRFileSet = false;
   }
@@ -716,8 +719,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
 
 // extern int32_t tsTsdbMetaCompactRatio;
 
-int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf,
+static int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf,
-                           SBlockIdx *pIdx) {
+                                  SBlockIdx *pIdx) {
   size_t      nSupBlocks;
   size_t      nSubBlocks;
   uint32_t    tlen;
@@ -769,7 +772,7 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray
 
   pIdx->uid = TABLE_UID(pTable);
   pIdx->hasLast = pBlock->last ? 1 : 0;
-  pIdx->maxKey = pBlock->keyLast;
+  pIdx->maxKey = pBlock->maxKey;
   pIdx->numOfBlocks = (uint32_t)nSupBlocks;
   pIdx->len = tlen;
   pIdx->offset = (uint32_t)offset;
@@ -777,7 +780,7 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray
   return 0;
 }
 
-int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
+static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
   SBlockIdx *pBlkIdx;
   size_t     nidx = taosArrayGetSize(pIdxA);
   int        tlen = 0, size;
@@ -890,7 +893,7 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
           return -1;
         }
       } else {
-        if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) {
+        if (tsdbCommitMemData(pCommith, pIter, pBlock->minKey.ts - 1, true) < 0) {
           return -1;
         }
       }
@@ -985,9 +988,9 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
   TSKEY   key = *(TSKEY *)arg1;
   SBlock *pBlock = (SBlock *)arg2;
 
-  if (key < pBlock->keyFirst) {
+  if (key < pBlock->minKey.ts) {
     return -1;
-  } else if (key > pBlock->keyLast) {
+  } else if (key > pBlock->maxKey.ts) {
     return 1;
   } else {
     return 0;
@@ -1011,8 +1014,8 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
  * @param ppExBuf
  * @return int
  */
-int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
+static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
-                       SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) {
+                              SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) {
   STsdbCfg     *pCfg = REPO_CFG(pRepo);
   SBlockData   *pBlockData = NULL;
   SAggrBlkData *pAggrBlkData = NULL;
@@ -1170,7 +1173,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
     tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)));
 
     if (ncol != 0) {
-      tsdbSetBlockColOffset(pBlockCol, toffset);
+      pBlockCol->offset = toffset;
       pBlockCol->len = flen;  // data + bitmaps
       pBlockCol->blen = tBitmapsLen;
       ++tcol;
@@ -1215,8 +1218,8 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   pBlock->numOfSubBlocks = isSuper ? 1 : 0;
   pBlock->numOfCols = nColsNotAllNull;
   pBlock->numOfBSma = nColsOfBlockSma;
-  pBlock->keyFirst = dataColsKeyFirst(pDataCols);
+  pBlock->minKey.ts = dataColsKeyFirst(pDataCols);
-  pBlock->keyLast = dataColsKeyLast(pDataCols);
+  pBlock->maxKey.ts = dataColsKeyLast(pDataCols);
   pBlock->aggrStat = aggrStatus;
   pBlock->blkVer = SBlockVerLatest;
   pBlock->aggrOffset = (uint64_t)offsetAggr;
@@ -1224,7 +1227,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   tsdbDebug("vgId:%d, uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64
             " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
             REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
-            pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast);
+            pBlock->numOfCols, pBlock->minKey.ts, pBlock->maxKey.ts);
 
   return 0;
 }
@@ -1307,7 +1310,7 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) {
   if (bidx == nBlocks - 1) {
     keyLimit = pCommith->maxKey;
   } else {
-    keyLimit = pBlock[1].keyFirst - 1;
+    keyLimit = pBlock[1].minKey.ts - 1;
   }
 
   SSkipListIterator titer = *(pIter->pIter);
@@ -1349,8 +1352,8 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) {
     }
     subBlocks[pBlock->numOfSubBlocks] = block;
     supBlock = *pBlock;
-    supBlock.keyFirst = mInfo.keyFirst;
+    supBlock.minKey.ts = mInfo.keyFirst;
-    supBlock.keyLast = mInfo.keyLast;
+    supBlock.maxKey.ts = mInfo.keyLast;
     supBlock.numOfSubBlocks++;
     supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed;
     supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock);

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -0,0 +1,436 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+
+typedef struct {
+  SMemTable *pMemTable;
+  int32_t    minutes;
+  int8_t     precision;
+  TSKEY      nCommitKey;
+  int32_t    fid;
+  TSKEY      minKey;
+  TSKEY      maxKey;
+  SReadH     readh;
+  SDFileSet  wSet;
+  SArray    *aBlkIdx;
+  SArray    *aSupBlk;
+  SArray    *aSubBlk;
+  SArray    *aDelInfo;
+} SCommitH;
+
+static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb);
+static int32_t tsdbCommitEnd(SCommitH *pCHandle);
+static int32_t tsdbCommitImpl(SCommitH *pCHandle);
+
+int32_t tsdbBegin2(STsdb *pTsdb) {
+  int32_t code = 0;
+
+  ASSERT(pTsdb->mem == NULL);
+  code = tsdbMemTableCreate2(pTsdb, (SMemTable **)&pTsdb->mem);
+  if (code) {
+    tsdbError("vgId:%d failed to begin TSDB since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+    goto _exit;
+  }
+
+_exit:
+  return code;
+}
+
+int32_t tsdbCommit2(STsdb *pTsdb) {
+  int32_t  code = 0;
+  SCommitH ch = {0};
+
+  // start to commit
+  code = tsdbCommitStart(&ch, pTsdb);
+  if (code) {
+    goto _exit;
+  }
+
+  // commit
+  code = tsdbCommitImpl(&ch);
+  if (code) {
+    goto _err;
+  }
+
+  // end commit
+  code = tsdbCommitEnd(&ch);
+  if (code) {
+    goto _exit;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb) {
+  int32_t    code = 0;
+  SMemTable *pMemTable = (SMemTable *)pTsdb->mem;
+
+  tsdbInfo("vgId:%d start to commit", TD_VID(pTsdb->pVnode));
+
+  // switch to commit
+  ASSERT(pTsdb->imem == NULL && pTsdb->mem);
+  pTsdb->imem = pTsdb->mem;
+  pTsdb->mem = NULL;
+
+  // open handle
+  pCHandle->pMemTable = pMemTable;
+  pCHandle->minutes = pTsdb->keepCfg.days;
+  pCHandle->precision = pTsdb->keepCfg.precision;
+  pCHandle->nCommitKey = pMemTable->minKey.ts;
+
+  code = tsdbInitReadH(&pCHandle->readh, pTsdb);
+  if (code) {
+    goto _err;
+  }
+  pCHandle->aBlkIdx = taosArrayInit(0, sizeof(SBlockIdx));
+  if (pCHandle->aBlkIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pCHandle->aSupBlk = taosArrayInit(0, sizeof(SBlock));
+  if (pCHandle->aSupBlk == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pCHandle->aSubBlk = taosArrayInit(0, sizeof(SBlock));
+  if (pCHandle->aSubBlk == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pCHandle->aDelInfo = taosArrayInit(0, sizeof(SDelInfo));
+  if (pCHandle->aDelInfo == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  // start FS transaction
+  tsdbStartFSTxn(pTsdb, 0, 0);
+
+  return code;
+
+_err:
+  return code;
+}
+
+static int32_t tsdbCommitEnd(SCommitH *pCHandle) {
+  int32_t    code = 0;
+  STsdb     *pTsdb = pCHandle->pMemTable->pTsdb;
+  SMemTable *pMemTable = (SMemTable *)pTsdb->imem;
+
+  // end transaction
+  code = tsdbEndFSTxn(pTsdb);
+  if (code) {
+    goto _err;
+  }
+
+  // close handle
+  taosArrayClear(pCHandle->aDelInfo);
+  taosArrayClear(pCHandle->aSubBlk);
+  taosArrayClear(pCHandle->aSupBlk);
+  taosArrayClear(pCHandle->aBlkIdx);
+  tsdbDestroyReadH(&pCHandle->readh);
+
+  // destroy memtable (todo: unref it)
+  pTsdb->imem = NULL;
+  tsdbMemTableDestroy2(pMemTable);
+
+  tsdbInfo("vgId:%d commit over", TD_VID(pTsdb->pVnode));
+  return code;
+
+_err:
+  return code;
+}
+
+static int32_t tsdbCommitTableStart(SCommitH *pCHandle) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbCommitTableEnd(SCommitH *pCHandle) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbCommitTable(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) {
+  int32_t      code = 0;
+  SMemDataIter iter = {0};
+
+  // commit table start
+  code = tsdbCommitTableStart(pCHandle);
+  if (code) {
+    goto _err;
+  }
+
+  // commit table impl
+  if (pMemData && pBlockIdx) {
+    // TODO
+  } else if (pMemData) {
+    // TODO
+  } else {
+    // TODO
+  }
+
+  // commit table end
+  code = tsdbCommitTableEnd(pCHandle);
+  if (code) {
+    goto _err;
+  }
+
+  return code;
+
+_err:
+  return code;
+}
+
+static int32_t tsdbTableIdCmprFn(const void *p1, const void *p2) {
+  TABLEID *pId1 = (TABLEID *)p1;
+  TABLEID *pId2 = (TABLEID *)p2;
+
+  if (pId1->suid < pId2->suid) {
+    return -1;
+  } else if (pId1->suid > pId2->suid) {
+    return 1;
+  }
+
+  if (pId1->uid < pId2->uid) {
+    return -1;
+  } else if (pId1->uid > pId2->uid) {
+    return 1;
+  }
+
+  return 0;
+}
+
+static int32_t tsdbWriteBlockIdx(SDFile *pFile, SArray *pArray, uint8_t **ppBuf) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbCommitFileStart(SCommitH *pCHandle) {
+  int32_t    code = 0;
+  STsdb     *pTsdb = pCHandle->pMemTable->pTsdb;
+  SDFileSet *pSet = NULL;
+
+  taosArrayClear(pCHandle->aBlkIdx);
+
+  return code;
+}
+
+static int32_t tsdbCommitFileEnd(SCommitH *pCHandle) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbCommitFile(SCommitH *pCHandle) {
+  int32_t    code = 0;
+  SMemData  *pMemData;
+  SBlockIdx *pBlockIdx;
+  int32_t    iMemData;
+  int32_t    nMemData;
+  int32_t    iBlockIdx;
+  int32_t    nBlockIdx;
+
+  // commit file start
+  code = tsdbCommitFileStart(pCHandle);
+  if (code) {
+    goto _err;
+  }
+
+  // commit file impl
+  iMemData = 0;
+  nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData);
+  iBlockIdx = 0;
+  nBlockIdx = 0;  // todo
+
+  for (;;) {
+    if (iMemData >= nMemData && iBlockIdx >= nBlockIdx) break;
+
+    pMemData = NULL;
+    pBlockIdx = NULL;
+    if (iMemData < nMemData) {
+      pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
+    }
+    if (iBlockIdx < nBlockIdx) {
+      // pBlockIdx = ;
+    }
+
+    if (pMemData && pBlockIdx) {
+      int32_t c = tsdbTableIdCmprFn(pMemData, pBlockIdx);
+      if (c < 0) {
+        iMemData++;
+        pBlockIdx = NULL;
+      } else if (c == 0) {
+        iMemData++;
+        iBlockIdx++;
+      } else {
+        iBlockIdx++;
+        pMemData = NULL;
+      }
+    } else {
+      if (pMemData) {
+        iMemData++;
+      } else {
+        iBlockIdx++;
+      }
+    }
+
+    code = tsdbCommitTable(pCHandle, pMemData, pBlockIdx);
+    if (code) {
+      goto _err;
+    }
+  }
+
+  // commit file end
+  code = tsdbCommitFileEnd(pCHandle);
+  if (code) {
+    goto _err;
+  }
+
+  return code;
+
+_err:
+  return code;
+}
+
+static int32_t tsdbCommitData(SCommitH *pCHandle) {
+  int32_t code = 0;
+  int32_t fid;
+
+  if (pCHandle->pMemTable->nRows == 0) goto _exit;
+
+  // loop to commit to each file
+  for (;;) {
+    if (pCHandle->nCommitKey == TSKEY_MAX) break;
+
+    pCHandle->fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision);
+    tsdbGetFidKeyRange(pCHandle->minutes, pCHandle->precision, pCHandle->fid, &pCHandle->minKey, &pCHandle->maxKey);
+    code = tsdbCommitFile(pCHandle);
+    if (code) {
+      goto _err;
+    }
+  }
+
+_exit:
+  return code;
+
+_err:
+  return code;
+}
+
+static int32_t delInfoCmprFn(const void *p1, const void *p2) {
+  SDelInfo *pDelInfo1 = (SDelInfo *)p1;
+  SDelInfo *pDelInfo2 = (SDelInfo *)p2;
+
+  if (pDelInfo1->suid < pDelInfo2->suid) {
+    return -1;
+  } else if (pDelInfo1->suid > pDelInfo2->suid) {
+    return 1;
+  }
+
+  if (pDelInfo1->uid < pDelInfo2->uid) {
+    return -1;
+  } else if (pDelInfo1->uid > pDelInfo2->uid) {
+    return 1;
+  }
+
+  if (pDelInfo1->version < pDelInfo2->version) {
+    return -1;
+  } else if (pDelInfo1->version > pDelInfo2->version) {
+    return 1;
+  }
+
+  return 0;
+}
+static int32_t tsdbCommitDelete(SCommitH *pCHandle) {
+  int32_t   code = 0;
+  SDelInfo  delInfo;
+  SMemData *pMemData;
+
+  if (pCHandle->pMemTable->nDelOp == 0) goto _exit;
+
+  // load del array (todo)
+
+  // loop to append SDelInfo
+  for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) {
+    pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
+
+    for (SDelOp *pDelOp = pMemData->delOpHead; pDelOp; pDelOp = pDelOp->pNext) {
+      delInfo = (SDelInfo){.suid = pMemData->suid,
+                           .uid = pMemData->uid,
+                           .version = pDelOp->version,
+                           .sKey = pDelOp->sKey,
+                           .eKey = pDelOp->eKey};
+      if (taosArrayPush(pCHandle->aDelInfo, &delInfo) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+    }
+  }
+
+  taosArraySort(pCHandle->aDelInfo, delInfoCmprFn);
+
+  // write to new file
+
+_exit:
+  return code;
+
+_err:
+  return code;
+}
+
+static int32_t tsdbCommitCache(SCommitH *pCHandle) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbCommitImpl(SCommitH *pCHandle) {
+  int32_t code = 0;
+
+  // commit data
+  code = tsdbCommitData(pCHandle);
+  if (code) {
+    goto _err;
+  }
+
+  // commit delete
+  code = tsdbCommitDelete(pCHandle);
+  if (code) {
+    goto _err;
+  }
+
+  // commit cache if need (todo)
+  if (0) {
+    code = tsdbCommitCache(pCHandle);
+    if (code) {
+      goto _err;
+    }
+  }
+
+  return code;
+
+_err:
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -37,11 +37,11 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
 // static int  tsdbProcessExpiredFS(STsdb *pRepo);
 // static int  tsdbCreateMeta(STsdb *pRepo);
 
-static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) {
+static void tsdbGetRootDir(int repoid, const char *dir, char dirName[]) {
   snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir);
 }
 
-static void tsdbGetDataDir(int repoid,  const char* dir, char dirName[]) {
+static void tsdbGetDataDir(int repoid, const char *dir, char dirName[]) {
   snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir);
 }
 
@@ -216,16 +216,7 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) {
     return NULL;
   }
 
-  pfs->metaCache = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-  if (pfs->metaCache == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    tsdbFreeFS(pfs);
-    return NULL;
-  }
-
   pfs->intxn = false;
-  pfs->metaCacheComp = NULL;
-
   pfs->nstatus = tsdbNewFSStatus(maxFSet);
   if (pfs->nstatus == NULL) {
     tsdbFreeFS(pfs);
@@ -238,8 +229,6 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) {
 void *tsdbFreeFS(STsdbFS *pfs) {
   if (pfs) {
     pfs->nstatus = tsdbFreeFSStatus(pfs->nstatus);
-    taosHashCleanup(pfs->metaCache);
-    pfs->metaCache = NULL;
     pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus);
     taosThreadRwlockDestroy(&(pfs->lock));
     taosMemoryFree(pfs);
@@ -963,13 +952,6 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
 }
 
 static int tsdbRestoreCurrent(STsdb *pRepo) {
-  // // Loop to recover mfile
-  // if (tsdbRestoreMeta(pRepo) < 0) {
-  //   tsdbError("vgId:%d, failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno));
-  //   return -1;
-  // }
-
-  // Loop to recover dfile set
   if (tsdbRestoreDFileSet(pRepo) < 0) {
     tsdbError("vgId:%d, failed to restore DFileSet since %s", REPO_ID(pRepo), tstrerror(terrno));
     return -1;
@@ -1052,3 +1034,30 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) {
     tsdbCloseDFileSet(&fset);
   }
 }
+
+int tsdbRLockFS(STsdbFS *pFs) {
+  int code = taosThreadRwlockRdlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+int tsdbWLockFS(STsdbFS *pFs) {
+  int code = taosThreadRwlockWrlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+int tsdbUnLockFS(STsdbFS *pFs) {
+  int code = taosThreadRwlockUnlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -25,7 +25,7 @@ static const char *TSDB_FNAME_SUFFIX[] = {
     "meta",  // TSDB_FILE_META
 };
 
-static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname);
+static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname);
 // static int   tsdbRollBackMFile(SMFile *pMFile);
 static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
 static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
@@ -447,4 +447,137 @@ static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, c
       snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver);
     }
   }
+}
+
+int tsdbOpenDFile(SDFile *pDFile, int flags) {
+  ASSERT(!TSDB_FILE_OPENED(pDFile));
+
+  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
+  if (pDFile->pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbCloseDFile(SDFile *pDFile) {
+  if (TSDB_FILE_OPENED(pDFile)) {
+    taosCloseFile(&pDFile->pFile);
+    TSDB_FILE_SET_CLOSED(pDFile);
+  }
+}
+
+int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence) {
+  // ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm) {
+  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM));
+}
+
+int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pDFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pDFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+int tsdbRemoveDFile(SDFile *pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
+
+int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) {
+  if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  pDest->info = pSrc->info;
+  return 0;
+}
+
+void tsdbCloseDFileSet(SDFileSet *pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+int tsdbOpenDFileSet(SDFileSet *pSet, int flags) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
+      tsdbCloseDFileSet(pSet);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+void tsdbRemoveDFileSet(SDFileSet *pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
+      tsdbRemoveDFileSet(pDest);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) {
+  *minKey = fid * days * tsTickPerMin[precision];
+  *maxKey = *minKey + days * tsTickPerMin[precision] - 1;
 }

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -297,8 +297,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo
   tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext);
 
 #ifdef TD_DEBUG_PRINT_ROW
-  printf("!!! %s:%d table %" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, pTbData->uid,
+  printf("!!! %s:%d vgId:%d dir:%s table:%" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__,
-         SL_SIZE(pTbData->pData));
+         TD_VID(pTsdb->pVnode), pTsdb->dir, pTbData->uid, SL_SIZE(pTbData->pData));
 #endif
 
   // Set statistics

source/dnode/vnode/src/tsdb/tsdbMemTable2.c

@@ -15,42 +15,15 @@
 
 #include "tsdb.h"
 
-typedef struct SMemData         SMemData;
-typedef struct SMemSkipList     SMemSkipList;
-typedef struct SMemSkipListNode SMemSkipListNode;
-
 struct SMemSkipListNode {
   int8_t            level;
   SMemSkipListNode *forwards[0];
 };
 
-struct SMemSkipList {
+typedef struct {
-  uint32_t          seed;
+  tb_uid_t  uid;
-  int32_t           size;
+  STSchema *pTSchema;
-  int8_t            maxLevel;
+} SSkmInfo;
-  int8_t            level;
-  SMemSkipListNode *pHead;
-  SMemSkipListNode *pTail;
-};
-
-struct SMemData {
-  tb_uid_t     suid;
-  tb_uid_t     uid;
-  TSDBKEY      minKey;
-  TSDBKEY      maxKey;
-  SDelOp      *delOpHead;
-  SDelOp      *delOpTail;
-  SMemSkipList sl;
-};
-
-struct SMemTable {
-  STsdb  *pTsdb;
-  int32_t nRef;
-  TSDBKEY minKey;
-  TSDBKEY maxKey;
-  int64_t nRows;
-  SArray *pArray;  // SArray<SMemData>
-};
 
 #define SL_MAX_LEVEL 5
 
@@ -59,14 +32,17 @@ struct SMemTable {
 #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
 #define SL_NODE_DATA(n)        (&SL_NODE_BACKWARD(n, (n)->level))
 
+#define SL_MOVE_BACKWARD 0x1
+#define SL_MOVE_FROM_POS 0x2
+
 static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData);
 static int     memDataPCmprFn(const void *p1, const void *p2);
 static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
 static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
 static int8_t  tsdbMemSkipListRandLevel(SMemSkipList *pSl);
-static void    memDataMovePos(SMemData *pMemData, TSDBROW *pRow, int8_t isForward, SMemSkipListNode **pos);
+static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version,
-static int32_t memDataPutRow(SVBufPool *pPool, SMemData *pMemData, TSDBROW *pRow, int8_t isForward,
+                                       SVSubmitBlk *pSubmitBlk);
-                             SMemSkipListNode **pos);
+static void    memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
 
 // SMemTable ==============================================
 int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) {
@@ -83,8 +59,9 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) {
   pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
   pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
   pMemTable->nRows = 0;
-  pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *));
+  pMemTable->nDelOp = 0;
-  if (pMemTable->pArray == NULL) {
+  pMemTable->aMemData = taosArrayInit(512, sizeof(SMemData *));
+  if (pMemTable->aMemData == NULL) {
     taosMemoryFree(pMemTable);
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -99,7 +76,7 @@ _err:
 }
 
 void tsdbMemTableDestroy2(SMemTable *pMemTable) {
-  taosArrayDestroyEx(pMemTable->pArray, NULL /*TODO*/);
+  taosArrayDestroyEx(pMemTable->aMemData, NULL /*TODO*/);
   taosMemoryFree(pMemTable);
 }
 
@@ -123,28 +100,9 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit
   }
 
   // do insert
-  int32_t           nt;
+  code = tsdbInsertTableDataImpl(pMemTable, pMemData, version, pSubmitBlk);
-  int32_t           n = 0;
+  if (code) {
-  uint8_t          *p = pSubmitBlk->pData;
+    goto _err;
-  int32_t           nRow = 0;
-  SMemSkipListNode *pos[SL_MAX_LEVEL] = {0};
-
-  for (int8_t iLevel = 0; iLevel < SL_MAX_LEVEL; iLevel++) {
-    pos[iLevel] = pMemData->sl.pTail;
-  }
-  while (n < pSubmitBlk->nData) {
-    nt = tGetTSRow(p + n, &row.tsRow);
-    n += nt;
-
-    ASSERT(n <= pSubmitBlk->nData);
-
-    memDataMovePos(pMemData, &row, nRow ? 1 : 0, pos);
-    code = memDataPutRow(pTsdb->pVnode->inUse, pMemData, &row, nRow ? 1 : 0, pos);
-    if (code) {
-      goto _err;
-    }
-
-    nRow++;
   }
 
   return code;
@@ -192,6 +150,8 @@ int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_ui
     // update the state of pMemTable, pMemData, last and lastrow (todo)
   }
 
+  pMemTable->nDelOp++;
+
   tsdbDebug("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
             " since %s",
             TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
@@ -204,6 +164,92 @@ _err:
   return code;
 }
 
+void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter) {
+  SMemSkipListNode *pos[SL_MAX_LEVEL];
+
+  pIter->pMemData = pMemData;
+  pIter->backward = backward;
+  pIter->pRow = NULL;
+  if (pKey == NULL) {
+    // create from head or tail
+    if (backward) {
+      pIter->pNode = SL_NODE_BACKWARD(pMemData->sl.pTail, 0);
+    } else {
+      pIter->pNode = SL_NODE_FORWARD(pMemData->sl.pHead, 0);
+    }
+  } else {
+    // create from a key
+    if (backward) {
+      memDataMovePosTo(pMemData, pos, pKey, SL_MOVE_BACKWARD);
+      pIter->pNode = SL_NODE_BACKWARD(pos[0], 0);
+    } else {
+      memDataMovePosTo(pMemData, pos, pKey, 0);
+      pIter->pNode = SL_NODE_FORWARD(pos[0], 0);
+    }
+  }
+}
+
+bool tsdbMemDataIterNext(SMemDataIter *pIter) {
+  SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
+  SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
+
+  pIter->pRow = NULL;
+  if (pIter->backward) {
+    ASSERT(pIter->pNode != pTail);
+
+    if (pIter->pNode == pHead) {
+      return false;
+    }
+
+    pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0);
+    if (pIter->pNode == pHead) {
+      return false;
+    }
+  } else {
+    ASSERT(pIter->pNode != pHead);
+
+    if (pIter->pNode == pTail) {
+      return false;
+    }
+
+    pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0);
+    if (pIter->pNode == pTail) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow) {
+  if (pIter->pRow) {
+    *ppRow = pIter->pRow;
+  } else {
+    SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
+    SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
+
+    if (pIter->backward) {
+      ASSERT(pIter->pNode != pTail);
+
+      if (pIter->pNode == pHead) {
+        *ppRow = NULL;
+      } else {
+        tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
+        *ppRow = &pIter->row;
+      }
+    } else {
+      ASSERT(pIter->pNode != pHead);
+
+      if (pIter->pNode == pTail) {
+        *ppRow = NULL;
+      } else {
+        tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
+        *ppRow = &pIter->row;
+      }
+    }
+  }
+}
+
 static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) {
   int32_t    code = 0;
   int32_t    idx = 0;
@@ -213,9 +259,9 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui
   int8_t     maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
 
   // get
-  idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE);
+  idx = taosArraySearchIdx(pMemTable->aMemData, &pMemDataT, memDataPCmprFn, TD_GE);
   if (idx >= 0) {
-    pMemData = (SMemData *)taosArrayGet(pMemTable->pArray, idx);
+    pMemData = (SMemData *)taosArrayGet(pMemTable->aMemData, idx);
     if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit;
   }
 
@@ -247,7 +293,7 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui
   }
 
   if (idx < 0) idx = 0;
-  if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) {
+  if (taosArrayInsert(pMemTable->aMemData, idx, &pMemData) == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
@@ -310,28 +356,62 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
   return level;
 }
 
-static void memDataMovePos(SMemData *pMemData, TSDBROW *pRow, int8_t isForward, SMemSkipListNode **pos) {
+static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) {
-  TSDBKEY *pKey;
+  SMemSkipListNode *px;
-  int      c;
+  SMemSkipListNode *pn;
+  TSDBKEY          *pTKey;
+  int               c;
+  int               backward = flags & SL_MOVE_BACKWARD;
+  int               fromPos = flags & SL_MOVE_FROM_POS;
 
-  if (isForward) {
+  if (backward) {
-    // TODO
+    px = pMemData->sl.pTail;
-  } else {
-    SMemSkipListNode *px = pMemData->sl.pTail;
 
-    for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) {
+    for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
-      if (iLevel < pMemData->sl.level) {
+      pos[iLevel] = px;
-        SMemSkipListNode *p = SL_NODE_BACKWARD(px, iLevel);
+    }
 
-        while (p != pMemData->sl.pHead) {
+    if (pMemData->sl.level) {
-          pKey = (TSDBKEY *)SL_NODE_DATA(p);
+      if (fromPos) px = pos[pMemData->sl.level - 1];
 
-          c = tsdbKeyCmprFn(pKey, pRow);
+      for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
+        pn = SL_NODE_BACKWARD(px, iLevel);
+        while (pn != pMemData->sl.pHead) {
+          pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
+
+          c = tsdbKeyCmprFn(pTKey, pKey);
           if (c <= 0) {
             break;
           } else {
-            px = p;
+            px = pn;
-            p = SL_NODE_BACKWARD(px, iLevel);
+            pn = SL_NODE_BACKWARD(px, iLevel);
+          }
+        }
+
+        pos[iLevel] = px;
+      }
+    }
+  } else {
+    px = pMemData->sl.pHead;
+
+    for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
+      pos[iLevel] = px;
+    }
+
+    if (pMemData->sl.level) {
+      if (fromPos) px = pos[pMemData->sl.level - 1];
+
+      for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
+        pn = SL_NODE_FORWARD(px, iLevel);
+        while (pn != pMemData->sl.pHead) {
+          pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
+
+          c = tsdbKeyCmprFn(pTKey, pKey);
+          if (c >= 0) {
+            break;
+          } else {
+            px = pn;
+            pn = SL_NODE_FORWARD(px, iLevel);
           }
         }
 
@@ -341,56 +421,115 @@ static void memDataMovePos(SMemData *pMemData, TSDBROW *pRow, int8_t isForward,
   }
 }
 
-static void memMovePosFrom(SMemData *pMemData, SMemSkipListNode *pNode, TSDBROW *pRow, int8_t isForward,
+static int32_t memDataDoPut(SMemTable *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow,
-                           SMemSkipListNode **pos) {
+                            int8_t forward) {
-  SMemSkipListNode *px = pNode;
-  TSDBKEY          *pKey;
-  SMemSkipListNode *p;
-  int               c;
-
-  if (isForward) {
-  } else {
-    ASSERT(pNode != pMemData->sl.pHead);
-
-    for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) {
-      p = SL_NODE_BACKWARD(px, iLevel);
-      while (p != pMemData->sl.pHead) {
-        pKey = (TSDBKEY *)SL_NODE_DATA(p);
-
-        c = tsdbKeyCmprFn(pKey, pRow);
-        if (c <= 0) {
-          break;
-        } else {
-          px = p;
-          p = SL_NODE_BACKWARD(px, iLevel);
-        }
-      }
-
-      pos[iLevel] = px;
-    }
-  }
-}
-
-static int32_t memDataPutRow(SVBufPool *pPool, SMemData *pMemData, TSDBROW *pRow, int8_t isForward,
-                             SMemSkipListNode **pos) {
   int32_t           code = 0;
   int8_t            level;
   SMemSkipListNode *pNode;
+  SVBufPool        *pPool = pMemTable->pTsdb->pVnode->inUse;
 
+  // node
   level = tsdbMemSkipListRandLevel(&pMemData->sl);
   pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
   if (pNode == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
+  pNode->level = level;
+  for (int8_t iLevel = 0; iLevel < level; iLevel++) {
+    SL_NODE_FORWARD(pNode, iLevel) = NULL;
+    SL_NODE_BACKWARD(pNode, iLevel) = NULL;
+  }
 
-  // do the read put
+  tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow);
-  if (isForward) {
+
-    // TODO
+  // put
-  } else {
+  for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) {
-    // TODO
+    SMemSkipListNode *px = pos[iLevel];
+
+    if (forward) {
+      SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel);
+
+      SL_NODE_FORWARD(pNode, iLevel) = pNext;
+      SL_NODE_BACKWARD(pNode, iLevel) = px;
+
+      SL_NODE_BACKWARD(pNext, iLevel) = pNode;
+      SL_NODE_FORWARD(px, iLevel) = pNode;
+    } else {
+      SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel);
+
+      SL_NODE_FORWARD(pNode, iLevel) = px;
+      SL_NODE_BACKWARD(pNode, iLevel) = pPrev;
+
+      SL_NODE_FORWARD(pPrev, iLevel) = pNode;
+      SL_NODE_BACKWARD(px, iLevel) = pNode;
+    }
+  }
+
+  pMemData->sl.size++;
+  if (pMemData->sl.level < pNode->level) {
+    pMemData->sl.level = pNode->level;
   }
 
 _exit:
   return code;
-}
+}
+
+static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version,
+                                       SVSubmitBlk *pSubmitBlk) {
+  int32_t  code = 0;
+  int32_t  n = 0;
+  uint8_t *p = pSubmitBlk->pData;
+  int32_t  nRow = 0;
+  TSDBROW  row = {.version = version};
+
+  SMemSkipListNode *pos[SL_MAX_LEVEL];
+
+  ASSERT(pSubmitBlk->nData);
+
+  // backward put first data
+  n += tGetTSRow(p + n, &row.tsRow);
+  ASSERT(n <= pSubmitBlk->nData);
+
+  memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD);
+  code = memDataDoPut(pMemTable, pMemData, pos, &row, 0);
+  if (code) {
+    goto _exit;
+  }
+  nRow++;
+
+  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->minKey) < 0) {
+    pMemData->minKey = *(TSDBKEY *)&row;
+  }
+  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->minKey) < 0) {
+    pMemTable->minKey = *(TSDBKEY *)&row;
+  }
+
+  // forward put rest
+  for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
+    pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel);
+  }
+  while (n < pSubmitBlk->nData) {
+    n += tGetTSRow(p + n, &row.tsRow);
+    ASSERT(n <= pSubmitBlk->nData);
+
+    memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS);
+    code = memDataDoPut(pMemTable, pMemData, pos, &row, 1);
+    if (code) {
+      goto _exit;
+    }
+
+    nRow++;
+  }
+
+  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->maxKey) > 0) {
+    pMemData->maxKey = *(TSDBKEY *)&row;
+  }
+  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->maxKey) > 0) {
+    pMemTable->maxKey = *(TSDBKEY *)&row;
+  }
+  pMemTable->nRows += nRow;
+
+_exit:
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -20,10 +20,10 @@
 #define ASCENDING_TRAVERSE(o)      (o == TSDB_ORDER_ASC)
 #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
 
-#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block)                                   \
+#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block)                                      \
-  ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \
+  ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \
-                    .numOfCols = (_block)->numOfCols,                                  \
+                    .numOfCols = (_block)->numOfCols,                                     \
-                    .rows = (_block)->numOfRows,                                       \
+                    .rows = (_block)->numOfRows,                                          \
                     .uid = (_checkInfo)->tableId})
 
 enum {
@@ -1105,12 +1105,12 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s
 
     if (numOfBlocks == 1) break;
 
-    if (skey > pBlock[midSlot].keyLast) {
+    if (skey > pBlock[midSlot].maxKey.ts) {
       if (numOfBlocks == 2) break;
-      if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break;
+      if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].minKey.ts)) break;
       firstSlot = midSlot + 1;
-    } else if (skey < pBlock[midSlot].keyFirst) {
+    } else if (skey < pBlock[midSlot].minKey.ts) {
-      if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break;
+      if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].maxKey.ts)) break;
       lastSlot = midSlot - 1;
     } else {
       break;  // got the slot
@@ -1177,12 +1177,12 @@ static int32_t loadBlockInfo(STsdbReadHandle* pTsdbReadHandle, int32_t index, in
   int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC);
   int32_t end = start;
 
-  if (s > pCompInfo->blocks[start].keyLast) {
+  if (s > pCompInfo->blocks[start].maxKey.ts) {
     return 0;
   }
 
   // todo speedup the procedure of located end block
-  while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) {
+  while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].minKey.ts <= e)) {
     end += 1;
   }
 
@@ -1275,7 +1275,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl
   pBlock->numOfRows = pCols->numOfRows;
 
   // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z
-  if (pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  if (pBlock->minKey.ts < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) {
     int64_t* src = pCols->cols[0].pData;
     for (int32_t i = 0; i < pBlock->numOfRows; ++i) {
       src[i] = tdGetKey(src[i]);
@@ -1287,7 +1287,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl
 
   tsdbDebug("%p load file block into buffer, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%" PRId64
             " us, %s",
-            pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime,
+            pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows, elapsedTime,
             pTsdbReadHandle->idStr);
   return TSDB_CODE_SUCCESS;
 
@@ -1295,7 +1295,8 @@ _error:
   pBlock->numOfRows = 0;
 
   tsdbError("%p error occurs in loading file block, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, %s",
-            pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->idStr);
+            pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows,
+            pTsdbReadHandle->idStr);
   return terrno;
 }
 
@@ -1423,7 +1424,7 @@ static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBloc
 
   if (asc) {
     // query ended in/started from current block
-    if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) {
+    if (pTsdbReadHandle->window.ekey < pBlock->maxKey.ts || pCheckInfo->lastKey > pBlock->minKey.ts) {
       if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
         *exists = false;
         return code;
@@ -1432,35 +1433,35 @@ static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBloc
       SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0];
       assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
 
-      if (pCheckInfo->lastKey > pBlock->keyFirst) {
+      if (pCheckInfo->lastKey > pBlock->minKey.ts) {
         cur->pos =
             binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order);
       } else {
         cur->pos = 0;
       }
 
-      assert(pCheckInfo->lastKey <= pBlock->keyLast);
+      assert(pCheckInfo->lastKey <= pBlock->maxKey.ts);
       doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock);
     } else {  // the whole block is loaded in to buffer
       cur->pos = asc ? 0 : (pBlock->numOfRows - 1);
       code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo);
     }
   } else {  // desc order, query ended in current block
-    if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) {
+    if (pTsdbReadHandle->window.ekey > pBlock->minKey.ts || pCheckInfo->lastKey < pBlock->maxKey.ts) {
       if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
         *exists = false;
         return code;
       }
 
       SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0];
-      if (pCheckInfo->lastKey < pBlock->keyLast) {
+      if (pCheckInfo->lastKey < pBlock->maxKey.ts) {
         cur->pos =
             binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order);
       } else {
         cur->pos = pBlock->numOfRows - 1;
       }
 
-      assert(pCheckInfo->lastKey >= pBlock->keyFirst);
+      assert(pCheckInfo->lastKey >= pBlock->minKey.ts);
       doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock);
     } else {
       cur->pos = asc ? 0 : (pBlock->numOfRows - 1);
@@ -1661,7 +1662,11 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
   }
 
 #ifdef TD_DEBUG_PRINT_ROW
-  tdSRowPrint(row1, pSchema1, __func__);
+  char   flags[70] = {0};
+  STsdb* pTsdb = pTsdbReadHandle->rhelper.pRepo;
+  snprintf(flags, 70, "%s:%d vgId:%d dir:%s row1%s=NULL,row2%s=NULL", __func__, __LINE__, TD_VID(pTsdb->pVnode),
+           pTsdb->dir, row1 ? "!" : "", row2 ? "!" : "");
+  tdSRowPrint(row1, pSchema1, flags);
 #endif
 
   if (isRow1DataRow) {
@@ -1981,8 +1986,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
          cur->pos >= 0 && cur->pos < pBlock->numOfRows);
   // Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData interface.
   TSKEY* tsArray = pCols->cols[0].pData;
-  assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst &&
+  assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->minKey.ts &&
-         tsArray[pBlock->numOfRows - 1] == pBlock->keyLast);
+         tsArray[pBlock->numOfRows - 1] == pBlock->maxKey.ts);
 
   bool    ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
   int32_t step = ascScan ? 1 : -1;
@@ -2829,6 +2834,12 @@ void* tsdbGetIdx(SMeta* pMeta) {
   }
   return metaGetIdx(pMeta);
 }
+void* tsdbGetIvtIdx(SMeta* pMeta) {
+  if (pMeta == NULL) {
+    return NULL;
+  }
+  return metaGetIvtIdx(pMeta);
+}
 int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
   SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid);
 
@@ -3576,8 +3587,8 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat
   assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
 
   pPrimaryColStatis->numOfNull = 0;
-  pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst;
+  pPrimaryColStatis->min = pBlockInfo->compBlock->minKey.ts;
-  pPrimaryColStatis->max = pBlockInfo->compBlock->keyLast;
+  pPrimaryColStatis->max = pBlockInfo->compBlock->maxKey.ts;
   pHandle->suppInfo.plist[0] = &pHandle->suppInfo.pstatis[0];
 
   // update the number of NULL data rows

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -339,8 +339,8 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   }
 
   ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
-  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts);
-  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts);
 
   return 0;
 }
@@ -457,8 +457,8 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
   }
 
   ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
-  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts);
-  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts);
 
   return 0;
 }
@@ -559,7 +559,7 @@ int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
   tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
   tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
   tlen += taosEncodeFixedU64(buf, pIdx->uid);
-  tlen += taosEncodeFixedU64(buf, pIdx->maxKey);
+  tlen += taosEncodeFixedU64(buf, pIdx->maxKey.ts);
 
   return tlen;
 }
@@ -579,7 +579,7 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
   if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
   pIdx->uid = (int64_t)value;
   if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
-  pIdx->maxKey = (TSKEY)value;
+  pIdx->maxKey.ts = (TSKEY)value;
 
   return buf;
 }
@@ -726,7 +726,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
     if (dcol != 0) {
       pBlockCol = &(pBlockData->cols[ccol]);
       tcolId = pBlockCol->colId;
-      toffset = tsdbGetBlockColOffset(pBlockCol);
+      toffset = pBlockCol->offset;
       tlen = pBlockCol->len;
       pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0;
     } else {
@@ -942,8 +942,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
   if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
 
-  int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) +
+  int64_t offset =
-                   tsdbGetBlockColOffset(pBlockCol);
+      pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + pBlockCol->offset;
   if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
     tsdbError("vgId:%d, failed to load block column data while seek file %s to offset %" PRId64 " since %s",
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));

source/libs/command/src/explain.c

@@ -153,7 +153,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo
       pPhysiChildren = pSortNode->node.pChildren;
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: {
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: {
       SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
       pPhysiChildren = pIntNode->window.node.pChildren;
       break;
@@ -164,12 +164,12 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo
       break;
     }
     case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: {
-      SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*) pNode;
+      SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode;
       pPhysiChildren = pStateNode->window.node.pChildren;
       break;
     }
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
-      SPartitionPhysiNode* partitionPhysiNode = (SPartitionPhysiNode*) pNode;
+      SPartitionPhysiNode *partitionPhysiNode = (SPartitionPhysiNode *)pNode;
       pPhysiChildren = partitionPhysiNode->node.pChildren;
       break;
     }
@@ -397,7 +397,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
         int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo);
         for (int32_t i = 0; i < nodeNum; ++i) {
-          SExplainExecInfo *     execInfo = taosArrayGet(pResNode->pExecInfo, i);
+          SExplainExecInfo      *execInfo = taosArrayGet(pResNode->pExecInfo, i);
           STableScanAnalyzeInfo *pScanInfo = (STableScanAnalyzeInfo *)execInfo->verboseInfo;
 
           EXPLAIN_ROW_APPEND("total_blocks=%d", pScanInfo->totalBlocks);
@@ -429,7 +429,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
 
-        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey);
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey,
+                        pTblScanNode->scanRange.ekey);
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
 
@@ -641,7 +642,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
       }
 
-      SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc;
+      SDataBlockDescNode *pDescNode = pSortNode->node.pOutputDataBlockDesc;
       EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pDescNode->pSlots));
       EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
       EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pDescNode->totalRowSize);
@@ -667,7 +668,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
         int32_t           nodeNum = taosArrayGetSize(pResNode->pExecInfo);
         SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0);
-        SSortExecInfo *   pExecInfo = (SSortExecInfo *)execInfo->verboseInfo;
+        SSortExecInfo    *pExecInfo = (SSortExecInfo *)execInfo->verboseInfo;
         EXPLAIN_ROW_APPEND("%s", pExecInfo->sortMethod == SORT_QSORT_T ? "quicksort" : "merge sort");
         if (pExecInfo->sortBuffer > 1024 * 1024) {
           EXPLAIN_ROW_APPEND("  Buffers:%.2f Mb", pExecInfo->sortBuffer / (1024 * 1024.0));
@@ -701,7 +702,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
       }
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: {
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: {
       SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
       EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk));
       EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
@@ -784,7 +785,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
     case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: {
       SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode;
 
-      EXPLAIN_ROW_NEW(level, EXPLAIN_STATE_WINDOW_FORMAT, nodesGetNameFromColumnNode(((STargetNode*)pStateNode->pStateKey)->pExpr));
+      EXPLAIN_ROW_NEW(level, EXPLAIN_STATE_WINDOW_FORMAT,
+                      nodesGetNameFromColumnNode(((STargetNode *)pStateNode->pStateKey)->pExpr));
       EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
       if (pResNode->pExecInfo) {
         QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
@@ -823,7 +825,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
       SPartitionPhysiNode *pPartNode = (SPartitionPhysiNode *)pNode;
 
-      SNode* p = nodesListGetNode(pPartNode->pPartitionKeys, 0);
+      SNode *p = nodesListGetNode(pPartNode->pPartitionKeys, 0);
       EXPLAIN_ROW_NEW(level, EXPLAIN_PARITION_FORMAT, nodesGetNameFromColumnNode(p));
       EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
       if (pResNode->pExecInfo) {

source/libs/executor/src/executorimpl.c

@@ -4039,12 +4039,12 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p
     goto _error;
   }
 
-  pInfo->limit      = *pLimit;
+  pInfo->limit = *pLimit;
-  pInfo->slimit     = *pSlimit;
+  pInfo->slimit = *pSlimit;
-  pInfo->curOffset  = pLimit->offset;
+  pInfo->curOffset = pLimit->offset;
   pInfo->curSOffset = pSlimit->offset;
   pInfo->binfo.pRes = pResBlock;
-  pInfo->pFilterNode= pCondition;
+  pInfo->pFilterNode = pCondition;
 
   int32_t numOfCols = num;
   int32_t numOfRows = 4096;
@@ -4389,7 +4389,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       SScanPhysiNode*      pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
       STimeWindowAggSupp   twSup = {
-            .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
+          .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
       tsdbReaderT pDataReader = NULL;
       if (pHandle->vnode) {
         pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
@@ -4447,7 +4447,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
     SLimit       limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset};
     SLimit       slimit = {.limit = pProjPhyNode->slimit, .offset = pProjPhyNode->soffset};
-    pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, pProjPhyNode->node.pConditions, pTaskInfo);
+    pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit,
+                                      pProjPhyNode->node.pConditions, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == type) {
     SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode;
     SExprInfo*     pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
@@ -4467,7 +4468,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       pOptr =
           createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr, pTaskInfo);
     }
-  } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
+  } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
     SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
 
     SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
@@ -4796,7 +4797,8 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa
 
   if (tableType == TSDB_SUPER_TABLE) {
     if (pTagCond) {
-      SIndexMetaArg metaArg = {.metaEx = metaHandle, .metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid};
+      SIndexMetaArg metaArg = {
+          .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
 
       SArray* res = taosArrayInit(8, sizeof(uint64_t));
       code = doFilterTag(pTagCond, &metaArg, res);
@@ -5108,8 +5110,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey,
+int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size_t size) {
-    size_t size) {
   pSup->keySize = sizeof(int64_t) + sizeof(TSKEY);
   pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize);
   pSup->pResultRows = taosArrayInit(1024, size);

source/libs/function/src/udfd.c

@@ -401,9 +401,17 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
     udf->bufSize = pFuncInfo->bufSize;
 
     char path[PATH_MAX] = {0};
+  #ifdef WINDOWS
+    snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name);
+  #else
     snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name);
+  #endif
     TdFilePtr file =
         taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
+    if (file == NULL) {
+      fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno));
+      msgInfo->code = TSDB_CODE_FILE_CORRUPTED;
+    }
     int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
     if (count != pFuncInfo->codeSize) {
       fnError("udfd write udf shared library failed");

source/libs/function/test/udf1.c

@@ -9,15 +9,15 @@
 #undef free
 #define free free
 
-int32_t udf1_init() {
+DLL_EXPORT int32_t udf1_init() {
   return 0;
 }
 
-int32_t udf1_destroy() {
+DLL_EXPORT int32_t udf1_destroy() {
   return 0;
 }
 
-int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) {
+DLL_EXPORT int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) {
   SUdfColumnMeta *meta = &resultCol->colMeta;
   meta->bytes = 4;
   meta->type = TSDB_DATA_TYPE_INT;

source/libs/function/test/udf2.c

@@ -9,22 +9,22 @@
 #undef free
 #define free free
 
-int32_t udf2_init() {
+DLL_EXPORT int32_t udf2_init() {
   return 0;
 }
 
-int32_t udf2_destroy() {
+DLL_EXPORT int32_t udf2_destroy() {
   return 0;
 }
 
-int32_t udf2_start(SUdfInterBuf *buf) {
+DLL_EXPORT int32_t udf2_start(SUdfInterBuf *buf) {
   *(int64_t*)(buf->buf) = 0;
   buf->bufLen = sizeof(double);
   buf->numOfResult = 0;
   return 0;
 }
 
-int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) {
+DLL_EXPORT int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) {
   double sumSquares = *(double*)interBuf->buf;
   int8_t numNotNull = 0;
   for (int32_t i = 0; i < block->numOfCols; ++i) {
@@ -71,7 +71,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte
   return 0;
 }
 
-int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) {
+DLL_EXPORT int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) {
   if (buf->numOfResult == 0) {
     resultData->numOfResult = 0;
     return 0;

source/libs/index/src/index.c

@@ -202,7 +202,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
     char      buf[128] = {0};
     ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
     int32_t   sz = indexSerialCacheKey(&key, buf);
-    indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
+    indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
 
     IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     assert(*cache != NULL);
@@ -330,7 +330,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
   char      buf[128] = {0};
   ICacheKey key = {
       .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType};
-  indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
+  indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
   int32_t sz = indexSerialCacheKey(&key, buf);
 
   taosThreadMutexLock(&sIdx->mtx);

source/libs/index/src/indexComm.c

@@ -402,16 +402,16 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, strlen(src));
+      tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
       *dst = (char*)*dst - tlen;
       break;
     }
     case TSDB_DATA_TYPE_VARBINARY:
-      tlen = taosEncodeBinary(NULL, src, strlen(src));
+      tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, strlen(src));
+      tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
       *dst = (char*)*dst - tlen;
       break;
     default:

source/libs/index/src/indexFilter.c

@@ -31,7 +31,7 @@ typedef struct SIFParam {
   SHashObj *pFilter;
 
   SArray *result;
-  char   *condValue;
+  char *  condValue;
 
   SIdxFltStatus status;
   uint8_t       colValType;
@@ -45,7 +45,7 @@ typedef struct SIFParam {
 
 typedef struct SIFCtx {
   int32_t       code;
-  SHashObj     *pRes;    /* element is SIFParam */
+  SHashObj *    pRes;    /* element is SIFParam */
   bool          noExec;  // true: just iterate condition tree, and add hint to executor plan
   SIndexMetaArg arg;
   // SIdxFltStatus st;
@@ -128,7 +128,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) {
   // covert data From snode;
   SValueNode *vn = (SValueNode *)node;
 
-  char      *pData = nodesGetValueFromNode(vn);
+  char *     pData = nodesGetValueFromNode(vn);
   SDataType *pType = &vn->node.resType;
   int32_t    type = pType->type;
   int32_t    valLen = 0;
@@ -162,12 +162,27 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
+  SOperatorNode *nd = (SOperatorNode *)node;
+  assert(nodeType(node) == QUERY_NODE_OPERATOR);
+  SColumnNode *l = (SColumnNode *)nd->pLeft;
+  SValueNode * r = (SValueNode *)nd->pRight;
+
+  param->colId = l->colId;
+  param->colValType = l->node.resType.type;
+  memcpy(param->dbName, l->dbName, sizeof(l->dbName));
+  sprintf(param->colName, "%s_%s", l->colName, r->literal);
+  param->colValType = r->typeData;
+  return 0;
+  // memcpy(param->colName, l->colName, sizeof(l->colName));
+}
 static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   switch (nodeType(node)) {
     case QUERY_NODE_VALUE: {
       SValueNode *vn = (SValueNode *)node;
       SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
       param->colId = -1;
+      param->colValType = (uint8_t)(vn->node.resType.type);
       break;
     }
     case QUERY_NODE_COLUMN: {
@@ -219,17 +234,31 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
     indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight);
     SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
+  if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
+    return code;
+  }
   SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam));
   if (NULL == paramList) {
     SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
-  SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
+  if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) {
-  if (nParam > 1) {
+    SNode *interNode = (node->pLeft);
-    SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
+    SIF_ERR_JRET(sifInitJsonParam(interNode, &paramList[0], ctx));
+    if (nParam > 1) {
+      SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
+    }
+    paramList[0].colValType = TSDB_DATA_TYPE_JSON;
+    *params = paramList;
+    return TSDB_CODE_SUCCESS;
+  } else {
+    SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
+    if (nParam > 1) {
+      SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
+    }
+    *params = paramList;
+    return TSDB_CODE_SUCCESS;
   }
-  *params = paramList;
-  return TSDB_CODE_SUCCESS;
 _return:
   taosMemoryFree(paramList);
   SIF_RET(code);
@@ -306,39 +335,34 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
 }
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
   SIndexMetaArg *arg = &output->arg;
-#ifdef USE_INVERTED_INDEX
+  int            ret = 0;
-  SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName),
+
-                                   right->condValue, strlen(right->condValue));
+  EIndexQueryType qtype = 0;
-  if (tm == NULL) {
+  SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
-    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  if (left->colValType == TSDB_DATA_TYPE_JSON) {
+    SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName),
+                                     right->condValue, strlen(right->condValue));
+    if (tm == NULL) {
+      return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    }
+
+    SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
+    indexMultiTermQueryAdd(mtm, tm, qtype);
+    ret = tIndexJsonSearch(arg->ivtIdx, mtm, output->result);
+  } else {
+    bool   reverse;
+    Filter filterFunc = sifGetFilterFunc(qtype, &reverse);
+
+    SMetaFltParam param = {.suid = arg->suid,
+                           .cid = left->colId,
+                           .type = left->colValType,
+                           .val = right->condValue,
+                           .reverse = reverse,
+                           .filterFunc = filterFunc};
+
+    ret = metaFilteTableIds(arg->metaEx, &param, output->result);
   }
-
-  EIndexQueryType qtype = 0;
-  SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
-
-  SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
-  indexMultiTermQueryAdd(mtm, tm, qtype);
-  int ret = indexSearch(arg->metaHandle, mtm, output->result);
-  indexDebug("index filter data size: %d", (int)taosArrayGetSize(output->result));
-  indexMultiTermQueryDestroy(mtm);
   return ret;
-#else
-  EIndexQueryType qtype = 0;
-  SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
-  bool   reverse;
-  Filter filterFunc = sifGetFilterFunc(qtype, &reverse);
-
-  SMetaFltParam param = {.suid = arg->suid,
-                         .cid = left->colId,
-                         .type = left->colValType,
-                         .val = right->condValue,
-                         .reverse = reverse,
-                         .filterFunc = filterFunc};
-
-  int ret = metaFilteTableIds(arg->metaEx, &param, output->result);
-  return ret;
-#endif
-  return 0;
 }
 
 static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
@@ -392,6 +416,14 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output
   int id = OP_TYPE_NMATCH;
   return sifDoIndex(left, right, id, output);
 }
+static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) {
+  // return 0
+  return 0;
+}
+static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) {
+  // return 0
+  return 0;
+}
 
 static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   // add more except
@@ -445,6 +477,14 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta
       *status = SFLT_NOT_INDEX;
       *func = sifNotMatchFunc;
       return 0;
+    case OP_TYPE_JSON_CONTAINS:
+      *status = SFLT_ACCURATE_INDEX;
+      *func = sifJsonContains;
+      return 0;
+    case OP_TYPE_JSON_GET_VALUE:
+      *status = SFLT_ACCURATE_INDEX;
+      *func = sifJsonGetValue;
+      return 0;
     default:
       *status = SFLT_NOT_INDEX;
       *func = sifNullFunc;
@@ -457,12 +497,17 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta
 static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
   int32_t code = 0;
   int32_t nParam = sifGetOperParamNum(node->opType);
+  if (nParam <= 1) {
+    SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+  if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
+    return code;
+  }
   SIFParam *params = NULL;
-  SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
 
+  SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
   // ugly code, refactor later
   output->arg = ctx->arg;
-
   sif_func_t operFn = sifNullFunc;
   code = sifGetOperFn(node->opType, &operFn, &output->status);
   if (!ctx->noExec) {
@@ -567,7 +612,9 @@ EDealRes sifCalcWalker(SNode *node, void *context) {
   if (QUERY_NODE_LOGIC_CONDITION == nodeType(node)) {
     return sifWalkLogic(node, ctx);
   }
+
   if (QUERY_NODE_OPERATOR == nodeType(node)) {
+    indexInfo("node type for index filter, type: %d", nodeType(node));
     return sifWalkOper(node, ctx);
   }
 
@@ -658,7 +705,7 @@ int32_t doFilterTag(const SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *re
   // todo move to the initialization function
   // SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0));
 
-  SArray  *output = taosArrayInit(8, sizeof(uint64_t));
+  SArray * output = taosArrayInit(8, sizeof(uint64_t));
   SIFParam param = {.arg = *metaArg, .result = output};
   SIF_ERR_RET(sifCalculate((SNode *)pFilterNode, &param));
 

source/libs/index/test/indexTests.cc

@@ -498,10 +498,15 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
   }
   taosArrayDestroy(data);
 
-  std::string     colName("voltage");
+  std::string colName("voltage");
-  std::string     colVal("ab");
+  std::string colVal("ab");
-  SIndexTerm*     term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+
-                                     colVal.c_str(), colVal.size());
+  char    buf[256] = {0};
+  int16_t sz = colVal.size();
+  memcpy(buf, (uint16_t*)&sz, 2);
+  memcpy(buf + 2, colVal.c_str(), colVal.size());
+  SIndexTerm* term =
+      indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), buf, sizeof(buf));
   SIndexTermQuery query = {term, QUERY_TERM};
 
   SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@@ -564,6 +569,18 @@ class IndexCacheEnv : public ::testing::Test {
   CacheObj* coj;
 };
 
+SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
+                             int32_t nColName, const char* colVal, int32_t nColVal) {
+  char    buf[256] = {0};
+  int16_t sz = nColVal;
+  memcpy(buf, (uint16_t*)&sz, 2);
+  memcpy(buf + 2, colVal, nColVal);
+  if (colType == TSDB_DATA_TYPE_BINARY) {
+    return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
+  } else {
+    return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
+  }
+}
 #define MAX_TERM_KEY_LEN 128
 TEST_F(IndexCacheEnv, cache_test) {
   int     version = 0;
@@ -574,37 +591,37 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::string colName("voltage");
   {
     std::string colVal("v1");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
     // indexTermDestry(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v2");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
@@ -612,15 +629,15 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::cout << "--------first----------" << std::endl;
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v4");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
   }
@@ -630,8 +647,8 @@ TEST_F(IndexCacheEnv, cache_test) {
     std::string colVal("v4");
     for (size_t i = 0; i < 10; i++) {
       colVal[colVal.size() - 1] = 'a' + i;
-      SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                         colVal.c_str(), colVal.size());
+                                          colVal.c_str(), colVal.size());
       coj->Put(term, colId, version++, suid++);
       indexTermDestroy(term);
     }
@@ -640,8 +657,8 @@ TEST_F(IndexCacheEnv, cache_test) {
   // begin query
   {
     std::string     colVal("v3");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
     STermValueType  valType;
@@ -655,8 +672,8 @@ TEST_F(IndexCacheEnv, cache_test) {
   }
   {
     std::string     colVal("v2");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
     STermValueType  valType;
@@ -690,8 +707,8 @@ class IndexObj {
     return ret;
   }
   void Del(const std::string& colName, const std::string& colVal, uint64_t uid) {
-    SIndexTerm*      term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
     Put(terms, uid);
@@ -699,8 +716,8 @@ class IndexObj {
   }
   int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                       size_t numOfTable = 100 * 10000) {
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
     for (size_t i = 0; i < numOfTable; i++) {
@@ -721,8 +738,8 @@ class IndexObj {
         // opt
         tColVal[taosRand() % colValSize] = 'a' + k % 26;
       }
-      SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                         tColVal.c_str(), tColVal.size());
+                                          tColVal.c_str(), tColVal.size());
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
       for (size_t j = 0; j < skip; j++) {
@@ -757,8 +774,8 @@ class IndexObj {
 
   int SearchOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
     SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@@ -779,8 +796,8 @@ class IndexObj {
   }
   int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
     SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@@ -804,16 +821,16 @@ class IndexObj {
 
   void PutOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, 10);
     indexMultiTermDestroy(terms);
   }
   void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, val);
     indexMultiTermDestroy(terms);
@@ -858,8 +875,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
     for (size_t i = 0; i < targetSize; i++) {
@@ -873,8 +890,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
     for (size_t i = 0; i < size; i++) {
@@ -888,8 +905,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
     for (size_t i = size * 3; i < size * 4; i++) {
@@ -903,8 +920,8 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string           colName("tag1"), colVal("Hello");
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
     SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@@ -926,8 +943,8 @@ TEST_F(IndexEnv2, testEmptyIndexOpen) {
   {
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
     for (size_t i = 0; i < targetSize; i++) {

source/libs/index/test/jsonUT.cc

@@ -19,6 +19,18 @@
 static std::string dir = TD_TMP_DIR_PATH "json";
 static std::string logDir = TD_TMP_DIR_PATH "log";
 
+SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
+                             int32_t nColName, const char* colVal, int32_t nColVal) {
+  char    buf[256] = {0};
+  int16_t sz = nColVal;
+  memcpy(buf, (uint16_t*)&sz, 2);
+  memcpy(buf + 2, colVal, nColVal);
+  if (colType == TSDB_DATA_TYPE_BINARY) {
+    return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
+  } else {
+    return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
+  }
+}
 static void initLog() {
   const char*   defaultLogFileNamePrefix = "taoslog";
   const int32_t maxLogFileNum = 10;
@@ -59,8 +71,8 @@ class JsonEnv : public ::testing::Test {
 
 static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
                       int8_t operType = ADD_VALUE) {
-  SIndexTerm* term =
+  SIndexTerm*      term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(),
-      indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+                                      (const char*)data, dlen);
   SIndexMultiTerm* terms = indexMultiTermCreate();
   indexMultiTermAdd(terms, term);
   tIndexJsonPut(index, terms, (int64_t)tableId);
@@ -70,8 +82,8 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp
 
 static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
                     int8_t operType = DEL_VALUE) {
-  SIndexTerm* term =
+  SIndexTerm*      term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(),
-      indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+                                      (const char*)data, dlen);
   SIndexMultiTerm* terms = indexMultiTermCreate();
   indexMultiTermAdd(terms, term);
   tIndexJsonPut(index, terms, (int64_t)tableId);
@@ -83,7 +95,7 @@ static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, v
   std::string colName(colNam);
 
   SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-  SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
+  SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
 
   SArray* res = taosArrayInit(1, sizeof(uint64_t));
   indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType);
@@ -95,8 +107,8 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -108,8 +120,8 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("ab1");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -121,8 +133,8 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("123");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -136,8 +148,8 @@ TEST_F(JsonEnv, testWrite) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                    colVal.c_str(), colVal.size());
+                                     colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -150,8 +162,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -165,8 +177,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("abxxxxxxxxxxxx");
     for (int i = 0; i < 10; i++) {
       colVal[i % colVal.size()] = '0' + i % 128;
-      SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                         colVal.c_str(), colVal.size());
+                                          colVal.c_str(), colVal.size());
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
@@ -179,8 +191,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("voltagefdadfa");
     std::string colVal("abxxxxxxxxxxxx");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -194,8 +206,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                    colVal.c_str(), colVal.size());
+                                     colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -209,8 +221,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
       std::string colVal("ab");
 
       SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-      SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                      colVal.c_str(), colVal.size());
+                                       colVal.c_str(), colVal.size());
 
       SArray* result = taosArrayInit(1, sizeof(uint64_t));
       indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -224,8 +236,8 @@ TEST_F(JsonEnv, testWriteMillonData) {
         std::string colVal("ab");
 
         SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-        SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+        SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                        colVal.c_str(), colVal.size());
+                                         colVal.c_str(), colVal.size());
 
         SArray* result = taosArrayInit(1, sizeof(uint64_t));
         indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@@ -241,8 +253,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     std::string colName("test");
     // std::string colVal("10");
     int         val = 10;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       (const char*)&val, sizeof(val));
+                                        (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -254,8 +266,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test2");
     int         val = 20;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       (const char*)&val, sizeof(val));
+                                        (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -267,8 +279,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test");
     int         val = 15;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       (const char*)&val, sizeof(val));
+                                        (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -280,8 +292,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
   {
     std::string colName("test2");
     const char* val = "test";
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       (const char*)val, strlen(val));
+                                        (const char*)val, strlen(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -294,8 +306,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     std::string           colName("test");
     int                   val = 15;
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -308,8 +320,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     int         val = 15;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -323,8 +335,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     ;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(int));
+                                     (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@@ -338,8 +350,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@@ -353,8 +365,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@@ -368,8 +380,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test1");
     int         val = 10;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       (const char*)&val, sizeof(val));
+                                        (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -381,8 +393,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test");
     std::string colVal("xxxxxxxxxxxxxxxxxxx");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                       colVal.c_str(), colVal.size());
+                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -396,8 +408,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
@@ -410,8 +422,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(int));
+                                     (const char*)&val, sizeof(int));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -425,8 +437,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
@@ -439,8 +451,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
@@ -453,8 +465,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     int         val = 10;
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
@@ -466,8 +478,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     std::string colName("other_column");
     int         val = 100;
 
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       (const char*)&val, sizeof(val));
+                                        (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -482,8 +494,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
@@ -494,8 +506,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
   {
     std::string colName("test1");
     int         val = 15;
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                       (const char*)&val, sizeof(val));
+                                        (const char*)&val, sizeof(val));
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -510,8 +522,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
     // std::string colVal("10");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
-                                    (const char*)&val, sizeof(val));
+                                     (const char*)&val, sizeof(val));
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);

source/libs/nodes/src/nodesCloneFuncs.c

@@ -316,7 +316,8 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   CLONE_NODE_LIST_FIELD(pScanCols);
   CLONE_NODE_LIST_FIELD(pScanPseudoCols);
-  CLONE_OBJECT_FIELD(pMeta, tableMetaClone);
+  COPY_SCALAR_FIELD(tableType);
+  COPY_SCALAR_FIELD(tableId);
   CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
   COPY_SCALAR_FIELD(scanType);
   COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2);
@@ -365,9 +366,15 @@ static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode*
   return (SNode*)pDst;
 }
 
-static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) {
+static SNode* logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModifyLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
+  COPY_SCALAR_FIELD(modifyType);
   COPY_SCALAR_FIELD(msgType);
+  CLONE_NODE_FIELD(pModifyRows);
+  COPY_SCALAR_FIELD(tableId);
+  COPY_SCALAR_FIELD(tableType);
+  COPY_CHAR_ARRAY_FIELD(tableFName);
+  COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow));
   return (SNode*)pDst;
 }
 
@@ -400,7 +407,7 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD
   COPY_SCALAR_FIELD(triggerType);
   COPY_SCALAR_FIELD(watermark);
   COPY_SCALAR_FIELD(filesFactor);
-  COPY_SCALAR_FIELD(stmInterAlgo);
+  COPY_SCALAR_FIELD(intervalAlgo);
   return (SNode*)pDst;
 }
 
@@ -542,8 +549,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
       return logicAggCopy((const SAggLogicNode*)pNode, (SAggLogicNode*)pDst);
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
       return logicProjectCopy((const SProjectLogicNode*)pNode, (SProjectLogicNode*)pDst);
-    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
+    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:
-      return logicVnodeModifCopy((const SVnodeModifLogicNode*)pNode, (SVnodeModifLogicNode*)pDst);
+      return logicVnodeModifCopy((const SVnodeModifyLogicNode*)pNode, (SVnodeModifyLogicNode*)pDst);
     case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
       return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst);
     case QUERY_NODE_LOGIC_PLAN_MERGE:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -188,8 +188,8 @@ const char* nodesNodeName(ENodeType type) {
       return "LogicAgg";
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
       return "LogicProject";
-    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
+    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:
-      return "LogicVnodeModif";
+      return "LogicVnodeModify";
     case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
       return "LogicExchange";
     case QUERY_NODE_LOGIC_PLAN_MERGE:
@@ -228,8 +228,8 @@ const char* nodesNodeName(ENodeType type) {
       return "PhysiMerge";
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return "PhysiSort";
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
-      return "PhysiInterval";
+      return "PhysiHashInterval";
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
       return "PhysiStreamInterval";
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
@@ -252,6 +252,8 @@ const char* nodesNodeName(ENodeType type) {
       return "PhysiDispatch";
     case QUERY_NODE_PHYSICAL_PLAN_INSERT:
       return "PhysiInsert";
+    case QUERY_NODE_PHYSICAL_PLAN_DELETE:
+      return "PhysiDelete";
     case QUERY_NODE_PHYSICAL_SUBPLAN:
       return "PhysiSubplan";
     case QUERY_NODE_PHYSICAL_PLAN:
@@ -504,8 +506,8 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) {
 
 static const char* jkScanLogicPlanScanCols = "ScanCols";
 static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
-static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize";
+static const char* jkScanLogicPlanTableId = "TableId";
-static const char* jkScanLogicPlanTableMeta = "TableMeta";
+static const char* jkScanLogicPlanTableType = "TableType";
 static const char* jkScanLogicPlanTagCond = "TagCond";
 
 static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
@@ -519,10 +521,10 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
     code = nodeListToJson(pJson, jkScanLogicPlanScanPseudoCols, pNode->pScanPseudoCols);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta));
+    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableId, pNode->tableId);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta);
+    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableType, pNode->tableType);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond);
@@ -543,10 +545,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
     code = jsonToNodeList(pJson, jkScanLogicPlanScanPseudoCols, &pNode->pScanPseudoCols);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize);
+    code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanTableId, &pNode->tableId);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize);
+    code = tjsonGetTinyIntValue(pJson, jkScanLogicPlanTableType, &pNode->tableType);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond);
@@ -578,7 +580,7 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
     code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSlimit, pNode->slimit);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, pNode->soffset);
+    code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSoffset, pNode->soffset);
   }
 
   return code;
@@ -601,7 +603,45 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
     code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSlimit, &pNode->slimit);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetBigIntValue(pJson, jkScanLogicPlanTableMetaSize, &pNode->soffset);
+    code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSoffset, &pNode->soffset);
+  }
+
+  return code;
+}
+
+static const char* jkVnodeModifyLogicPlanModifyType = "ModifyType";
+static const char* jkVnodeModifyLogicPlanMsgType = "MsgType";
+static const char* jkVnodeModifyLogicPlanModifyRows = "ModifyRows";
+
+static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) {
+  const SVnodeModifyLogicNode* pNode = (const SVnodeModifyLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanModifyType, pNode->modifyType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanMsgType, pNode->msgType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanModifyRows, nodeToJson, pNode->pModifyRows);
+  }
+
+  return code;
+}
+
+static int32_t jsonToLogicVnodeModifyNode(const SJson* pJson, void* pObj) {
+  SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pObj;
+
+  int32_t code = jsonToLogicPlanNode(pJson, pObj);
+  if (TSDB_CODE_SUCCESS == code) {
+    tjsonGetNumberValue(pJson, jkVnodeModifyLogicPlanModifyType, pNode->modifyType, code);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkVnodeModifyLogicPlanMsgType, &pNode->msgType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanModifyRows, &pNode->pModifyRows);
   }
 
   return code;
@@ -1955,6 +1995,58 @@ static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return
 
 static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); }
 
+static const char* jkDeletePhysiPlanTableId = "TableId";
+static const char* jkDeletePhysiPlanTableType = "TableType";
+static const char* jkDeletePhysiPlanTableFName = "TableFName";
+static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey";
+static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey";
+
+static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) {
+  const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj;
+
+  int32_t code = physicDataSinkNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanTableId, pNode->tableId);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanTableType, pNode->tableType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, pNode->deleteTimeRange.skey);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, pNode->deleteTimeRange.ekey);
+  }
+
+  return code;
+}
+
+static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) {
+  SDataDeleterNode* pNode = (SDataDeleterNode*)pObj;
+
+  int32_t code = jsonToPhysicDataSinkNode(pJson, pObj);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetUBigIntValue(pJson, jkDeletePhysiPlanTableId, &pNode->tableId);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDeletePhysiPlanTableType, &pNode->tableType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, &pNode->deleteTimeRange.skey);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, &pNode->deleteTimeRange.ekey);
+  }
+
+  return code;
+}
+
 static const char* jkQueryNodeAddrId = "Id";
 static const char* jkQueryNodeAddrInUse = "InUse";
 static const char* jkQueryNodeAddrNumOfEps = "NumOfEps";
@@ -2751,7 +2843,7 @@ static const char* jkStateWindowExpr = "StateWindowExpr";
 
 static int32_t stateWindowNodeToJson(const void* pObj, SJson* pJson) {
   const SStateWindowNode* pNode = (const SStateWindowNode*)pObj;
-    int32_t code = tjsonAddObject(pJson, jkStateWindowCol, nodeToJson, pNode->pCol);
+  int32_t                 code = tjsonAddObject(pJson, jkStateWindowCol, nodeToJson, pNode->pCol);
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddObject(pJson, jkStateWindowExpr, nodeToJson, pNode->pExpr);
   }
@@ -3522,6 +3614,73 @@ static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkDeleteStmtFromTable = "FromTable";
+static const char* jkDeleteStmtWhere = "Where";
+static const char* jkDeleteStmtCountFunc = "CountFunc";
+static const char* jkDeleteStmtTagIndexCond = "TagIndexCond";
+static const char* jkDeleteStmtTimeRangeStartKey = "TimeRangeStartKey";
+static const char* jkDeleteStmtTimeRangeEndKey = "TimeRangeEndKey";
+static const char* jkDeleteStmtPrecision = "Precision";
+static const char* jkDeleteStmtDeleteZeroRows = "DeleteZeroRows";
+
+static int32_t deleteStmtToJson(const void* pObj, SJson* pJson) {
+  const SDeleteStmt* pNode = (const SDeleteStmt*)pObj;
+
+  int32_t code = tjsonAddObject(pJson, jkDeleteStmtFromTable, nodeToJson, pNode->pFromTable);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkDeleteStmtWhere, nodeToJson, pNode->pWhere);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkDeleteStmtCountFunc, nodeToJson, pNode->pCountFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagIndexCond);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeStartKey, pNode->timeRange.skey);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeEndKey, pNode->timeRange.ekey);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDeleteStmtPrecision, pNode->precision);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkDeleteStmtDeleteZeroRows, pNode->deleteZeroRows);
+  }
+
+  return code;
+}
+
+static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) {
+  SDeleteStmt* pNode = (SDeleteStmt*)pObj;
+
+  int32_t code = jsonToNodeObject(pJson, jkDeleteStmtFromTable, &pNode->pFromTable);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkDeleteStmtWhere, &pNode->pWhere);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkDeleteStmtCountFunc, &pNode->pCountFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagIndexCond);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeStartKey, &pNode->timeRange.skey);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeEndKey, &pNode->timeRange.ekey);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetUTinyIntValue(pJson, jkDeleteStmtPrecision, &pNode->precision);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkDeleteStmtDeleteZeroRows, &pNode->deleteZeroRows);
+  }
+
+  return code;
+}
+
 static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
   switch (nodeType(pObj)) {
     case QUERY_NODE_COLUMN:
@@ -3594,6 +3753,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       break;
     case QUERY_NODE_CREATE_TOPIC_STMT:
       return createTopicStmtToJson(pObj, pJson);
+    case QUERY_NODE_DELETE_STMT:
+      return deleteStmtToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_SCAN:
       return logicScanNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_JOIN:
@@ -3602,8 +3763,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return logicAggNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
       return logicProjectNodeToJson(pObj, pJson);
-    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
+    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:
-      break;
+      return logicVnodeModifyNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
       return logicExchangeNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_MERGE:
@@ -3640,7 +3801,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return physiMergeNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return physiSortNodeToJson(pObj, pJson);
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
@@ -3659,6 +3820,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return physiDispatchNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_INSERT:
       break;
+    case QUERY_NODE_PHYSICAL_PLAN_DELETE:
+      return physiDeleteNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_SUBPLAN:
       return subplanToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN:
@@ -3722,10 +3885,14 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToAlterDnodeStmt(pJson, pObj);
     case QUERY_NODE_CREATE_TOPIC_STMT:
       return jsonToCreateTopicStmt(pJson, pObj);
+    case QUERY_NODE_DELETE_STMT:
+      return jsonToDeleteStmt(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_SCAN:
       return jsonToLogicScanNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
       return jsonToLogicProjectNode(pJson, pObj);
+    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:
+      return jsonToLogicVnodeModifyNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
       return jsonToLogicExchangeNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_MERGE:
@@ -3762,7 +3929,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToPhysiMergeNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return jsonToPhysiSortNode(pJson, pObj);
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
@@ -3779,6 +3946,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToPhysiPartitionNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return jsonToPhysiDispatchNode(pJson, pObj);
+    case QUERY_NODE_PHYSICAL_PLAN_DELETE:
+      return jsonToPhysiDeleteNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_SUBPLAN:
       return jsonToSubplan(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN:

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -512,7 +512,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
       }
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
       res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
       break;
@@ -520,7 +520,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW:
       res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);
       break;
-    case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: 
+    case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: {
       SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)pNode;
       res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext);

source/libs/nodes/src/nodesUtilFuncs.c

@@ -218,8 +218,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SAggLogicNode));
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
       return makeNode(type, sizeof(SProjectLogicNode));
-    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
+    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:
-      return makeNode(type, sizeof(SVnodeModifLogicNode));
+      return makeNode(type, sizeof(SVnodeModifyLogicNode));
     case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
       return makeNode(type, sizeof(SExchangeLogicNode));
     case QUERY_NODE_LOGIC_PLAN_MERGE:
@@ -258,8 +258,10 @@ SNodeptr nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SMergePhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return makeNode(type, sizeof(SSortPhysiNode));
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
       return makeNode(type, sizeof(SIntervalPhysiNode));
+    case QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL:
+      return makeNode(type, sizeof(SSortMergeIntervalPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
       return makeNode(type, sizeof(SStreamIntervalPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
@@ -282,6 +284,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SDataDispatcherNode));
     case QUERY_NODE_PHYSICAL_PLAN_INSERT:
       return makeNode(type, sizeof(SDataInserterNode));
+    case QUERY_NODE_PHYSICAL_PLAN_DELETE:
+      return makeNode(type, sizeof(SDataDeleterNode));
     case QUERY_NODE_PHYSICAL_SUBPLAN:
       return makeNode(type, sizeof(SSubplan));
     case QUERY_NODE_PHYSICAL_PLAN:
@@ -561,7 +565,6 @@ void nodesDestroyNode(SNodeptr pNode) {
       SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode;
       destroyLogicNode((SLogicNode*)pLogicNode);
       nodesDestroyList(pLogicNode->pScanCols);
-      taosMemoryFreeClear(pLogicNode->pMeta);
       taosMemoryFreeClear(pLogicNode->pVgroupList);
       break;
     }
@@ -584,8 +587,8 @@ void nodesDestroyNode(SNodeptr pNode) {
       nodesDestroyList(pLogicNode->pProjections);
       break;
     }
-    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: {
+    case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: {
-      SVnodeModifLogicNode* pLogicNode = (SVnodeModifLogicNode*)pNode;
+      SVnodeModifyLogicNode* pLogicNode = (SVnodeModifyLogicNode*)pNode;
       destroyLogicNode((SLogicNode*)pLogicNode);
       destroyVgDataBlockArray(pLogicNode->pDataBlocks);
       // pVgDataBlocks is weak reference
@@ -673,7 +676,7 @@ void nodesDestroyNode(SNodeptr pNode) {
       nodesDestroyNode(pPhyNode->pSortKeys);
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
       destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode);
       break;
@@ -1066,7 +1069,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) {
 bool nodesIsExprNode(const SNode* pNode) {
   ENodeType type = nodeType(pNode);
   return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type ||
-          QUERY_NODE_FUNCTION == type);
+          QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type);
 }
 
 bool nodesIsUnaryOp(const SOperatorNode* pOp) {
@@ -1093,7 +1096,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp) {
     case OP_TYPE_SUB:
     case OP_TYPE_MULTI:
     case OP_TYPE_DIV:
-    case OP_TYPE_MOD:
+    case OP_TYPE_REM:
       return true;
     default:
       break;

source/libs/parser/inc/sql.y

@@ -611,7 +611,7 @@ expression(A) ::= expression(B) NK_SLASH expression(C).
 expression(A) ::= expression(B) NK_REM expression(C).                             {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, B);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, C);
-                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); 
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
                                                                                   }
 expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C).                      {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, B);

source/libs/parser/src/parAstCreater.c

@@ -1489,11 +1489,27 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb
   return (SNode*)pStmt;
 }
 
+SNode* createCountFuncForDelete(SAstCreateContext* pCxt) {
+  SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION);
+  CHECK_OUT_OF_MEM(pFunc);
+  strcpy(pFunc->functionName, "count");
+  if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt))) {
+    nodesDestroyNode(pFunc);
+    CHECK_OUT_OF_MEM(NULL);
+  }
+  return (SNode*)pFunc;
+}
+
 SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) {
   CHECK_PARSER_STATUS(pCxt);
   SDeleteStmt* pStmt = nodesMakeNode(QUERY_NODE_DELETE_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   pStmt->pFromTable = pTable;
   pStmt->pWhere = pWhere;
+  pStmt->pCountFunc = createCountFuncForDelete(pCxt);
+  if (NULL == pStmt->pCountFunc) {
+    nodesDestroyNode(pStmt);
+    CHECK_OUT_OF_MEM(NULL);
+  }
   return (SNode*)pStmt;
 }

source/libs/parser/src/parTranslater.c

@@ -938,7 +938,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
   if (hasInvalidFuncNesting(pFunc->pParameterList)) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING);
   }
-  if (pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc) {
+  if (NULL != pCxt->pCurrSelectStmt && pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
   }
 
@@ -2124,6 +2124,7 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) {
     code = translateDeleteWhere(pCxt, pDelete);
   }
   if (TSDB_CODE_SUCCESS == code) {
+    pCxt->currClause = SQL_CLAUSE_SELECT;
     code = translateExpr(pCxt, &pDelete->pCountFunc);
   }
   return code;

source/libs/parser/src/sql.c

@@ -4079,7 +4079,7 @@ static YYACTIONTYPE yy_reduce(
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172);
-                                                                                    yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); 
+                                                                                    yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172)));
                                                                                   }
   yymsp[-2].minor.yy172 = yylhsminor.yy172;
         break;

source/libs/parser/test/parInitialDTest.cpp

@@ -28,6 +28,10 @@ TEST_F(ParserInitialDTest, delete) {
   run("DELETE FROM t1");
 
   run("DELETE FROM t1 WHERE ts > now - 2d and ts < now - 1d");
+
+  run("DELETE FROM st1");
+
+  run("DELETE FROM st1 WHERE ts > now - 2d and ts < now - 1d AND tag1 = 10");
 }
 
 TEST_F(ParserInitialDTest, deleteSemanticCheck) {

source/libs/planner/src/planLogicCreater.c

@@ -98,7 +98,15 @@ static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESq
   return cxt.errCode;
 }
 
+static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) {
+  nodesWalkExprs(pExprs, doNameExpr, NULL);
+  SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs};
+  nodesRewriteExpr(pTarget, doRewriteExpr, &cxt);
+  return cxt.errCode;
+}
+
 static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) {
+  nodesWalkExprs(pExprs, doNameExpr, NULL);
   SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs};
   nodesRewriteExprs(pTarget, doRewriteExpr, &cxt);
   return cxt.errCode;
@@ -141,7 +149,7 @@ static int32_t createSelectRootLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
 }
 
 static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols,
-                             STableMeta* pMeta) {
+                             int8_t tableType) {
   if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) {
     return SCAN_TYPE_STREAM;
   }
@@ -151,7 +159,7 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
     return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG;
   }
 
-  if (TSDB_SYSTEM_TABLE == pMeta->tableType) {
+  if (TSDB_SYSTEM_TABLE == tableType) {
     return SCAN_TYPE_SYSTEM_TABLE;
   }
 
@@ -204,16 +212,18 @@ static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable,
+static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealTable, bool hasRepeatScanFuncs,
-                                   SLogicNode** pLogicNode) {
+                                 SLogicNode** pLogicNode) {
   SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
   if (NULL == pScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  TSWAP(pScan->pMeta, pRealTable->pMeta);
+  // TSWAP(pScan->pMeta, pRealTable->pMeta);
   TSWAP(pScan->pVgroupList, pRealTable->pVgroupList);
-  pScan->scanSeq[0] = pSelect->hasRepeatScanFuncs ? 2 : 1;
+  pScan->tableId = pRealTable->pMeta->uid;
+  pScan->tableType = pRealTable->pMeta->tableType;
+  pScan->scanSeq[0] = hasRepeatScanFuncs ? 2 : 1;
   pScan->scanSeq[1] = 0;
   pScan->scanRange = TSWINDOW_INITIALIZER;
   pScan->tableName.type = TSDB_TABLE_NAME_T;
@@ -224,9 +234,21 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   pScan->ratio = pRealTable->ratio;
   pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD;
 
+  *pLogicNode = (SLogicNode*)pScan;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable,
+                                   SLogicNode** pLogicNode) {
+  SScanLogicNode* pScan = NULL;
+  int32_t         code = makeScanLogicNode(pCxt, pRealTable, pSelect->hasRepeatScanFuncs, (SLogicNode**)&pScan);
+
   // set columns to scan
-  int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL,
+  if (TSDB_CODE_SUCCESS == code) {
-                                     &pScan->pScanCols);
+    code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL,
+                               &pScan->pScanCols);
+  }
 
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG,
@@ -242,10 +264,10 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
     code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM);
   }
 
-  pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta);
+  pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType);
 
   if (TSDB_CODE_SUCCESS == code) {
-    code = addPrimaryKeyCol(pScan->pMeta->uid, &pScan->pScanCols);
+    code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols);
   }
 
   // set output
@@ -518,7 +540,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva
   pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval);
   pWindow->slidingUnit =
       (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit);
-  pWindow->stmInterAlgo = STREAM_INTERVAL_ALGO_SINGLE;
+  pWindow->intervalAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH;
 
   pWindow->pTspk = nodesCloneNode(pInterval->pCol);
   if (NULL == pWindow->pTspk) {
@@ -946,10 +968,11 @@ static int32_t getMsgType(ENodeType sqlType) {
 }
 
 static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) {
-  SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF);
+  SVnodeModifyLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY);
   if (NULL == pModif) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pModif->modifyType = MODIFY_TABLE_TYPE_INSERT;
   TSWAP(pModif->pDataBlocks, pStmt->pDataBlocks);
   pModif->msgType = getMsgType(pStmt->sqlNodeType);
   *pLogicNode = (SLogicNode*)pModif;
@@ -962,26 +985,96 @@ static int32_t createDeleteRootLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p
 }
 
 static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) {
-  return TSDB_CODE_FAILED;
+  SScanLogicNode* pScan = NULL;
+  int32_t         code = makeScanLogicNode(pCxt, (SRealTableNode*)pDelete->pFromTable, false, (SLogicNode**)&pScan);
+
+  // set columns to scan
+  if (TSDB_CODE_SUCCESS == code) {
+    pScan->scanType = SCAN_TYPE_TABLE;
+    pScan->pScanCols = nodesCloneList(((SFunctionNode*)pDelete->pCountFunc)->pParameterList);
+    if (NULL == pScan->pScanCols) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagIndexCond) {
+    pScan->pTagCond = nodesCloneNode(pDelete->pTagIndexCond);
+    if (NULL == pScan->pTagCond) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  // set output
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createColumnByRewriteExps(pScan->pScanCols, &pScan->node.pTargets);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pLogicNode = (SLogicNode*)pScan;
+  } else {
+    nodesDestroyNode(pScan);
+  }
+
+  return code;
 }
 
 static int32_t createDeleteAggLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) {
-  return TSDB_CODE_FAILED;
+  SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
+  if (NULL == pAgg) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, nodesCloneNode(pDelete->pCountFunc));
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pCountFunc);
+  }
+  // set the output
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createColumnByRewriteExps(pAgg->pAggFuncs, &pAgg->node.pTargets);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pLogicNode = (SLogicNode*)pAgg;
+  } else {
+    nodesDestroyNode(pAgg);
+  }
+
+  return code;
 }
 
-static int32_t createDeleteModifyTableLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete,
+static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDeleteStmt* pDelete,
-                                                SLogicNode** pLogicNode) {
+                                                 SLogicNode** pLogicNode) {
-  return TSDB_CODE_FAILED;
+  SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY);
+  if (NULL == pModify) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  SRealTableNode* pRealTable = (SRealTableNode*)pDelete->pFromTable;
+
+  pModify->modifyType = MODIFY_TABLE_TYPE_DELETE;
+  pModify->tableId = pRealTable->pMeta->uid;
+  pModify->tableType = pRealTable->pMeta->tableType;
+  snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId,
+           pRealTable->table.dbName, pRealTable->table.tableName);
+  pModify->deleteTimeRange = pDelete->timeRange;
+  pModify->pModifyRows = nodesCloneNode(pDelete->pCountFunc);
+  if (NULL == pModify->pModifyRows) {
+    nodesDestroyNode(pModify);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  *pLogicNode = (SLogicNode*)pModify;
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t createDeleteLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) {
   SLogicNode* pRoot = NULL;
-  int32_t     code = createDeleteRootLogicNode(pCxt, pDelete, createDeleteScanLogicNode, &pRoot);
+  int32_t     code = createDeleteScanLogicNode(pCxt, pDelete, &pRoot);
   if (TSDB_CODE_SUCCESS == code) {
     code = createDeleteRootLogicNode(pCxt, pDelete, createDeleteAggLogicNode, &pRoot);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = createDeleteRootLogicNode(pCxt, pDelete, createDeleteModifyTableLogicNode, &pRoot);
+    code = createDeleteRootLogicNode(pCxt, pDelete, createVnodeModifLogicNodeByDelete, &pRoot);
   }
 
   if (TSDB_CODE_SUCCESS == code) {

source/libs/planner/src/planOptimizer.c

@@ -99,7 +99,7 @@ static bool osdMayBeOptimized(SLogicNode* pNode) {
     return false;
   }
   // todo: release after function splitting
-  if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType &&
+  if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->tableType &&
       SCAN_TYPE_STREAM != ((SScanLogicNode*)pNode)->scanType) {
     return false;
   }
@@ -328,7 +328,7 @@ static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode**
 
 static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
   if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) ||
-      TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) {
+      TSDB_SYSTEM_TABLE == pScan->tableType) {
     return TSDB_CODE_SUCCESS;
   }
 
@@ -662,7 +662,7 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL
 
   switch (nodeType(pNode)) {
     case QUERY_NODE_LOGIC_PLAN_SCAN:
-      if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->pMeta->tableType) {
+      if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->tableType) {
         return nodesListMakeAppend(pScanNodes, pNode);
       }
       break;

source/libs/planner/src/planPhysiCreater.c

@@ -342,18 +342,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i
   return TSDB_CODE_SUCCESS;
 }
 
-static uint8_t getPrecision(SNodeList* pChildren) {
+static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, ENodeType type) {
-  if (1 == LIST_LENGTH(pChildren)) {
-    return (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc)->precision;
-  } else if (2 == LIST_LENGTH(pChildren)) {
-    uint8_t lp = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc)->precision;
-    uint8_t rp = (((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc)->precision;
-    return (lp > rp ? rp : lp);
-  }
-  return 0;
-}
-
-static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, uint8_t precision, SLogicNode* pLogicNode, ENodeType type) {
   SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type);
   if (NULL == pPhysiNode) {
     return NULL;
@@ -364,7 +353,7 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, uint8_t precision, SLo
     nodesDestroyNode(pPhysiNode);
     return NULL;
   }
-  pPhysiNode->pOutputDataBlockDesc->precision = precision;
+  pPhysiNode->pOutputDataBlockDesc->precision = pLogicNode->precision;
   return pPhysiNode;
 }
 
@@ -435,8 +424,8 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
+    pScanPhysiNode->uid = pScanLogicNode->tableId;
-    pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
+    pScanPhysiNode->tableType = pScanLogicNode->tableType;
     memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName));
     if (NULL != pScanLogicNode->pTagCond) {
       pSubplan->pTagCond = nodesCloneNode(pScanLogicNode->pTagCond);
@@ -462,8 +451,8 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd
 
 static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
                                       SPhysiNode** pPhyNode) {
-  STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(
+  STagScanPhysiNode* pTagScan =
-      pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
+      (STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
   if (NULL == pTagScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -476,8 +465,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla
 static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
                                         SPhysiNode** pPhyNode) {
   STableScanPhysiNode* pTableScan =
-      (STableScanPhysiNode*)makePhysiNode(pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode,
+      (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
-                                          QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
   if (NULL == pTableScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -515,9 +503,8 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
 
 static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
                                               SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) {
-  SSystemTableScanPhysiNode* pScan =
+  SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode,
-      (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, pScanLogicNode->pMeta->tableInfo.precision,
+                                                                               QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN);
-                                                (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN);
   if (NULL == pScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -567,8 +554,8 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
 
 static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode,
                                    SPhysiNode** pPhyNode) {
-  SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pJoinLogicNode,
+  SJoinPhysiNode* pJoin =
-                                                         QUERY_NODE_PHYSICAL_PLAN_JOIN);
+      (SJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_JOIN);
   if (NULL == pJoin) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -750,8 +737,7 @@ static int32_t rewritePrecalcExpr(SPhysiPlanContext* pCxt, SNode* pNode, SNodeLi
 
 static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode,
                                   SPhysiNode** pPhyNode) {
-  SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pAggLogicNode,
+  SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG);
-                                                      QUERY_NODE_PHYSICAL_PLAN_AGG);
   if (NULL == pAgg) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -806,8 +792,8 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
 
 static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                       SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) {
-  SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(
+  SProjectPhysiNode* pProject =
-      pCxt, getPrecision(pChildren), (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT);
+      (SProjectPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT);
   if (NULL == pProject) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -838,8 +824,8 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild
 
 static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode,
                                          SPhysiNode** pPhyNode) {
-  SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(
+  SExchangePhysiNode* pExchange =
-      pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE);
+      (SExchangePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE);
   if (NULL == pExchange) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -852,8 +838,8 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic
 
 static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode,
                                                    SPhysiNode** pPhyNode) {
-  SScanPhysiNode* pScan = (SScanPhysiNode*)makePhysiNode(
+  SScanPhysiNode* pScan =
-      pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
+      (SScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
   if (NULL == pScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -933,22 +919,28 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList*
   return code;
 }
 
-static ENodeType getIntervalOperatorType(bool streamQuery, EStreamIntervalAlgorithm stmAlgo) {
+static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) {
-  if (streamQuery) {
+  switch (intervalAlgo) {
-    return STREAM_INTERVAL_ALGO_FINAL == stmAlgo
+    case INTERVAL_ALGO_HASH:
-               ? QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL
+      return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
-               : (STREAM_INTERVAL_ALGO_SEMI == stmAlgo ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL
+    case INTERVAL_ALGO_SORT_MERGE:
-                                                       : QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL);
+      return QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL;
-  } else {
+    case INTERVAL_ALGO_STREAM_FINAL:
-    return QUERY_NODE_PHYSICAL_PLAN_INTERVAL;
+      return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
+    case INTERVAL_ALGO_STREAM_SEMI:
+      return QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL;
+    case INTERVAL_ALGO_STREAM_SINGLE:
+      return QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
+    default:
+      break;
   }
+  return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
 }
 
 static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                        SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
   SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(
-      pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode,
+      pCxt, (SLogicNode*)pWindowLogicNode, getIntervalOperatorType(pWindowLogicNode->intervalAlgo));
-      getIntervalOperatorType(pCxt->pPlanCxt->streamQuery, pWindowLogicNode->stmInterAlgo));
   if (NULL == pInterval) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -965,7 +957,7 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil
 static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                             SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
   SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode(
-      pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode,
+      pCxt, (SLogicNode*)pWindowLogicNode,
       (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW
                                    : QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW));
   if (NULL == pSession) {
@@ -979,10 +971,10 @@ static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList*
 
 static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                           SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
-  SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)makePhysiNode(
+  SStateWinodwPhysiNode* pState =
-      pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode,
+      (SStateWinodwPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode,
-      (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: 
+                                            (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW
-                                     QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW));
+                                                                         : QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW));
   if (NULL == pState) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -1032,8 +1024,8 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr
 
 static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode,
                                    SPhysiNode** pPhyNode) {
-  SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pSortLogicNode,
+  SSortPhysiNode* pSort =
-                                                         QUERY_NODE_PHYSICAL_PLAN_SORT);
+      (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT);
   if (NULL == pSort) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -1073,8 +1065,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
 
 static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                         SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) {
-  SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode(
+  SPartitionPhysiNode* pPart =
-      pCxt, getPrecision(pChildren), (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION);
+      (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION);
   if (NULL == pPart) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -1114,8 +1106,7 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi
 
 static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode,
                                    SPhysiNode** pPhyNode) {
-  SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pFillNode,
+  SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pFillNode, QUERY_NODE_PHYSICAL_PLAN_FILL);
-                                                         QUERY_NODE_PHYSICAL_PLAN_FILL);
   if (NULL == pFill) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -1168,8 +1159,8 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) {
 }
 
 static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pMergeLogicNode, SPhysiNode** pPhyNode) {
-  SMergePhysiNode* pMerge = (SMergePhysiNode*)makePhysiNode(
+  SMergePhysiNode* pMerge =
-      pCxt, pMergeLogicNode->node.precision, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE);
+      (SMergePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE);
   if (NULL == pMerge) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -1308,6 +1299,62 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl
   return pSubplan;
 }
 
+static int32_t buildInsertSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {
+  pSubplan->msgType = pModify->msgType;
+  pSubplan->execNode.epSet = pModify->pVgDataBlocks->vg.epSet;
+  SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
+  taosArrayPush(pCxt->pExecNodeList, &node);
+  return createDataInserter(pCxt, pModify->pVgDataBlocks, &pSubplan->pDataSink);
+}
+
+static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, const SPhysiNode* pRoot,
+                                 SDataSinkNode** pSink) {
+  SDataDeleterNode* pDeleter = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DELETE);
+  if (NULL == pDeleter) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pDeleter->tableId = pModify->tableId;
+  pDeleter->tableType = pModify->tableType;
+  strcpy(pDeleter->tableFName, pModify->tableFName);
+  pDeleter->deleteTimeRange = pModify->deleteTimeRange;
+
+  pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc);
+  if (NULL == pDeleter->sink.pInputDataBlockDesc) {
+    nodesDestroyNode(pDeleter);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  *pSink = (SDataSinkNode*)pDeleter;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildDeleteSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) {
+  int32_t code =
+      createPhysiNode(pCxt, (SLogicNode*)nodesListGetNode(pModify->node.pChildren, 0), pSubplan, &pSubplan->pNode);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createDataDeleter(pCxt, pModify, pSubplan->pNode, &pSubplan->pDataSink);
+  }
+  return code;
+}
+
+static int32_t buildVnodeModifySubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pSubplan) {
+  int32_t                code = TSDB_CODE_SUCCESS;
+  SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pLogicSubplan->pNode;
+  switch (pModify->modifyType) {
+    case MODIFY_TABLE_TYPE_INSERT:
+      code = buildInsertSubplan(pCxt, pModify, pSubplan);
+      break;
+    case MODIFY_TABLE_TYPE_DELETE:
+      code = buildDeleteSubplan(pCxt, pModify, pSubplan);
+      break;
+    default:
+      code = TSDB_CODE_FAILED;
+      break;
+  }
+  return code;
+}
+
 static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan** pPhysiSubplan) {
   SSubplan* pSubplan = makeSubplan(pCxt, pLogicSubplan);
   if (NULL == pSubplan) {
@@ -1317,12 +1364,7 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic
   int32_t code = TSDB_CODE_SUCCESS;
 
   if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) {
-    SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode;
+    code = buildVnodeModifySubplan(pCxt, pLogicSubplan, pSubplan);
-    pSubplan->msgType = pModif->msgType;
-    pSubplan->execNode.epSet = pModif->pVgDataBlocks->vg.epSet;
-    SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
-    taosArrayPush(pCxt->pExecNodeList, &node);
-    code = createDataInserter(pCxt, pModif->pVgDataBlocks, &pSubplan->pDataSink);
   } else {
     pSubplan->msgType = TDMT_VND_QUERY;
     code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode);

source/libs/planner/src/planScaleOut.c

@@ -38,26 +38,6 @@ static SLogicSubplan* singleCloneSubLogicPlan(SScaleOutContext* pCxt, SLogicSubp
   return pDst;
 }
 
-static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
-  SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode;
-  size_t                numOfVgroups = taosArrayGetSize(pNode->pDataBlocks);
-  for (int32_t i = 0; i < numOfVgroups; ++i) {
-    SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level);
-    if (NULL == pNewSubplan) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i);
-    if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
-  return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level));
-}
-
 static int32_t doSetScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) {
   if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
     SScanLogicNode* pScan = (SScanLogicNode*)pNode;
@@ -84,23 +64,52 @@ static int32_t setScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup) {
   return doSetScanVgroup(pNode, pVgroup, &found);
 }
 
-static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
+static int32_t scaleOutByVgroups(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
-  if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) {
+  int32_t code = TSDB_CODE_SUCCESS;
-    int32_t code = TSDB_CODE_SUCCESS;
+  for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) {
-    for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) {
+    SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level);
+    if (NULL == pNewSubplan) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = nodesListStrictAppend(pGroup, pNewSubplan);
+    }
+    if (TSDB_CODE_SUCCESS != code) {
+      break;
+    }
+  }
+  return code;
+}
+
+static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
+  SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pSubplan->pNode;
+  if (MODIFY_TABLE_TYPE_DELETE == pNode->modifyType) {
+    return scaleOutByVgroups(pCxt, pSubplan, level, pGroup);
+  } else {
+    size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks);
+    for (int32_t i = 0; i < numOfVgroups; ++i) {
       SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level);
       if (NULL == pNewSubplan) {
         return TSDB_CODE_OUT_OF_MEMORY;
       }
-      code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i);
+      ((SVnodeModifyLogicNode*)pNewSubplan->pNode)->pVgDataBlocks =
-      if (TSDB_CODE_SUCCESS == code) {
+          (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i);
-        code = nodesListStrictAppend(pGroup, pNewSubplan);
+      if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) {
-      }
+        return TSDB_CODE_OUT_OF_MEMORY;
-      if (TSDB_CODE_SUCCESS != code) {
-        break;
       }
     }
-    return code;
+    return TSDB_CODE_SUCCESS;
+  }
+}
+
+static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
+  return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level));
+}
+
+static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) {
+  if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) {
+    return scaleOutByVgroups(pCxt, pSubplan, level, pGroup);
   } else {
     return scaleOutForMerge(pCxt, pSubplan, level, pGroup);
   }

source/libs/planner/src/planSpliter.c

@@ -138,7 +138,7 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) {
 
 static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) {
   return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) ||
-         (streamQuery && TSDB_SUPER_TABLE == pScan->pMeta->tableType);
+         (streamQuery && TSDB_SUPER_TABLE == pScan->tableType);
 }
 
 static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) {
@@ -315,27 +315,6 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, S
   return nodesListMakeAppend(&pParent->pChildren, pMerge);
 }
 
-static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
-  SLogicNode* pPartWindow = NULL;
-  int32_t     code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow);
-  if (TSDB_CODE_SUCCESS == code) {
-    SNodeList* pMergeKeys = NULL;
-    code = nodesListMakeStrictAppend(&pMergeKeys, nodesCloneNode(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk));
-    if (TSDB_CODE_SUCCESS == code) {
-      code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pMergeKeys, pPartWindow);
-    }
-    if (TSDB_CODE_SUCCESS != code) {
-      nodesDestroyList(pMergeKeys);
-    }
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
-                                     splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT));
-  }
-  pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
-  return code;
-}
-
 static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) {
   SExchangeLogicNode* pExchange = NULL;
   int32_t             code = splCreateExchangeNode(pCxt, pPartChild, &pExchange);
@@ -345,12 +324,28 @@ static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent
   return code;
 }
 
+static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
+  SLogicNode* pPartWindow = NULL;
+  int32_t     code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow);
+  if (TSDB_CODE_SUCCESS == code) {
+    ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_HASH;
+    ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_SORT_MERGE;
+    code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
+                                     splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT));
+  }
+  pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
+  return code;
+}
+
 static int32_t stbSplSplitWindowNodeForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
   SLogicNode* pPartWindow = NULL;
   int32_t     code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow);
   if (TSDB_CODE_SUCCESS == code) {
-    ((SWindowLogicNode*)pPartWindow)->stmInterAlgo = STREAM_INTERVAL_ALGO_SEMI;
+    ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_STREAM_SEMI;
-    ((SWindowLogicNode*)pInfo->pSplitNode)->stmInterAlgo = STREAM_INTERVAL_ALGO_FINAL;
+    ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_STREAM_FINAL;
     code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow);
   }
   if (TSDB_CODE_SUCCESS == code) {
@@ -834,6 +829,16 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) {
 
 static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); }
 
+static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) {
+  if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
+    TSWAP(((SScanLogicNode*)pNode)->pVgroupList, pSubplan->pVgroupList);
+    return;
+  }
+
+  SNode* pChild;
+  FOREACH(pChild, pNode->pChildren) { setVgroupsInfo((SLogicNode*)pChild, pSubplan); }
+}
+
 int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) {
   SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
   if (NULL == pSubplan) {
@@ -845,17 +850,21 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan
     nodesDestroyNode(pSubplan);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) {
+
-    pSubplan->subplanType = SUBPLAN_TYPE_MODIFY;
-    TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)pSubplan->pNode)->pDataBlocks);
-  } else {
-    pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
-  }
   pSubplan->id.queryId = pCxt->queryId;
   pSubplan->id.groupId = 1;
   setLogicNodeParent(pSubplan->pNode);
 
-  int32_t code = applySplitRule(pCxt, pSubplan);
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicNode)) {
+    pSubplan->subplanType = SUBPLAN_TYPE_MODIFY;
+    TSWAP(((SVnodeModifyLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifyLogicNode*)pSubplan->pNode)->pDataBlocks);
+    setVgroupsInfo(pSubplan->pNode, pSubplan);
+  } else {
+    pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
+    code = applySplitRule(pCxt, pSubplan);
+  }
+
   if (TSDB_CODE_SUCCESS == code) {
     *pLogicSubplan = pSubplan;
   } else {

source/libs/planner/test/planOtherTest.cpp

@@ -60,3 +60,15 @@ TEST_F(PlanOtherTest, show) {
 
   run("SHOW DATABASES");
 }
+
+TEST_F(PlanOtherTest, delete) {
+  useDb("root", "test");
+
+  run("DELETE FROM t1");
+
+  run("DELETE FROM t1 WHERE ts > now - 2d and ts < now - 1d");
+
+  run("DELETE FROM st1");
+
+  run("DELETE FROM st1 WHERE ts > now - 2d and ts < now - 1d AND tag1 = 10");
+}

source/libs/scalar/inc/filterInt.h

@@ -350,7 +350,7 @@ struct SFilterInfo {
 
 extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
 extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
-
+extern OptrStr gOptrStr[];
 
 #ifdef __cplusplus
 }

source/libs/scalar/src/filter.c

@@ -29,8 +29,9 @@ OptrStr gOptrStr[] = {
   {OP_TYPE_SUB,                            "-"},
   {OP_TYPE_MULTI,                          "*"},
   {OP_TYPE_DIV,                            "/"},
-  {OP_TYPE_MOD,                            "%"},
+  {OP_TYPE_REM,                            "%"},
-
+  {OP_TYPE_MINUS,                          "minus"},
+  {OP_TYPE_ASSIGN,                         "assign"},
   // bit operator
   {OP_TYPE_BIT_AND,                        "&"},
   {OP_TYPE_BIT_OR,                         "|"},

source/libs/scalar/src/sclvector.c

@@ -29,6 +29,12 @@
 #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
 #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
 
+#define IS_NULL colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) \
+    || IS_JSON_NULL(pLeft->columnData->info.type, colDataGetVarData(pLeft->columnData, i)) \
+    || IS_JSON_NULL(pRight->columnData->info.type, colDataGetVarData(pRight->columnData, i))
+
+#define IS_HELPER_NULL(col,i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i))
+
 void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
   switch (outType) {
     case TSDB_DATA_TYPE_BOOL: {
@@ -387,7 +393,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
 
   pOut->numOfRows = pIn->numOfRows;
   for (int32_t i = 0; i < pIn->numOfRows; ++i) {
-    if (colDataIsNull_s(pIn->columnData, i)) {
+    if (IS_HELPER_NULL(pIn->columnData, i)) {
       colDataAppendNULL(pOut->columnData, i);
       continue;
     }
@@ -396,8 +402,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     int32_t convertType = inType;
     if(inType == TSDB_DATA_TYPE_JSON){
       if(*data == TSDB_DATA_TYPE_NULL) {
-        colDataAppendNULL(pOut->columnData, i);
+        ASSERT(0);
-        continue;
       }
       else if(*data == TSDB_DATA_TYPE_NCHAR) {
         data += CHAR_BYTES;
@@ -447,13 +452,13 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
   return out;
 }
 
-void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
+bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
   if(optr == OP_TYPE_JSON_CONTAINS) {
-    return;
+    return true;
   }
 
   if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
-    return;
+    return true;
   }
 
   if(typeLeft == TSDB_DATA_TYPE_JSON){
@@ -464,15 +469,22 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
     typeRight = **pRightData;
     (*pRightData) ++;
   }
+
+  if(optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH){
+    if(typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY){
+      return false;
+    }
+  }
+
   if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
     *isNull = true;
-    return;
+    return true;
   }
   int8_t type = vectorGetConvertType(typeLeft, typeRight);
 
   if(type == 0) {
     *fp = filterGetCompFunc(typeLeft, optr);
-    return;
+    return true;
   }
 
   *fp = filterGetCompFunc(type, optr);
@@ -492,6 +504,7 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
     convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
     *pRightData = pRightOut;
   }
+  return true;
 }
 
 int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int16_t inType, int16_t outType) {
@@ -867,11 +880,11 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
 
   double *output = (double *)pOutputCol->pData;
 
-  if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+  if (IS_HELPER_NULL(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i)) {
+      if (IS_HELPER_NULL(pLeftCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -887,11 +900,11 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData
 
   int64_t *output = (int64_t *)pOutputCol->pData;
 
-  if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+  if (IS_HELPER_NULL(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i)) {
+      if (IS_HELPER_NULL(pLeftCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -982,7 +995,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        if (IS_NULL) {
           colDataAppendNULL(pOutputCol, i);
           continue;  // TODO set null or ignore
         }
@@ -1000,7 +1013,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        if (IS_NULL){
           colDataAppendNULL(pOutputCol, i);
           continue;  // TODO set null or ignore
         }
@@ -1024,11 +1037,11 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
 
   double *output = (double *)pOutputCol->pData;
 
-  if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+  if (IS_HELPER_NULL(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i)) {
+      if (IS_HELPER_NULL(pLeftCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1044,11 +1057,11 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData
 
   int64_t *output = (int64_t *)pOutputCol->pData;
 
-  if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+  if (IS_HELPER_NULL(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i)) {
+      if (IS_HELPER_NULL(pLeftCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1077,7 +1090,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        if (IS_NULL) {
           colDataAppendNULL(pOutputCol, i);
           continue;  // TODO set null or ignore
         }
@@ -1095,7 +1108,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        if (IS_NULL) {
           colDataAppendNULL(pOutputCol, i);
           continue;  // TODO set null or ignore
         }
@@ -1119,11 +1132,11 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
 
   double *output = (double *)pOutputCol->pData;
 
-  if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+  if (IS_HELPER_NULL(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i)) {
+      if (IS_HELPER_NULL(pLeftCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1149,7 +1162,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+      if (IS_NULL) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1182,7 +1195,7 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check
+      if (IS_NULL || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check
         colDataAppendNULL(pOutputCol, i);
         continue;
       }
@@ -1190,11 +1203,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
                 / getVectorDoubleValueFnRight(RIGHT_COL, i);
     }
   } else if (pLeft->numOfRows == 1) {
-    if (colDataIsNull_s(pLeftCol, 0)) {  // Set pLeft->numOfRows NULL value
+    if (IS_HELPER_NULL(pLeftCol, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
       for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
-        if (colDataIsNull_s(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check
+        if (IS_HELPER_NULL(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
@@ -1203,11 +1216,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
       }
     }
   } else if (pRight->numOfRows == 1) {
-    if (colDataIsNull_s(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) {  // Set pLeft->numOfRows NULL value (divde by 0 check)
+    if (IS_HELPER_NULL(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) {  // Set pLeft->numOfRows NULL value (divde by 0 check)
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
       for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
-        if (colDataIsNull_s(pLeftCol, i)) {
+        if (IS_HELPER_NULL(pLeftCol, i)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
@@ -1236,18 +1249,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
   _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type);
 
   double *output = (double *)pOutputCol->pData;
-  double zero = 0.0;
 
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) {
+      if (IS_NULL) {
         colDataAppendNULL(pOutputCol, i);
         continue;
       }
 
       double lx = getVectorDoubleValueFnLeft(LEFT_COL, i);
       double rx = getVectorDoubleValueFnRight(RIGHT_COL, i);
-      if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
+      if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
         colDataAppendNULL(pOutputCol, i);
         continue;
       }
@@ -1256,11 +1268,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
     }
   } else if (pLeft->numOfRows == 1) {
     double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0);
-    if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) {  // Set pLeft->numOfRows NULL value
+    if (IS_HELPER_NULL(pLeftCol, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
       for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
-        if (colDataIsNull_s(pRightCol, i)) {
+        if (IS_HELPER_NULL(pRightCol, i)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
@@ -1276,11 +1288,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
     }
   } else if (pRight->numOfRows == 1) {
     double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0);
-    if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) {  // Set pLeft->numOfRows NULL value
+    if (IS_HELPER_NULL(pRightCol, 0) || FLT_EQUAL(rx, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
       for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
-        if (colDataIsNull_s(pLeftCol, i)) {
+        if (IS_HELPER_NULL(pLeftCol, i)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
@@ -1315,7 +1327,7 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
 
   double *output = (double *)pOutputCol->pData;
   for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
-    if (colDataIsNull_s(pLeft->columnData, i)) {
+    if (IS_HELPER_NULL(pLeftCol, i)) {
       colDataAppendNULL(pOutputCol, i);
       continue;
     }
@@ -1331,7 +1343,7 @@ void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
 
   pOut->numOfRows = pLeft->numOfRows;
 
-  if (colDataIsNull_s(pRight->columnData, 0)) {
+  if (IS_HELPER_NULL(pRight->columnData, 0)) {
     for (int32_t i = 0; i < pOut->numOfRows; ++i) {
       colDataAppend(pOutputCol, i, NULL, true);
     }
@@ -1397,13 +1409,13 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh
   _getBigintValue_fn_t getVectorBigintValueFnLeft  = getVectorBigintValueFn(pLeftCol->info.type);
   _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type);
 
-  double *output = (double *)pOutputCol->pData;
+  int64_t *output = (int64_t *)pOutputCol->pData;
 
-  if (colDataIsNull_s(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
+  if (IS_HELPER_NULL(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i)) {
+      if (IS_HELPER_NULL(pLeftCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1429,7 +1441,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   int64_t *output = (int64_t *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+      if (IS_NULL) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1451,12 +1463,12 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight
 
   int64_t *output = (int64_t *)pOutputCol->pData;
 
-  if (colDataIsNull_s(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
+  if (IS_HELPER_NULL(pRightCol, 0)) {  // Set pLeft->numOfRows NULL value
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0);
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      if (colDataIsNull_s(pLeftCol, i)) {
+      if (IS_HELPER_NULL(pLeftCol, i)) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1482,7 +1494,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   int64_t *output = (int64_t *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+      if (IS_NULL) {
         colDataAppendNULL(pOutputCol, i);
         continue;  // TODO set null or ignore
       }
@@ -1507,7 +1519,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
 
   if (pRight->pHashFilter != NULL) {
     for (; i >= 0 && i < pLeft->numOfRows; i += step) {
-      if (colDataIsNull_s(pLeft->columnData, i)) {
+      if (IS_HELPER_NULL(pLeft->columnData, i)) {
         bool  res = false;
         colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
         continue;
@@ -1522,7 +1534,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
 
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step) {
-      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+      if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) {
         bool  res = false;
         colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
         continue;  // TODO set null or ignore
@@ -1534,18 +1546,21 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       int64_t leftOut = 0;
       int64_t rightOut = 0;
       bool isJsonnull = false;
-      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
+      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
       if(isJsonnull){
-        colDataAppendNULL(pOut->columnData, i);
+        ASSERT(0);
-        continue;  // TODO set null or ignore
+      }
+      if(!result){
+        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
+      }else{
+        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
+        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
       }
-      bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-      colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
   } else if (pRight->numOfRows == 1) {
     ASSERT(pLeft->pHashFilter == NULL);
     for (; i >= 0 && i < pLeft->numOfRows; i += step) {
-      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, 0)) {
+      if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) {
         bool  res = false;
         colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
         continue;
@@ -1556,17 +1571,20 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       int64_t leftOut = 0;
       int64_t rightOut = 0;
       bool isJsonnull = false;
-      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
+      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
       if(isJsonnull){
-        colDataAppendNULL(pOut->columnData, i);
+        ASSERT(0);
-        continue;  // TODO set null or ignore
+      }
+      if(!result){
+        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
+      }else{
+        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
+        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
       }
-      bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-      colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
   } else if (pLeft->numOfRows == 1) {
     for (; i >= 0 && i < pRight->numOfRows; i += step) {
-      if (colDataIsNull_s(pRight->columnData, i) || colDataIsNull_s(pLeft->columnData, 0)) {
+      if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) {
         bool  res = false;
         colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
         continue;
@@ -1577,13 +1595,16 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
       int64_t leftOut = 0;
       int64_t rightOut = 0;
       bool isJsonnull = false;
-      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
+      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
       if(isJsonnull){
-        colDataAppendNULL(pOut->columnData, i);
+        ASSERT(0);
-        continue;  // TODO set null or ignore
+      }
+      if(!result){
+        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
+      }else{
+        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
+        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
       }
-      bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-      colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
   }
 }
@@ -1668,7 +1689,7 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
 
 void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
-    int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
+    int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0;
     colDataAppendInt8(pOut->columnData, i, &v);
   }
   pOut->numOfRows = pLeft->numOfRows;
@@ -1676,7 +1697,7 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
 
 void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
-    int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1;
+    int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1;
     colDataAppendInt8(pOut->columnData, i, &v);
   }
   pOut->numOfRows = pLeft->numOfRows;
@@ -1696,7 +1717,7 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
       return vectorMathMultiply;
     case OP_TYPE_DIV:
       return vectorMathDivide;
-    case OP_TYPE_MOD:
+    case OP_TYPE_REM:
       return vectorMathRemainder;
     case OP_TYPE_MINUS:
       return vectorMathMinus;

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -42,6 +42,7 @@
 #include "nodes.h"
 #include "tlog.h"
 #include "parUtil.h"
+#include "filterInt.h"
 
 #define _DEBUG_PRINT_ 0
 
@@ -212,6 +213,24 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
   *pNode = (SNode *)rnode;
 }
 
+void scltMakeOpNode2(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight, bool isReverse) {
+  SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
+  SOperatorNode *onode = (SOperatorNode *)node;
+  onode->node.resType.type = resType;
+  onode->node.resType.bytes = tDataTypes[resType].bytes;
+
+  onode->opType = opType;
+  if(isReverse){
+    onode->pLeft = pRight;
+    onode->pRight = pLeft;
+  }else{
+    onode->pLeft = pLeft;
+    onode->pRight = pRight;
+  }
+
+  *pNode = (SNode *)onode;
+}
+
 void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) {
   SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
   SOperatorNode *onode = (SOperatorNode *)node;
@@ -1039,10 +1058,10 @@ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
   scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
 }
 
-void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){
+void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData, bool isReverse){
   int32_t resType = TSDB_DATA_TYPE_NULL;
   if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI ||
-      opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
+      opType == OP_TYPE_DIV || opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){
     resType = TSDB_DATA_TYPE_DOUBLE;
   }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
     resType = TSDB_DATA_TYPE_BIGINT;
@@ -1057,7 +1076,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32
 
   SNode *right = NULL;
   scltMakeValueNode(&right, rightType, rightData);
-  scltMakeOpNode(opNode, opType, resType, *opNode, right);
+  scltMakeOpNode2(opNode, opType, resType, *opNode, right, isReverse);
 
   SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes);
   int16_t dataBlockId = 0, slotId = 0;
@@ -1065,7 +1084,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32
   scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
 }
 
-void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){
+void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType, bool isReverse){
   SArray *blockList = taosArrayInit(2, POINTER_BYTES);
   SSDataBlock *src = NULL;
   SNode *opNode = NULL;
@@ -1073,7 +1092,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
   makeJsonArrow(&src, &opNode, json, (char*)key);
   taosArrayPush(blockList, &src);
 
-  makeOperator(&opNode, blockList, opType, rightType, rightData);
+  makeOperator(&opNode, blockList, opType, rightType, rightData, isReverse);
 
   int32_t code = scalarCalculate(opNode, blockList, NULL);
   ASSERT_EQ(code, 0);
@@ -1087,17 +1106,17 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
     printf("result:NULL\n");
 
   }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
-             opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){
+             opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){
-    printf("1result:%f,except:%f\n", *((double *)colDataGetData(column, 0)), exceptValue);
+    printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue);
     ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001);
   }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
-    printf("2result:%ld,except:%f\n", *((int64_t *)colDataGetData(column, 0)), exceptValue);
+    printf("op:%s,2result:%ld,except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue);
     ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
   }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN ||
              opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
              opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
              opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){
-    printf("3result:%d,except:%f\n", *((bool *)colDataGetData(column, 0)), exceptValue);
+    printf("op:%s,3result:%d,except:%f\n", gOptrStr[opType].str, *((bool *)colDataGetData(column, 0)), exceptValue);
     ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
   }
 
@@ -1107,7 +1126,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
 
 TEST(columnTest, json_column_arith_op) {
   scltInitLogFile();
-  char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
+  char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"8hel\"}";
 
   char rightv[256] = {0};
   memcpy(rightv, rightvTmp, strlen(rightvTmp));
@@ -1117,54 +1136,126 @@ TEST(columnTest, json_column_arith_op) {
 
   const int32_t len = 8;
   EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
-                         OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
+                           OP_TYPE_REM, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR};
   int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9};
 
-  printf("--------------------json int---------------------\n");
+  printf("--------------------json int-4 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n");
   char *key = "k1";
-  double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
+  double eRes00[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9};
+  double eRes01[len] = {5.0, 4, 8.0, 0.5, 3, 0, 4&-4, 4|9};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes00[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes01[i], op[i], true);
   }
 
-  printf("--------------------json string---------------------\n");
+  printf("--------------------json string- 0 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n");
 
   key = "k2";
-  double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
+  double eRes10[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
+  double eRes11[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes11[i], op[i], true);
   }
 
-  printf("---------------------json null--------------------\n");
+  printf("---------------------json null- null op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n");
 
   key = "k3";
-  double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  double eRes20[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  double eRes21[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes20[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes21[i], op[i], true);
   }
 
-  printf("---------------------json bool--------------------\n");
+  printf("---------------------json bool- true op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n");
 
   key = "k4";
-  double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
+  double eRes30[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9};
+  double eRes31[len] = {2.0, 7, 2, 2, 0, 0, 1&-4, 1|9};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes30[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes31[i], op[i], true);
   }
 
-  printf("----------------------json double-------------------\n");
+  printf("----------------------json double-- 5.44 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
 
   key = "k5";
-  double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
+  double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9};
+  double eRes41[len] = {6.44, 2.56, 10.88, 0.3676470588235294, 3, 0, 5&-4, 5|9};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes40[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes41[i], op[i], true);
   }
 
-  printf("---------------------json not exist--------------------\n");
+  printf("----------------------json int-- -10 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
+
+  key = "k6";
+  double eRes50[len] = {-9, -18, -20, -5, -10%3, 10, -10&-4, -10|9};
+  double eRes51[len] = {-9, 18, -20, -0.2, 3%-10, 0, -10&-4, -10|9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes50[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes51[i], op[i], true);
+  }
+
+  printf("----------------------json double-- -9.8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
+
+  key = "k7";
+  double eRes60[len] = {-8.8, -17.8, -19.6, -4.9, -0.8, 9.8, -9&-4, -9|9};
+  double eRes61[len] = {-8.8, 17.8, -19.6, -0.2040816326530612, 3, 0, -9&-4, -9|9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes60[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes61[i], op[i], true);
+  }
+
+  printf("----------------------json bool-- 0 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
+
+  key = "k8";
+  double eRes70[len] = {1.0, -8, 0, 0, 0, 0, 0, 9};
+  double eRes71[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes70[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes71[i], op[i], true);
+  }
+
+  printf("----------------------json string-- 8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
+
+  key = "k9";
+  double eRes80[len] = {9, 0, 16, 4, 8%3, -8, 8&-4, 8|9};
+  double eRes81[len] = {9, 0, 16, 0.25, 3%8, 0, 8&-4, 8|9};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes80[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes81[i], op[i], true);
+  }
+
+  printf("---------------------json not exist-- NULL op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n");
 
   key = "k10";
-  double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  double eRes90[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX};
+  double eRes91[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes90[i], op[i], false);
+  }
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes91[i], op[i], true);
   }
 
   taosArrayDestroy(tags);
@@ -1183,7 +1274,7 @@ void *prepareNchar(char* rightData){
 
 TEST(columnTest, json_column_logic_op) {
   scltInitLogFile();
-  char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
+  char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"6.6hello\"}";
 
   char rightv[256] = {0};
   memcpy(rightv, rightvTmp, strlen(rightvTmp));
@@ -1191,6 +1282,7 @@ TEST(columnTest, json_column_logic_op) {
   STag* row = NULL;
   parseJsontoTagData(rightv, tags, &row, NULL);
 
+  const int32_t len0 = 6;
   const int32_t len = 9;
   const int32_t len1 = 4;
   EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL,
@@ -1199,93 +1291,183 @@ TEST(columnTest, json_column_logic_op) {
   int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0};
   char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"};
 
-  printf("--------------------json int---------------------\n");
+  printf("--------------------json int---4 {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n");
   char *key = "k1";
   bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i], false);
+  }
+  bool eRes_0[len0] = {false, true, true, true, false, true};
+  for(int i = 0; i < len0; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_0[i], op[i], true);
   }
   for(int i = len; i < len + len1; i++){
     void* rightData = prepareNchar(inputNchar[i-len]);
-    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i], false);
     taosMemoryFree(rightData);
   }
 
-  printf("--------------------json string---------------------\n");
+  printf("--------------------json string--0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
 
   key = "k2";
   bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i], false);
   }
+  bool eRes_1[len0] = {true, true, false, false, false, false};
+  for(int i = 0; i < len0; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_1[i], op[i], true);
+  }
+
   for(int i = len; i < len + len1; i++){
     void* rightData = prepareNchar(inputNchar[i-len]);
-    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i], false);
     taosMemoryFree(rightData);
   }
 
-  printf("--------------------json null---------------------\n");
+  printf("--------------------json null---null {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n");
 
   key = "k3";   // (null is true) return NULL, so use DBL_MAX represent NULL
   double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i], false);
   }
+  bool eRes_2[len0] = {false, false, false, false, false, false};
+  for(int i = 0; i < len0; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_2[i], op[i], true);
+  }
+
   for(int i = len; i < len + len1; i++){
     void* rightData = prepareNchar(inputNchar[i-len]);
-    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i], false);
     taosMemoryFree(rightData);
   }
 
-  printf("--------------------json bool---------------------\n");
+  printf("--------------------json bool--1 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
 
   key = "k4";
   bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i], false);
   }
+  bool eRes_3[len0] = {false, true, false, false, false, true};
+  for(int i = 0; i < len0; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_3[i], op[i], true);
+  }
+
   for(int i = len; i < len + len1; i++){
     void* rightData = prepareNchar(inputNchar[i-len]);
-    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i], false);
     taosMemoryFree(rightData);
   }
 
-  printf("--------------------json double---------------------\n");
+  printf("--------------------json double--5.44 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
 
   key = "k5";
   bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i], false);
   }
+  bool eRes_4[len0] = {false, true, true, true, false, true};
+  for(int i = 0; i < len0; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_4[i], op[i], true);
+  }
+
   for(int i = len; i < len + len1; i++){
     void* rightData = prepareNchar(inputNchar[i-len]);
-    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i], false);
     taosMemoryFree(rightData);
   }
 
-  printf("--------------------json double---------------------\n");
+  printf("--------------------json int--  -10 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
 
   key = "k6";
-  bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true};
+  bool eRes5[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i], false);
   }
+  bool eRes_5[len0] = {true, true, false, false, false, true};
+  for(int i = 0; i < len0; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_5[i], op[i], true);
+  }
+
   for(int i = len; i < len + len1; i++){
     void* rightData = prepareNchar(inputNchar[i-len]);
-    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false);
     taosMemoryFree(rightData);
   }
 
-  printf("---------------------json not exist--------------------\n");
+  printf("--------------------json double--  -9.8 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
 
-  key = "k10";    // (NULL is true) return NULL, so use DBL_MAX represent NULL
+  key = "k7";
-  double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
+  bool eRes6[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false};
   for(int i = 0; i < len; i++){
-    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes6[i], op[i], false);
   }
+  bool eRes_6[len0] = {true, true, false, false, false, true};
+  for(int i = 0; i < len0; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_6[i], op[i], true);
+  }
+
   for(int i = len; i < len + len1; i++){
     void* rightData = prepareNchar(inputNchar[i-len]);
-    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes6[i], op[i], false);
+    taosMemoryFree(rightData);
+  }
+
+
+  printf("--------------------json bool--  0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
+
+  key = "k8";
+  bool eRes7[len+len1] = {false, false, true, true, false, false, false, true, false, false, false, false, false};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes7[i], op[i], false);
+  }
+  bool eRes_7[len0] = {true, true, false, false, false, false};
+  for(int i = 0; i < len0; i++) {
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_7[i], op[i], true);
+  }
+
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes7[i], op[i], false);
+    taosMemoryFree(rightData);
+  }
+
+
+  printf("--------------------json string--  6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
+
+  key = "k9";
+  bool eRes8[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false);
+  }
+  bool eRes_8[len0] = {false, true, true, true, false, true};
+  for(int i = 0; i < len0; i++) {
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_8[i], op[i], true);
+  }
+
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes8[i], op[i], false);
+    taosMemoryFree(rightData);
+  }
+
+  printf("---------------------json not exist-- NULL {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n");
+
+  key = "k10";    // (NULL is true) return NULL, so use DBL_MAX represent NULL
+  double eRes9[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
+  for(int i = 0; i < len; i++){
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes9[i], op[i], false);
+  }
+  bool eRes_9[len0] = {false, false, false, false, false, false};
+  for(int i = 0; i < len0; i++) {
+    makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_9[i], op[i], true);
+  }
+
+  for(int i = len; i < len + len1; i++){
+    void* rightData = prepareNchar(inputNchar[i-len]);
+    makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes9[i], op[i], false);
     taosMemoryFree(rightData);
   }
 

source/os/src/osFile.c

@@ -399,6 +399,9 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
 }
 
 int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
+  if (pFile == NULL) {
+    return 0;
+  }
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&(pFile->rwlock));
 #endif
@@ -430,6 +433,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
 }
 
 int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
+  if (pFile == NULL) {
+    return 0;
+  }
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif

source/os/src/osSysinfo.c

@@ -276,7 +276,8 @@ int32_t taosGetEmail(char *email, int32_t maxLen) {
 
 int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) {
 #ifdef WINDOWS
-  assert(0);
+  snprintf(releaseName, maxLen, "Windows");
+  return 0;
 #elif defined(_TD_DARWIN_64)
   char   *line = NULL;
   size_t  size = 0;
@@ -332,7 +333,15 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) {
 
 int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
 #ifdef WINDOWS
-  assert(0);
+  char value[100];
+  DWORD bufferSize = sizeof(value);
+  RegGetValue(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize);
+  tstrncpy(cpuModel, value, maxLen);
+  SYSTEM_INFO si;
+  memset(&si,0,sizeof(SYSTEM_INFO));
+  GetSystemInfo(&si);
+  *numOfCores = si.dwNumberOfProcessors;
+  return 0;
 #elif defined(_TD_DARWIN_64)
   char   *line = NULL;
   size_t  size = 0;

source/os/src/osTimezone.c

@@ -767,32 +767,36 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8
         keyValue[4] = (keyValue[4] == '+' ? '-' : '+');
         keyValue[10] = 0;
         sprintf(winStr, "TZ=%s:00", &(keyValue[1]));
+        *tsTimezone = taosStr2Int32(&keyValue[4], NULL, 10);
       }
       break;
     }
   }
-  char *p = strchr(inTimezoneStr, '+');
+  if (winStr[0] == 0) {
-  if (p == NULL) p = strchr(inTimezoneStr, '-');
+    char *p = strchr(inTimezoneStr, '+');
-  if (p == NULL) {
+    if (p == NULL) p = strchr(inTimezoneStr, '-');
-    sprintf(winStr, "TZ=UTC+00:00:00");
+    if (p != NULL) {
-  } else {
+      char *pp = strchr(inTimezoneStr, '(');
-    sprintf(winStr, "TZ=UTC%c%c%c:%c%c:00", (p[0] == '+' ? '-' : '+'), p[1], p[2], p[3], p[4]);
+      char *ppp = strchr(inTimezoneStr, ',');
+      int indexStr;
+      if (pp == NULL || ppp == NULL) {
+        indexStr = sprintf(winStr, "TZ=UTC");
+      } else {
+        memcpy(winStr, "TZ=", 3);
+        pp++;
+        memcpy(&winStr[3], pp, ppp - pp);
+        indexStr = ppp - pp + 3;
+      }
+      sprintf(&winStr[indexStr], "%c%c%c:%c%c:00", (p[0] == '+'? '-' : '+'), p[1], p[2], p[3], p[4]);
+      *tsTimezone = taosStr2Int32(p, NULL, 10);
+    } else {
+      *tsTimezone = 0;
+    }
   }
   _putenv(winStr);
   _tzset();
-#ifdef _MSC_VER
+  strcpy(outTimezoneStr, inTimezoneStr);
-#if _MSC_VER >= 1900
+  *outDaylight = 0;
-  int64_t timezone = _timezone;
-  int32_t daylight = _daylight;
-  char  **tzname = _tzname;
-#endif
-#endif
-
-  int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
-  *tsTimezone = tz;
-  tz += daylight;
-  sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
-  *outDaylight = daylight;
 
 #elif defined(_TD_DARWIN_64)
 
@@ -822,34 +826,27 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8
 void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) {
 #ifdef WINDOWS 
   char value[100];
+  char keyPath[100];
   DWORD bufferSize = sizeof(value);
-  char *buf = getenv("TZ");
+  RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize);
-  if (buf == NULL || strlen(buf) == 0) {
+  strcpy(outTimezoneStr, "not configured");
-    RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize);
+  *tsTimezone = 0;
-    strcpy(outTimezoneStr, "not configured");
+  if (bufferSize > 0) {
-    if (bufferSize > 0) {
+    for (size_t i = 0; i < 139; i++) {
-      for (size_t i = 0; i < 139; i++) {
+      if (strcmp(win_tz[i][0],value) == 0) {
-        if (strcmp(win_tz[i][0],value) == 0) {
+        strcpy(outTimezoneStr, win_tz[i][1]);
-          strcpy(outTimezoneStr, win_tz[i][1]);
+        bufferSize = sizeof(value);
-          break;
+        sprintf(keyPath, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s",value);
+        RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize);
+        if (bufferSize > 0) {
+          // value[4] = (value[4] == '+' ? '-' : '+');
+          sprintf(outTimezoneStr, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], value[9]);
+          *tsTimezone = taosStr2Int32(&value[4], NULL, 10);
         }
+        break;
       }
     }
-  } else {
-    strcpy(outTimezoneStr, buf);
   }
-#ifdef _MSC_VER
-#if _MSC_VER >= 1900
-  // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
-  int64_t timezone = _timezone;
-  int32_t daylight = _daylight;
-  char  **tzname = _tzname;
-#endif
-#endif
-  int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
-  *tsTimezone = tz;
-  tz += daylight;
-  sprintf(outTimezoneStr, "%s (%s, %s%02d00)", outTimezoneStr, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
 #elif defined(_TD_DARWIN_64)
   char  buf[4096] = {0};
   char *tz = NULL;

tests/pytest/util/dnodes.py

@@ -145,6 +145,12 @@ class TDDnode:
     def init(self, path, remoteIP = ""):
         self.path = path
         self.remoteIP = remoteIP
+        if (not self.remoteIP == ""):
+            try:
+                self.config = eval(self.remoteIP)
+                self.remote_conn = Connection(host=self.config["host"], port=self.config["port"], user=self.config["user"], connect_kwargs={'password':self.config["password"]})
+            except Exception as r:
+                print(r)
 
     def setTestCluster(self, value):
         self.testCluster = value
@@ -169,13 +175,6 @@ class TDDnode:
         self.cfgDict.update({option: value})
 
     def remoteExec(self, updateCfgDict, execCmd):
-        try:
-            config = eval(self.remoteIP)
-            remote_conn = Connection(host=config["host"], port=config["port"], user=config["user"], connect_kwargs={'password':config["password"]})
-            remote_top_dir = config["path"]
-        except Exception as r:
-            remote_conn = Connection(host=self.remoteIP, port=22, user='root', connect_kwargs={'password':'123456'})
-            remote_top_dir = '~/test'
         valgrindStr = ''
         if (self.valgrind==1):
             valgrindStr = '-g'
@@ -188,8 +187,8 @@ class TDDnode:
             del remoteCfgDict["cfgDir"]
         remoteCfgDictStr = base64.b64encode(json.dumps(remoteCfgDict).encode()).decode()
         execCmdStr = base64.b64encode(execCmd.encode()).decode()
-        with remote_conn.cd((remote_top_dir+sys.path[0].replace(self.path, '')).replace('\\','/')):
+        with self.remote_conn.cd((self.config["path"]+sys.path[0].replace(self.path, '')).replace('\\','/')):
-            remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr))
+            self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr))
 
     def deploy(self, *updatecfgDict):
         self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index)
@@ -319,7 +318,7 @@ class TDDnode:
             print(cmd)
 
         if (not self.remoteIP == ""):
-            self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.start(%d)"%(self.index, self.index))
+            self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.start(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index))
             self.running = 1
         else:
             if os.system(cmd) != 0:

tests/system-test/0-others/udfTest.py

@@ -3,6 +3,7 @@ import taos
 import sys
 import time
 import os
+import platform
 
 from util.log import *
 from util.sql import *
@@ -25,7 +26,7 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
 
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if ("taosd" in files or "taosd.exe" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
@@ -41,11 +42,19 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
         print(projPath)
 
-        libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+        if platform.system().lower() == 'windows':
-        libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+            self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
-        os.system("mkdir /tmp/udf/")
+            self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
-        os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,""))
+            if (not tdDnodes.dnodes[0].remoteIP == ""):
-        os.system("cp  %s /tmp/udf/ "%libudf2.replace("\n" ,""))
+                tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\")
+                tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\")
+                self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so')
+                self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so')
+        else:
+            self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+            self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+        self.libudf1 = self.libudf1.replace('\r','').replace('\n','')
+        self.libudf2 = self.libudf2.replace('\r','').replace('\n','')
 
 
     def prepare_data(self):
@@ -136,11 +145,11 @@ class TDTestCase:
 
         for i in range(5):
             # create  scalar functions
-            tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+            tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
 
             # create aggregate functions
 
-            tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+            tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
             
             functions = tdSql.getResult("show functions")
             function_nums = len(functions)
@@ -161,11 +170,11 @@ class TDTestCase:
                 tdLog.info("drop two udf functions success ")
 
         # create  scalar functions
-        tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+        tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
 
         # create aggregate functions
 
-        tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+        tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
         
         functions = tdSql.getResult("show functions")
         function_nums = len(functions)
@@ -533,8 +542,8 @@ class TDTestCase:
         tdSql.query("drop function udf2 ")
 
         # create function without buffer
-        tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int")
+        tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1)
-        tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double")
+        tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2)
         udf1_sqls ,udf2_sqls = self.try_query_sql()
 
         for scalar_sql in udf1_sqls:
@@ -549,8 +558,8 @@ class TDTestCase:
         tdSql.query("drop function udf2 ")
 
         # create function without buffer
-        tdSql.execute("create  aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute("create  aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1)
-        tdSql.execute("create  function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.execute("create  function udf2 as '%s' outputtype double bufSize 8"%self.libudf2)
         udf1_sqls ,udf2_sqls = self.try_query_sql()
 
         for scalar_sql in udf1_sqls:
@@ -558,8 +567,8 @@ class TDTestCase:
         for aggregate_sql in udf2_sqls:
             tdSql.error(aggregate_sql)
 
-        tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1)
-        tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1)
         tdSql.error(" select db(c1) from stb1 ")
         tdSql.error(" select db(c1,c6), db(c6) from stb1 ")
         tdSql.error(" select db(num1,num2), db(num1) from tb ")
@@ -607,17 +616,17 @@ class TDTestCase:
         tdLog.info(" create function name is not build_in functions ")
         tdSql.execute(" drop function udf1 ")
         tdSql.execute(" drop function udf2 ")
-        tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
+        tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
-        tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
+        tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
-        tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2)
 
     def restart_taosd_query_udf(self):
 

tests/system-test/0-others/udf_create.py

@@ -25,7 +25,7 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
 
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if ("taosd" in files or "taosd.exe" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
@@ -41,11 +41,19 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
         print(projPath)
 
-        libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+        if platform.system().lower() == 'windows':
-        libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+            self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
-        os.system("mkdir /tmp/udf/")
+            self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
-        os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,""))
+            if (not tdDnodes.dnodes[0].remoteIP == ""):
-        os.system("cp  %s /tmp/udf/ "%libudf2.replace("\n" ,""))
+                tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\")
+                tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\")
+                self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so')
+                self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so')
+        else:
+            self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+            self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+        self.libudf1 = self.libudf1.replace('\r','').replace('\n','')
+        self.libudf2 = self.libudf2.replace('\r','').replace('\n','')
 
 
     def prepare_data(self):
@@ -136,11 +144,11 @@ class TDTestCase:
 
         for i in range(5):
             # create  scalar functions
-            tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+            tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
 
             # create aggregate functions
 
-            tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+            tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
             
             functions = tdSql.getResult("show functions")
             function_nums = len(functions)
@@ -161,11 +169,11 @@ class TDTestCase:
                 tdLog.info("drop two udf functions success ")
 
         # create  scalar functions
-        tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+        tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
 
         # create aggregate functions
 
-        tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+        tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
         
         functions = tdSql.getResult("show functions")
         function_nums = len(functions)
@@ -533,8 +541,8 @@ class TDTestCase:
         tdSql.query("drop function udf2 ")
 
         # create function without buffer
-        tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int")
+        tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1)
-        tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double")
+        tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2)
         udf1_sqls ,udf2_sqls = self.try_query_sql()
 
         for scalar_sql in udf1_sqls:
@@ -549,8 +557,8 @@ class TDTestCase:
         tdSql.query("drop function udf2 ")
 
         # create function without buffer
-        tdSql.execute("create  aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute("create  aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1)
-        tdSql.execute("create  function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.execute("create  function udf2 as '%s' outputtype double bufSize 8"%self.libudf2)
         udf1_sqls ,udf2_sqls = self.try_query_sql()
 
         for scalar_sql in udf1_sqls:
@@ -558,8 +566,8 @@ class TDTestCase:
         for aggregate_sql in udf2_sqls:
             tdSql.error(aggregate_sql)
 
-        tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1)
-        tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1)
         tdSql.error(" select db(c1) from stb1 ")
         tdSql.error(" select db(c1,c6), db(c6) from stb1 ")
         tdSql.error(" select db(num1,num2), db(num1) from tb ")
@@ -607,17 +615,17 @@ class TDTestCase:
         tdLog.info(" create function name is not build_in functions ")
         tdSql.execute(" drop function udf1 ")
         tdSql.execute(" drop function udf2 ")
-        tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
+        tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
-        tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
+        tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
-        tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2)
 
     def restart_taosd_query_udf(self):
 

tests/system-test/0-others/udf_restart_taosd.py

@@ -25,7 +25,7 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
 
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if ("taosd" in files or "taosd.exe" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
@@ -41,11 +41,19 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
         print(projPath)
 
-        libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+        if platform.system().lower() == 'windows':
-        libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+            self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
-        os.system("mkdir /tmp/udf/")
+            self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
-        os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,""))
+            if (not tdDnodes.dnodes[0].remoteIP == ""):
-        os.system("cp  %s /tmp/udf/ "%libudf2.replace("\n" ,""))
+                tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\")
+                tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\")
+                self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so')
+                self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so')
+        else:
+            self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+            self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+        self.libudf1 = self.libudf1.replace('\r','').replace('\n','')
+        self.libudf2 = self.libudf2.replace('\r','').replace('\n','')
 
 
     def prepare_data(self):
@@ -136,11 +144,11 @@ class TDTestCase:
 
         for i in range(5):
             # create  scalar functions
-            tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+            tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
 
             # create aggregate functions
 
-            tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+            tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
             
             functions = tdSql.getResult("show functions")
             function_nums = len(functions)
@@ -161,11 +169,11 @@ class TDTestCase:
                 tdLog.info("drop two udf functions success ")
 
         # create  scalar functions
-        tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+        tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1)
 
         # create aggregate functions
 
-        tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+        tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
         
         functions = tdSql.getResult("show functions")
         function_nums = len(functions)
@@ -533,8 +541,8 @@ class TDTestCase:
         tdSql.query("drop function udf2 ")
 
         # create function without buffer
-        tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int")
+        tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1)
-        tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double")
+        tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2)
         udf1_sqls ,udf2_sqls = self.try_query_sql()
 
         for scalar_sql in udf1_sqls:
@@ -549,8 +557,8 @@ class TDTestCase:
         tdSql.query("drop function udf2 ")
 
         # create function without buffer
-        tdSql.execute("create  aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute("create  aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1)
-        tdSql.execute("create  function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.execute("create  function udf2 as '%s' outputtype double bufSize 8"%self.libudf2)
         udf1_sqls ,udf2_sqls = self.try_query_sql()
 
         for scalar_sql in udf1_sqls:
@@ -558,8 +566,8 @@ class TDTestCase:
         for aggregate_sql in udf2_sqls:
             tdSql.error(aggregate_sql)
 
-        tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1)
-        tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ")
+        tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1)
         tdSql.error(" select db(c1) from stb1 ")
         tdSql.error(" select db(c1,c6), db(c6) from stb1 ")
         tdSql.error(" select db(num1,num2), db(num1) from tb ")
@@ -607,17 +615,17 @@ class TDTestCase:
         tdLog.info(" create function name is not build_in functions ")
         tdSql.execute(" drop function udf1 ")
         tdSql.execute(" drop function udf2 ")
-        tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
+        tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
-        tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8")
+        tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1)
-        tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2)
-        tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8")
+        tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2)
 
     def restart_taosd_query_udf(self):
 

tests/system-test/2-query/json_tag.py

@@ -137,9 +137,9 @@ class TDTestCase:
         tdSql.checkRows(9)
         tdSql.query("select jtag from jsons1")
         tdSql.checkRows(13)
-        # tdSql.query("select jtag from jsons1 where jtag is null")
+        tdSql.query("select jtag from jsons1 where jtag is null")
         # tdSql.checkRows(5)
-        # tdSql.query("select jtag from jsons1 where jtag is not null")
+        tdSql.query("select jtag from jsons1 where jtag is not null")
         # tdSql.checkRows(8)
 
         # test jtag is NULL
@@ -260,9 +260,9 @@ class TDTestCase:
         # tdSql.checkRows(1)
         #
         # # where json is null
-        # tdSql.query("select * from jsons1 where jtag is null")
+        tdSql.query("select * from jsons1 where jtag is null")
         # tdSql.checkRows(1)
-        # tdSql.query("select * from jsons1 where jtag is not null")
+        tdSql.query("select * from jsons1 where jtag is not null")
         # tdSql.checkRows(8)
         #
         # # where json key is null
@@ -389,8 +389,8 @@ class TDTestCase:
         tdSql.checkData(2, 1, "11.000000000")
         tdSql.checkData(5, 0, 1)
         tdSql.checkData(5, 1, "false")
-        # tdSql.checkData(6, 0, 1)
+        tdSql.checkData(6, 0, 1)
-        # tdSql.checkData(6, 1, "null")
+        tdSql.checkData(6, 1, "null")
         tdSql.checkData(7, 0, 2)
         tdSql.checkData(7, 1, None)
 
@@ -409,7 +409,7 @@ class TDTestCase:
         tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
         tdSql.checkRows(8)
         tdSql.checkData(0, 0, 10)
-        # tdSql.checkData(0, 1, None)
+        tdSql.checkData(0, 1, None)
         tdSql.checkData(4, 0, 0)
         tdSql.checkData(4, 1, "5.000000000")
         tdSql.checkData(7, 0, 11)
@@ -424,10 +424,10 @@ class TDTestCase:
         # test top/bottom with group by json tag
         tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
         tdSql.checkRows(11)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(2, 0, 4)
         tdSql.checkData(3, 0, 3)
         tdSql.checkData(3, 1, "false")
-        # tdSql.checkData(3, 0, 24)
-        # tdSql.checkData(3, 1, None)
         tdSql.checkData(10, 0, 23)
         tdSql.checkData(10, 1, '"femail"')
 
@@ -436,7 +436,7 @@ class TDTestCase:
         # tdSql.checkRows(2)
 
         # subquery with json tag
-        tdSql.query("select * from (select jtag, dataint from jsons1)")
+        tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint")
         tdSql.checkRows(11)
         tdSql.checkData(1, 1, 1)
         tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}')

tests/system-test/fulltest.bat

@@ -1,4 +1,7 @@
 
 python3 .\test.py -f 0-others\taosShell.py
 python3 .\test.py -f 0-others\taosShellError.py
-python3 .\test.py -f 0-others\taosShellNetChk.py
+python3 .\test.py -f 0-others\taosShellNetChk.py
+python3 .\test.py -f 0-others\udfTest.py
+python3 .\test.py -f 0-others\udf_create.py
+python3 .\test.py -f 0-others\udf_restart_taosd.py