Merge pull request #13388 from taosdata/3.0

3.0

CMakeLists.txt

@@ -35,7 +35,7 @@ endif(${BUILD_TEST})
 add_subdirectory(source)
 add_subdirectory(tools)
 add_subdirectory(tests)
-add_subdirectory(example)
+add_subdirectory(examples/c)
 
 # docs 
 add_subdirectory(docs)

cmake/cmake.define

@@ -46,7 +46,7 @@ ENDIF ()
 
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
-    SET(COMMON_FLAGS "/w /D_WIN32 /Zi")
+    SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi")
     SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
     # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
     #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")

docs-cn/12-taos-sql/07-function.md

@@ -1,6 +1,6 @@
 ---
-sidebar_label: SQL 函数
+sidebar_label: 函数
-title: SQL 函数
+title: 函数
 toc_max_heading_level: 4
 ---
 
@@ -20,7 +20,7 @@ toc_max_heading_level: 4
 
 **返回结果类型**：如果输入值为整数，输出值是 UBIGINT 类型。如果输入值是 FLOAT/DOUBLE 数据类型，输出值是 DOUBLE 数据类型。
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -38,7 +38,7 @@ toc_max_heading_level: 4
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -56,7 +56,7 @@ toc_max_heading_level: 4
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -75,7 +75,7 @@ toc_max_heading_level: 4
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -94,7 +94,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：与指定列的原始数据类型一致。例如，如果指定列的原始数据类型为 Float，那么返回的数据类型也为 Float；如果指定列的原始数据类型为 Double，那么返回的数据类型也为 Double。
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列，无论 tag 列的类型是什么类型。
+**适用数据类型**：数值类型。
 
 **适用于**: 普通表、超级表。
 
@@ -115,7 +115,7 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -142,7 +142,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -161,7 +161,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -190,7 +190,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -208,7 +208,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -226,7 +226,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：DOUBLE。如果输入值为 NULL，输出值也为 NULL
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在 tag 列
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -248,7 +248,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：INT。如果输入值为NULL，输出值为NULL。
 
-**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。
+**适用数据类型**：VARCHAR, NCHAR
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -262,9 +262,9 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **功能说明**：字符串连接函数。
 
-**返回结果类型**：如果所有参数均为BINARY类型，则结果类型为BINARY。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。
+**返回结果类型**：如果所有参数均为 VARCHAR 类型，则结果类型为 VARCHAR。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。
 
-**适用数据类型**：BINARY, NCHAR。不能应用在 TAG 列。 该函数最小参数个数为2个，最大参数个数为8个。
+**适用数据类型**：VARCHAR, NCHAR。 该函数最小参数个数为2个，最大参数个数为8个。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -279,9 +279,9 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **功能说明**：带分隔符的字符串连接函数。
 
-**返回结果类型**：如果所有参数均为BINARY类型，则结果类型为BINARY。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。如果separator值不为NULL，其他输入为NULL，输出为空串。
+**返回结果类型**：如果所有参数均为VARCHAR类型，则结果类型为VARCHAR。如果参数包含NCHAR类型，则结果类型为NCHAR。如果输入值为NULL，输出值为NULL。如果separator值不为NULL，其他输入为NULL，输出为空串。
 
-**适用数据类型**：BINARY, NCHAR。不能应用在 TAG 列。 该函数最小参数个数为3个，最大参数个数为9个。
+**适用数据类型**：VARCHAR, NCHAR。 该函数最小参数个数为3个，最大参数个数为9个。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -298,7 +298,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：INT。
 
-**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。
+**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -315,7 +315,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。
 
-**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。
+**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -332,7 +332,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。
 
-**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。
+**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -349,7 +349,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。
 
-**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。
+**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -366,7 +366,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。
 
-**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。输入参数pos可以为正数，也可以为负数。如果pos是正数，表示开始位置从字符串开头正数计算。如果pos为负数，表示开始位置从字符串结尾倒数计算。如果输入参数len被忽略，返回的子串包含从pos开始的整个字串。
+**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。输入参数pos可以为正数，也可以为负数。如果pos是正数，表示开始位置从字符串开头正数计算。如果pos为负数，表示开始位置从字符串结尾倒数计算。如果输入参数len被忽略，返回的子串包含从pos开始的整个字串。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -383,7 +383,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回结果类型**：同输入类型。如果输入值为NULL，输出值为NULL。
 
-**适用数据类型**：输入参数是 BINARY 类型或者 NCHAR 类型的字符串或者列。不能应用在 TAG 列。
+**适用数据类型**：输入参数是 VARCHAR 类型或者 NCHAR 类型的字符串或者列。
 
 **嵌套子查询支持**：适用于内层查询和外层查询。
 
@@ -400,7 +400,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
   SELECT CAST(expression AS type_name) FROM { tb_name | stb_name } [WHERE clause]
 ```
 
-**功能说明**：数据类型转换函数，输入参数 expression 支持普通列、常量、标量函数及它们之间的四则运算，不支持 tag 列，只适用于 select 子句中。
+**功能说明**：数据类型转换函数，输入参数 expression 支持普通列、常量、标量函数及它们之间的四则运算，只适用于 select 子句中。
 
 **返回结果类型**：CAST 中指定的类型（type_name），可以是 BIGINT、BIGINT UNSIGNED、BINARY、VARCHAR、NCHAR和TIMESTAMP。
 
@@ -423,7 +423,7 @@ SELECT TO_ISO8601(ts_val | ts_col) FROM { tb_name | stb_name } [WHERE clause];
 
 **功能说明**：将 UNIX 时间戳转换成为 ISO8601 标准的日期时间格式，并附加客户端时区信息。
 
-**返回结果数据类型**：BINARY 类型。
+**返回结果数据类型**：VARCHAR 类型。
 
 **适用数据类型**：UNIX 时间戳常量或是 TIMESTAMP 类型的列
 
@@ -462,7 +462,7 @@ SELECT TO_UNIXTIMESTAMP(datetime_string | ts_col) FROM { tb_name | stb_name } [W
 
 **返回结果数据类型**：长整型 INT64。
 
-**应用字段**：字符串常量或是 BINARY/NCHAR 类型的列。
+**应用字段**：字符串常量或是 VARCHAR/NCHAR 类型的列。
 
 **适用于**：表、超级表。
 
@@ -549,7 +549,7 @@ SELECT TIMEZONE() FROM { tb_name | stb_name } [WHERE clause];
 
 **功能说明**：返回客户端当前时区信息。
 
-**返回结果数据类型**：BINARY 类型。
+**返回结果数据类型**：VARCHAR 类型。
 
 **应用字段**：无
 
@@ -668,7 +668,7 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
 SELECT MODE(field_name) FROM tb_name [WHERE clause];
 ```
 
-**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出空。不能匹配标签、时间戳输出。
+**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出空。
 
 **返回数据类型**：同应用的字段。
 
@@ -808,6 +808,25 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
 - 系统同时返回该记录关联的时间戳列；
 - 限制：BOTTOM 函数不支持 FILL 子句。
 
+### FIRST
+
+```
+SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
+```
+
+**功能说明**：统计表/超级表中某列的值最先写入的非 NULL 值。
+
+**返回数据类型**：同应用的字段。
+
+**适用数据类型**：所有字段。
+
+**适用于**：表和超级表。
+
+**使用说明**:
+
+- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；
+- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；
+- 如果结果集中所有列全部为 NULL 值，则不返回结果。
 
 ### INTERP
 
@@ -919,25 +938,6 @@ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
 
 **使用说明**：*P*值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX。
 
-### FIRST
-
-```
-SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
-```
-
-**功能说明**：统计表/超级表中某列的值最先写入的非 NULL 值。
-
-**返回数据类型**：同应用的字段。
-
-**适用数据类型**：所有字段。
-
-**适用于**：表和超级表。
-
-**使用说明**:
-
-- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；
-- 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；
-- 如果结果集中所有列全部为 NULL 值，则不返回结果。
 
 ### TAIL
 
@@ -1005,7 +1005,7 @@ SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause];
 
 **返回结果类型**： 输入列如果是整数类型返回值为长整型 （int64_t），浮点数返回值为双精度浮点数（Double）。无符号整数类型返回值为无符号长整型（uint64_t）。 返回结果中同时带有每行记录对应的时间戳。
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上；在超级表查询中使用时，不能应用在标签之上。
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**： 适用于内层查询和外层查询。
 
@@ -1076,7 +1076,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause;
 
   **返回结果类型**： 返回双精度浮点数类型。
 
-  **适用数据类型**： 不能应用在 timestamp、binary、nchar、bool 类型上；在超级表查询中使用时，不能应用在标签之上。
+  **适用数据类型**： 数值类型。
 
   **嵌套子查询支持**： 适用于内层查询和外层查询。
 
@@ -1124,7 +1124,7 @@ SELECT STATECOUNT(field_name, oper, val) FROM { tb_name | stb_name } [WHERE clau
 
 **返回结果类型**：整形。
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：不支持应用在子查询上。
 
@@ -1152,7 +1152,7 @@ SELECT stateDuration(field_name, oper, val, unit) FROM { tb_name | stb_name } [W
 
 **返回结果类型**：整形。
 
-**适用数据类型**：不能应用在 timestamp、binary、nchar、bool 类型字段上。
+**适用数据类型**：数值类型。
 
 **嵌套子查询支持**：不支持应用在子查询上。
 

docs-cn/12-taos-sql/13-operators.md

@@ -35,8 +35,8 @@ TDengine 支持 `UNION ALL` 和 `UNION` 操作符。UNION ALL 将查询返回的
 | --- | :---------------: | -------------------------------------------------------------------- | -------------------- |
 | 1   |         =         | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型                             | 相等                 |
 | 2   |      <\>, !=      | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 不相等               |
-| 3   |      \>, \<       | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型                             | 大于，小于           |
+| 3   |      \>, <       | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型                             | 大于，小于           |
-| 4   |     \>=, \<=      | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型                             | 大于等于，小于等于   |
+| 4   |     \>=, <=      | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型                             | 大于等于，小于等于   |
 | 5   |   IS [NOT] NULL   | 所有类型                                                             | 是否为空值           |
 | 6   | [NOT] BETWEEN AND | 除 BOOL、BLOB、MEDIUMBLOB 和 JSON 外的所有类型                       | 闭区间比较           |
 | 7   |        IN         | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 与列表内的任意值相等 |

docs-en/12-taos-sql/13-operators.md

@@ -35,8 +35,8 @@ TDengine provides 2 set operators: `UNION ALL` and `UNION`. `UNION ALL` combines
 | --- | :---------------: | ------------------------------------------------------------------- | ----------------------------------------------- |
 | 1   |         =         | Except for BLOB, MEDIUMBLOB and JSON                                | Equal                                           |
 | 2   |      <\>, !=      | Except for BLOB, MEDIUMBLOB, JSON and primary key of timestamp type | Not equal                                       |
-| 3   |      \>, \<       | Except for BLOB, MEDIUMBLOB and JSON                                | Greater than, less than                         |
+| 3   |      \>, <       | Except for BLOB, MEDIUMBLOB and JSON                                | Greater than, less than                         |
-| 4   |     \>=, \<=      | Except for BLOB, MEDIUMBLOB and JSON                                | Greater than or equal to, less than or equal to |
+| 4   |     \>=, <=      | Except for BLOB, MEDIUMBLOB and JSON                                | Greater than or equal to, less than or equal to |
 | 5   |   IS [NOT] NULL   | Any types                                                           | Is NULL or NOT                                  |
 | 6   | [NOT] BETWEEN AND | Except for BLOB, MEDIUMBLOB and JSON                                | In a value range or not                         |
 | 7   |        IN         | Except for BLOB, MEDIUMBLOB, JSON and primary key of timestamp type | In a list of values or not                      |

example/CMakeLists.txt

@@ -1,49 +0,0 @@
-add_executable(tmq "")
-add_executable(tstream "")
-add_executable(demoapi "")
-
-target_sources(tmq
-    PRIVATE
-    "src/tmq.c"
-)
-
-target_sources(tstream
-    PRIVATE
-    "src/tstream.c"
-)
-
-target_sources(demoapi
-    PRIVATE
-    "src/demoapi.c"
-)
-
-target_link_libraries(tmq
-    taos_static
-)
-
-target_link_libraries(tstream
-    taos_static
-)
-
-target_link_libraries(demoapi
-    taos_static
-)
-
-target_include_directories(tmq
-    PUBLIC "${TD_SOURCE_DIR}/include/os"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-
-target_include_directories(tstream
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-
-target_include_directories(demoapi
-    PUBLIC "${TD_SOURCE_DIR}/include/client"
-    PUBLIC "${TD_SOURCE_DIR}/include/os"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-
-SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq)
-SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream)
-SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi)

examples/c/CMakeLists.txt

@@ -3,20 +3,70 @@ PROJECT(TDengine)
 IF (TD_LINUX)
   INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
-  ADD_EXECUTABLE(demo apitest.c)
+  #  ADD_EXECUTABLE(demo apitest.c)
-  TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
+  #TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
-  ADD_EXECUTABLE(sml schemaless.c)
+  #ADD_EXECUTABLE(sml schemaless.c)
-  TARGET_LINK_LIBRARIES(sml taos_static trpc tutil pthread )
+  #TARGET_LINK_LIBRARIES(sml taos_static trpc tutil pthread )
-  ADD_EXECUTABLE(subscribe subscribe.c)
+  #ADD_EXECUTABLE(subscribe subscribe.c)
-  TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread )
+  #TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread )
-  ADD_EXECUTABLE(epoll epoll.c)
+  #ADD_EXECUTABLE(epoll epoll.c)
-  TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
+  #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
+
+  add_executable(tmq "")
+  add_executable(tstream "")
+  add_executable(demoapi "")
+
+  target_sources(tmq
+      PRIVATE
+      "tmq.c"
+      )
+
+  target_sources(tstream
+      PRIVATE
+      "tstream.c"
+      )
+
+  target_sources(demoapi
+      PRIVATE
+      "demoapi.c"
+      )
+
+  target_link_libraries(tmq
+      taos_static
+      )
+
+  target_link_libraries(tstream
+      taos_static
+      )
+
+  target_link_libraries(demoapi
+      taos_static
+      )
+
+  target_include_directories(tmq
+      PUBLIC "${TD_SOURCE_DIR}/include/os"
+      PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+      )
+
+  target_include_directories(tstream
+      PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+      )
+
+  target_include_directories(demoapi
+      PUBLIC "${TD_SOURCE_DIR}/include/client"
+      PUBLIC "${TD_SOURCE_DIR}/include/os"
+      PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+      )
+
+  SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq)
+  SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream)
+  SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi)
 ENDIF ()
 IF (TD_DARWIN)
   INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
-  ADD_EXECUTABLE(demo demo.c)
+  #ADD_EXECUTABLE(demo demo.c)
-  TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)
+  #TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)
-  ADD_EXECUTABLE(epoll epoll.c)
+  #ADD_EXECUTABLE(epoll epoll.c)
-  TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
+  #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
 ENDIF ()

examples/c/tmq.c

@@ -165,7 +165,6 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "group.id", "tg2");
   tmq_conf_set(conf, "td.connect.user", "root");
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
-  /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
   tmq_conf_set(conf, "msg.with.table.name", "true");
   tmq_conf_set(conf, "enable.auto.commit", "false");
   tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
@@ -191,7 +190,6 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     return;
   }
   int32_t cnt = 0;
-  /*clock_t startTime = clock();*/
   while (running) {
     TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0);
     if (tmqmessage) {
@@ -204,8 +202,6 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
       /*break;*/
     }
   }
-  /*clock_t endTime = clock();*/
-  /*printf("log cnt: %d %f s\n", cnt, (double)(endTime - startTime) / CLOCKS_PER_SEC);*/
 
   err = tmq_consumer_close(tmq);
   if (err)

include/common/tdataformat.h

@@ -18,6 +18,7 @@
 
 #include "os.h"
 #include "talgo.h"
+#include "tarray.h"
 #include "tencode.h"
 #include "ttypes.h"
 #include "tutil.h"
@@ -29,6 +30,7 @@ extern "C" {
 typedef struct SSchema       SSchema;
 typedef struct STColumn      STColumn;
 typedef struct STSchema      STSchema;
+typedef struct SValue        SValue;
 typedef struct SColVal       SColVal;
 typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
@@ -39,32 +41,36 @@ typedef struct STag          STag;
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
 void    tTSchemaDestroy(STSchema *pTSchema);
 
-// SColVal
-#define ColValNONE               ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL})
-#define ColValNULL               ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL})
-#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)})
-
 // STSRow2
+#define COL_VAL_NONE(CID)     ((SColVal){.cid = (CID), .isNone = 1})
+#define COL_VAL_NULL(CID)     ((SColVal){.cid = (CID), .isNull = 1})
+#define COL_VAL_VALUE(CID, V) ((SColVal){.cid = (CID), .value = (V)})
+
+int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow);
+void    tTSRowFree(STSRow2 *pRow);
+void    tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
+int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray);
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
-int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow);
-void    tTSRowFree(STSRow2 *pRow);
-int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 
 // STSRowBuilder
+#if 0
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
 void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
 int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow);
+#endif
 
 // STag
-int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag);
+int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag);
 void    tTagFree(STag *pTag);
-int32_t tTagSet(STag *pTag, SSchema *pSchema, int32_t nCols, int iCol, uint8_t *pData, uint32_t nData, STag **ppTag);
+bool    tTagGet(const STag *pTag, STagVal *pTagVal);
-void    tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, uint32_t *nData);
+char*   tTagValToData(const STagVal *pTagVal, bool isJson);
 int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
 int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
+int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
+void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);
 
 // STRUCT =================
 struct STColumn {
@@ -87,7 +93,9 @@ struct STSchema {
 #define TSROW_HAS_NONE ((uint8_t)0x1)
 #define TSROW_HAS_NULL ((uint8_t)0x2U)
 #define TSROW_HAS_VAL  ((uint8_t)0x4U)
-#define TSROW_KV_ROW   ((uint8_t)0x10U)
+#define TSROW_KV_SMALL ((uint8_t)0x10U)
+#define TSROW_KV_MID   ((uint8_t)0x20U)
+#define TSROW_KV_BIG   ((uint8_t)0x40U)
 struct STSRow2 {
   TSKEY    ts;
   uint8_t  flags;
@@ -110,20 +118,60 @@ struct STSRowBuilder {
   STSRow2   row;
 };
 
-typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT;
+struct SValue {
-struct SColVal {
+  union {
-  EColValT type;
+    int8_t   i8;   // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT
-  uint32_t nData;
+    uint8_t  u8;   // TSDB_DATA_TYPE_UTINYINT
-  uint8_t *pData;
+    int16_t  i16;  // TSDB_DATA_TYPE_SMALLINT
+    uint16_t u16;  // TSDB_DATA_TYPE_USMALLINT
+    int32_t  i32;  // TSDB_DATA_TYPE_INT
+    uint32_t u32;  // TSDB_DATA_TYPE_UINT
+    int64_t  i64;  // TSDB_DATA_TYPE_BIGINT
+    uint64_t u64;  // TSDB_DATA_TYPE_UBIGINT
+    TSKEY    ts;   // TSDB_DATA_TYPE_TIMESTAMP
+    float    f;    // TSDB_DATA_TYPE_FLOAT
+    double   d;    // TSDB_DATA_TYPE_DOUBLE
+    struct {
+      uint32_t nData;
+      uint8_t *pData;
+    };
+  };
 };
 
-struct STagVal {
+struct SColVal {
-  int16_t  cid;
+  int16_t cid;
-  int8_t   type;
+  int8_t  isNone;
-  uint32_t nData;
+  int8_t  isNull;
-  uint8_t *pData;
+  SValue  value;
 };
 
+#pragma pack(push, 1)
+struct STagVal {
+  union {
+    int16_t cid;
+    char   *pKey;
+  };
+  int8_t type;
+  union {
+    int64_t  i64;
+    struct {
+      uint32_t nData;
+      uint8_t *pData;
+    };
+  };
+};
+
+#define TD_TAG_JSON  ((int8_t)0x40)   // distinguish JSON string and JSON value with the highest bit
+#define TD_TAG_LARGE ((int8_t)0x20)
+struct STag {
+  int8_t  flags;
+  int16_t len;
+  int16_t nTag;
+  int32_t ver;
+  int8_t  idx[];
+};
+#pragma pack(pop)
+
 #if 1  //================================================================================================================================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP
@@ -366,109 +414,6 @@ SDataCols *tdFreeDataCols(SDataCols *pCols);
 int32_t    tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update,
                            TDRowVerT maxVer);
 
-// ----------------- K-V data row structure
-/* |<-------------------------------------- len -------------------------------------------->|
- * |<----- header  ----->|<--------------------------- body -------------------------------->|
- * +----------+----------+---------------------------------+---------------------------------+
- * | uint16_t |  int16_t |                                 |                                 |
- * +----------+----------+---------------------------------+---------------------------------+
- * |    len   |   ncols  |           cols index            |             data part           |
- * +----------+----------+---------------------------------+---------------------------------+
- */
-typedef void *SKVRow;
-
-typedef struct {
-  int16_t  colId;
-  uint16_t offset;
-} SColIdx;
-
-#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
-
-#define kvRowLen(r)            (*(uint16_t *)(r))
-#define kvRowNCols(r)          (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
-#define kvRowSetLen(r, len)    kvRowLen(r) = (len)
-#define kvRowSetNCols(r, n)    kvRowNCols(r) = (n)
-#define kvRowColIdx(r)         (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)
-#define kvRowValues(r)         POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r))
-#define kvRowCpy(dst, r)       memcpy((dst), (r), kvRowLen(r))
-#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
-#define kvRowColIdxAt(r, i)    (kvRowColIdx(r) + (i))
-#define kvRowFree(r)           taosMemoryFreeClear(r)
-#define kvRowEnd(r)            POINTER_SHIFT(r, kvRowLen(r))
-#define kvRowValLen(r)         (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r))
-#define kvRowTKey(r)           (*(TKEY *)(kvRowValues(r)))
-#define kvRowKey(r)            tdGetKey(kvRowTKey(r))
-#define kvRowKeys(r)           POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t)))
-#define kvRowDeleted(r)        TKEY_IS_DELETED(kvRowTKey(r))
-
-SKVRow  tdKVRowDup(SKVRow row);
-int32_t tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value);
-int32_t tdEncodeKVRow(void **buf, SKVRow row);
-void   *tdDecodeKVRow(void *buf, SKVRow *row);
-void    tdSortKVRowByColIdx(SKVRow row);
-
-static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) {
-  if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
-    return 1;
-  } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
-    return -1;
-  } else {
-    return 0;
-  }
-}
-
-static FORCE_INLINE void *tdGetKVRowValOfCol(const SKVRow row, int16_t colId) {
-  void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
-  if (ret == NULL) return NULL;
-  return kvRowColVal(row, (SColIdx *)ret);
-}
-
-static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
-  return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
-}
-
-// ----------------- K-V data row builder
-typedef struct {
-  int16_t  tCols;
-  int16_t  nCols;
-  SColIdx *pColIdx;
-  uint16_t alloc;
-  uint16_t size;
-  void    *buf;
-} SKVRowBuilder;
-
-int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder);
-void    tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
-void    tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
-SKVRow  tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
-
-static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) {
-  if (pBuilder->nCols >= pBuilder->tCols) {
-    pBuilder->tCols *= 2;
-    SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
-    if (pColIdx == NULL) return -1;
-    pBuilder->pColIdx = pColIdx;
-  }
-
-  pBuilder->pColIdx[pBuilder->nCols].colId = colId;
-  pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size;
-
-  pBuilder->nCols++;
-
-  if (tlen > pBuilder->alloc - pBuilder->size) {
-    while (tlen > pBuilder->alloc - pBuilder->size) {
-      pBuilder->alloc *= 2;
-    }
-    void *buf = taosMemoryRealloc(pBuilder->buf, pBuilder->alloc);
-    if (buf == NULL) return -1;
-    pBuilder->buf = buf;
-  }
-
-  memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen);
-  pBuilder->size += tlen;
-
-  return 0;
-}
 #endif
 
 #ifdef __cplusplus
@@ -476,3 +421,4 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
 #endif
 
 #endif /*_TD_COMMON_DATA_FORMAT_H_*/
+

include/common/tmsg.h

@@ -244,7 +244,7 @@ typedef struct {
   const void* pMsg;
 } SSubmitMsgIter;
 
-int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter);
+int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter);
 int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
 int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
 STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
@@ -287,7 +287,7 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
-#define COL_IS_SET(FLG)  ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0)
+#define COL_IS_SET(FLG)  (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0)
 #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL)))
 
 #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
@@ -946,7 +946,6 @@ typedef struct {
   int64_t timeInFetchQueue;
 } SQnodeLoad;
 
-
 typedef struct {
   int32_t     sver;      // software version
   int64_t     dnodeVer;  // dnode table version in sdb
@@ -1003,7 +1002,6 @@ typedef struct {
 
 typedef struct {
   int32_t  vgId;
-  int32_t  dnodeId;
   char     db[TSDB_DB_FNAME_LEN];
   int64_t  dbUid;
   int32_t  vgVersion;
@@ -1026,16 +1024,14 @@ typedef struct {
   int8_t   compression;
   int8_t   strict;
   int8_t   cacheLastRow;
+  int8_t   isTsma;
+  int8_t   standby;
   int8_t   replica;
   int8_t   selfIndex;
   SReplica replicas[TSDB_MAX_REPLICA];
   int32_t  numOfRetensions;
   SArray*  pRetensions;  // SRetention
-
+  void*    pTsma;
-  // for tsma
-  int8_t isTsma;
-  void*  pTsma;
-
 } SCreateVnodeReq;
 
 int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
@@ -1073,8 +1069,8 @@ typedef struct {
   int8_t   walLevel;
   int8_t   strict;
   int8_t   cacheLastRow;
-  int8_t   replica;
   int8_t   selfIndex;
+  int8_t   replica;
   SReplica replicas[TSDB_MAX_REPLICA];
 } SAlterVnodeReq;
 
@@ -1779,6 +1775,15 @@ typedef struct SVCreateTbReq {
 int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq);
 int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq);
 
+static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) {
+  taosMemoryFreeClear(req->name);
+  if (req->type == TSDB_CHILD_TABLE) {
+    taosMemoryFreeClear(req->ctb.pTag);
+  } else if (req->type == TSDB_NORMAL_TABLE) {
+    taosMemoryFreeClear(req->ntb.schemaRow.pSchema);
+  }
+}
+
 typedef struct {
   int32_t nReqs;
   union {
@@ -1969,7 +1974,7 @@ typedef struct {
   int8_t   killConnection;
   int8_t   align[3];
   SEpSet   epSet;
-  SArray  *pQnodeList;
+  SArray*  pQnodeList;
 } SQueryHbRspBasic;
 
 typedef struct {
@@ -2297,6 +2302,7 @@ typedef struct {
   int8_t   intervalUnit;  // MACRO: TIME_UNIT_XXX
   int8_t   slidingUnit;   // MACRO: TIME_UNIT_XXX
   int8_t   timezoneInt;   // sma data expired if timezone changes.
+  int32_t  dstVgId;
   char     indexName[TSDB_INDEX_NAME_LEN];
   int32_t  exprLen;
   int32_t  tagsFilterLen;
@@ -2654,6 +2660,23 @@ typedef struct {
 int32_t tEncodeSVSubmitReq(SEncoder* pCoder, const SVSubmitReq* pReq);
 int32_t tDecodeSVSubmitReq(SDecoder* pCoder, SVSubmitReq* pReq);
 
+// TDMT_VND_DELETE
+typedef struct {
+  TSKEY sKey;
+  TSKEY eKey;
+
+  // super table
+  char* stbName;
+
+  // child/normal
+  char* tbName;
+} SVDeleteReq;
+
+typedef struct {
+  int32_t code;
+  // TODO
+} SVDeleteRsp;
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -221,9 +221,11 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CONFIG_CHANGE, "vnode-sync-config-change", NULL, NULL)
   
-  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "vnode-alter-config", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "vnode-alter-replica", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL)
+
+  TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "vnode-delete-data", SVDeleteReq, SVDeleteRsp)
 
   // Requests handled by QNODE
   TD_NEW_MSG_SEG(TDMT_QND_MSG)

include/libs/stream/tstream.h

@@ -61,11 +61,8 @@ enum {
 };
 
 typedef struct {
-  int8_t type;
+  int8_t      type;
-
+  int64_t     ver;
-  int32_t sourceVg;
-  int64_t sourceVer;
-
   int32_t*    dataRef;
   SSubmitReq* data;
 } SStreamDataSubmit;
@@ -111,6 +108,8 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit)
   }
 }
 
+SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit);
+
 int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
 void*   streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
 
@@ -209,8 +208,6 @@ struct SStreamTask {
   int32_t nodeId;
   SEpSet  epSet;
 
-  // source preprocess
-
   // exec
   STaskExec exec;
 
@@ -318,8 +315,6 @@ int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
 
 int32_t streamTaskRun(SStreamTask* pTask);
 
-int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
-
 int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb);
 int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
 int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp);

include/os/osDir.h

@@ -33,8 +33,19 @@ extern "C" {
 
 #ifdef WINDOWS
 #define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\"
+#define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\"
+#define TD_DATA_DIR_PATH "C:\\TDengine\\data\\"
+#define TD_LOG_DIR_PATH "C:\\TDengine\\log\\"
+#elif defined(_TD_DARWIN_64)
+#define TD_TMP_DIR_PATH "/tmp/taosd/"
+#define TD_CFG_DIR_PATH "/usr/local/etc/taos/"
+#define TD_DATA_DIR_PATH "/usr/local/var/lib/taos/"
+#define TD_LOG_DIR_PATH "/usr/local/var/log/taos/"
 #else
 #define TD_TMP_DIR_PATH "/tmp/"
+#define TD_CFG_DIR_PATH "/etc/taos/"
+#define TD_DATA_DIR_PATH "/var/lib/taos/"
+#define TD_LOG_DIR_PATH "/var/log/taos/"
 #endif
 
 typedef struct TdDir *TdDirPtr;

include/util/taoserror.h

@@ -70,6 +70,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_NEED_RETRY                    TAOS_DEF_ERROR_CODE(0, 0x0028)
 #define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE       TAOS_DEF_ERROR_CODE(0, 0x0029)
 #define TSDB_CODE_INVALID_TIMESTAMP             TAOS_DEF_ERROR_CODE(0, 0x0030)
+#define TSDB_CODE_MSG_DECODE_ERROR              TAOS_DEF_ERROR_CODE(0, 0x0031)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0040)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0041)

include/util/tencode.h

@@ -534,6 +534,26 @@ static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) {
   return sizeof(int64_t);
 }
 
+static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) {
+  union {
+    uint32_t ui;
+    float    f;
+  } v;
+  v.f = f;
+
+  return tPutU32(p, v.ui);
+}
+
+static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) {
+  union {
+    uint64_t ui;
+    double   d;
+  } v;
+  v.d = d;
+
+  return tPutU64(p, v.ui);
+}
+
 static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); }
 
 static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); }
@@ -623,6 +643,34 @@ static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) {
   return n;
 }
 
+static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) {
+  int32_t n = 0;
+
+  union {
+    uint32_t ui;
+    float    f;
+  } v;
+
+  n = tGetU32(p, &v.ui);
+
+  *f = v.f;
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) {
+  int32_t n = 0;
+
+  union {
+    uint64_t ui;
+    double   d;
+  } v;
+
+  n = tGetU64(p, &v.ui);
+
+  *d = v.d;
+  return n;
+}
+
 // =====================
 static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) {
   int n = 0;
@@ -646,6 +694,11 @@ static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* n
   return n;
 }
 
+static FORCE_INLINE int32_t tPutCStr(uint8_t* p, char* pData) {
+  return tPutBinary(p, (uint8_t*)pData, strlen(pData) + 1);
+}
+static FORCE_INLINE int32_t tGetCStr(uint8_t* p, char** ppData) { return tGetBinary(p, (uint8_t**)ppData, NULL); }
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientImpl.c

@@ -1034,27 +1034,20 @@ static char* parseTagDatatoJson(void* p) {
     goto end;
   }
 
-  int16_t nCols = kvRowNCols(p);
+  SArray* pTagVals = NULL;
+  if (tTagToValArray((const STag*)p, &pTagVals) != 0) {
+    goto end;
+  }
+
+  int16_t nCols = taosArrayGetSize(pTagVals);
   char    tagJsonKey[256] = {0};
   for (int j = 0; j < nCols; ++j) {
-    SColIdx* pColIdx = kvRowColIdxAt(p, j);
+    STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j);
-    char*    val = (char*)(kvRowColVal(p, pColIdx));
-    if (j == 0) {
-      if (*val == TSDB_DATA_TYPE_NULL) {
-        string = taosMemoryCalloc(1, 8);
-        sprintf(string, "%s", TSDB_DATA_NULL_STR_L);
-        goto end;
-      }
-      continue;
-    }
-
     // json key  encode by binary
     memset(tagJsonKey, 0, sizeof(tagJsonKey));
-    memcpy(tagJsonKey, varDataVal(val), varDataLen(val));
+    memcpy(tagJsonKey, pTagVal->pKey, strlen(pTagVal->pKey));
     // json value
-    val += varDataTLen(val);
+    char  type = pTagVal->type;
-    char* realData = POINTER_SHIFT(val, CHAR_BYTES);
-    char  type = *val;
     if (type == TSDB_DATA_TYPE_NULL) {
       cJSON* value = cJSON_CreateNull();
       if (value == NULL) {
@@ -1063,11 +1056,11 @@ static char* parseTagDatatoJson(void* p) {
       cJSON_AddItemToObject(json, tagJsonKey, value);
     } else if (type == TSDB_DATA_TYPE_NCHAR) {
       cJSON* value = NULL;
-      if (varDataLen(realData) > 0) {
+      if (pTagVal->nData > 0) {
-        char*   tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1);
+        char*   tagJsonValue = taosMemoryCalloc(pTagVal->nData, 1);
-        int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(realData), varDataLen(realData), tagJsonValue);
+        int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue);
         if (length < 0) {
-          tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, val);
+          tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, pTagVal->pData);
           taosMemoryFree(tagJsonValue);
           goto end;
         }
@@ -1076,7 +1069,7 @@ static char* parseTagDatatoJson(void* p) {
         if (value == NULL) {
           goto end;
         }
-      } else if (varDataLen(realData) == 0) {
+      } else if (pTagVal->nData == 0) {
         value = cJSON_CreateString("");
       } else {
         ASSERT(0);
@@ -1084,22 +1077,14 @@ static char* parseTagDatatoJson(void* p) {
 
       cJSON_AddItemToObject(json, tagJsonKey, value);
     } else if (type == TSDB_DATA_TYPE_DOUBLE) {
-      double jsonVd = *(double*)(realData);
+      double jsonVd = *(double*)(&pTagVal->i64);
       cJSON* value = cJSON_CreateNumber(jsonVd);
       if (value == NULL) {
         goto end;
       }
       cJSON_AddItemToObject(json, tagJsonKey, value);
-      //    }else if(type == TSDB_DATA_TYPE_BIGINT){
-      //      int64_t jsonVd = *(int64_t*)(realData);
-      //      cJSON* value = cJSON_CreateNumber((double)jsonVd);
-      //      if (value == NULL)
-      //      {
-      //        goto end;
-      //      }
-      //      cJSON_AddItemToObject(json, tagJsonKey, value);
     } else if (type == TSDB_DATA_TYPE_BOOL) {
-      char   jsonVd = *(char*)(realData);
+      char   jsonVd = *(char*)(&pTagVal->i64);
       cJSON* value = cJSON_CreateBool(jsonVd);
       if (value == NULL) {
         goto end;
@@ -1164,7 +1149,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
           if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
             sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
             varDataSetLen(dst, strlen(varDataVal(dst)));
-          } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) {
+          } else if (jsonInnerType == TD_TAG_JSON) {
             char* jsonString = parseTagDatatoJson(jsonInnerData);
             STR_TO_VARSTR(dst, jsonString);
             taosMemoryFree(jsonString);
@@ -1183,10 +1168,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
             double jsonVd = *(double*)(jsonInnerData);
             sprintf(varDataVal(dst), "%.9lf", jsonVd);
             varDataSetLen(dst, strlen(varDataVal(dst)));
-          } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) {
-            int64_t jsonVd = *(int64_t*)(jsonInnerData);
-            sprintf(varDataVal(dst), "%" PRId64, jsonVd);
-            varDataSetLen(dst, strlen(varDataVal(dst)));
           } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
             sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
             varDataSetLen(dst, strlen(varDataVal(dst)));

source/common/src/tdatablock.c

@@ -116,22 +116,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
-    int32_t dataLen = varDataTLen(pData);
+    int32_t dataLen = 0;
     if (type == TSDB_DATA_TYPE_JSON) {
       if (*pData == TSDB_DATA_TYPE_NULL) {
-        dataLen = 0;
-      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
-        dataLen = varDataTLen(pData + CHAR_BYTES);
-      } else if (*pData == TSDB_DATA_TYPE_DOUBLE) {
-        dataLen = DOUBLE_BYTES;
-      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
         dataLen = CHAR_BYTES;
-      } else if (*pData == TSDB_DATA_TYPE_JSON) {
+      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
-        dataLen = kvRowLen(pData + CHAR_BYTES);
+        dataLen = varDataTLen(pData + CHAR_BYTES) + CHAR_BYTES;
+      } else if (*pData == TSDB_DATA_TYPE_DOUBLE) {
+        dataLen = DOUBLE_BYTES + CHAR_BYTES;
+      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
+        dataLen = CHAR_BYTES + CHAR_BYTES;
+      } else if (*pData == TD_TAG_JSON) {   // json string
+        dataLen = ((STag*)(pData))->len;
       } else {
         ASSERT(0);
       }
-      dataLen += CHAR_BYTES;
+    }else {
+      dataLen = varDataTLen(pData);
     }
 
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;
@@ -1634,6 +1635,11 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
 SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid,
                             const char* stbFullName, int32_t vgId) {
   SSubmitReq* ret = NULL;
+  SArray*     tagArray = taosArrayInit(1, sizeof(STagVal));
+  if(!tagArray) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
 
   // cal size
   int32_t cap = sizeof(SSubmitReq);
@@ -1655,18 +1661,33 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       createTbReq.type = TSDB_CHILD_TABLE;
       createTbReq.ctb.suid = suid;
 
-      SKVRowBuilder kvRowBuilder = {0};
+
-      if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
+
-        ASSERT(0);
+      STagVal tagVal = {.cid = 1,
+                        .type = TSDB_DATA_TYPE_UBIGINT,
+                        .pData = (uint8_t*)&pDataBlock->info.groupId,
+                        .nData = sizeof(uint64_t)};
+      STag*   pTag = NULL;
+      taosArrayClear(tagArray);
+      taosArrayPush(tagArray, &tagVal);
+      tTagNew(tagArray, 1, false, &pTag);
+      if (!pTag) {
+        tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        return NULL;
       }
-      tdAddColToKVRow(&kvRowBuilder, 1, &pDataBlock->info.groupId, sizeof(uint64_t));
+      createTbReq.ctb.pTag = (uint8_t*)pTag;
-      createTbReq.ctb.pTag = tdGetKVRowFromBuilder(&kvRowBuilder);
-      tdDestroyKVRowBuilder(&kvRowBuilder);
 
       int32_t code;
       tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
-      if (code < 0) return NULL;
+
-      taosMemoryFree(cname);
+      tdDestroySVCreateTbReq(&createTbReq);
+
+      if (code < 0) {
+        tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        return NULL;
+      }
     }
 
     cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen;
@@ -1709,22 +1730,42 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       createTbReq.type = TSDB_CHILD_TABLE;
       createTbReq.ctb.suid = suid;
 
-      SKVRowBuilder kvRowBuilder = {0};
+      STagVal tagVal = {.cid = 1,
-      if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
+                        .type = TSDB_DATA_TYPE_UBIGINT,
-        ASSERT(0);
+                        .pData = (uint8_t*)&pDataBlock->info.groupId,
+                        .nData = sizeof(uint64_t)};
+      taosArrayClear(tagArray);
+      taosArrayPush(tagArray, &tagVal);
+      STag* pTag = NULL;
+      tTagNew(tagArray, 1, false, &pTag);
+      if (!pTag) {
+        tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        taosMemoryFreeClear(ret);
+        return NULL;
       }
-      tdAddColToKVRow(&kvRowBuilder, 1, &pDataBlock->info.groupId, sizeof(uint64_t));
+      createTbReq.ctb.pTag = (uint8_t*)pTag;
-      createTbReq.ctb.pTag = tdGetKVRowFromBuilder(&kvRowBuilder);
-      tdDestroyKVRowBuilder(&kvRowBuilder);
 
       int32_t code;
       tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
-      if (code < 0) return NULL;
+      if (code < 0) {
+        tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        taosMemoryFreeClear(ret);
+        return NULL;
+      }
 
       SEncoder encoder = {0};
       tEncoderInit(&encoder, blockData, schemaLen);
-      if (tEncodeSVCreateTbReq(&encoder, &createTbReq) < 0) return NULL;
+      code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
       tEncoderClear(&encoder);
+      tdDestroySVCreateTbReq(&createTbReq);
+
+      if (code < 0) {
+        taosArrayDestroy(tagArray);
+        taosMemoryFreeClear(ret);
+        return NULL;
+      }
     }
     blkHead->schemaLen = htonl(schemaLen);
 
@@ -1759,5 +1800,6 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
   }
 
   ret->length = htonl(ret->length);
+        taosArrayDestroy(tagArray);
   return ret;
 }

source/common/src/tmsg.c

@@ -28,7 +28,7 @@
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
-int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
+int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
   if (pMsg == NULL) {
     terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
     return -1;
@@ -165,7 +165,6 @@ int32_t tDecodeSQueryNodeLoad(SDecoder *pDecoder, SQueryNodeLoad *pLoad) {
   return 0;
 }
 
-
 int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI8(buf, pEp->inUse);
@@ -2936,7 +2935,6 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
@@ -2959,6 +2957,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
   if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->standby) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
@@ -2993,7 +2992,6 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
@@ -3016,6 +3014,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
   if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->standby) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
@@ -3055,7 +3054,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
 int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
   taosArrayDestroy(pReq->pRetensions);
   pReq->pRetensions = NULL;
-  if(pReq->isTsma) {
+  if (pReq->isTsma) {
     taosMemoryFreeClear(pReq->pTsma);
   }
   return 0;
@@ -3136,8 +3135,8 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
     SReplica *pReplica = &pReq->replicas[i];
     if (tEncodeSReplica(&encoder, pReplica) < 0) return -1;
@@ -3167,8 +3166,8 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
     SReplica *pReplica = &pReq->replicas[i];
     if (tDecodeSReplica(&decoder, pReplica) < 0) return -1;
@@ -3656,6 +3655,7 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) {
   if (tEncodeI8(pCoder, pSma->intervalUnit) < 0) return -1;
   if (tEncodeI8(pCoder, pSma->slidingUnit) < 0) return -1;
   if (tEncodeI8(pCoder, pSma->timezoneInt) < 0) return -1;
+  if (tEncodeI32(pCoder, pSma->dstVgId) < 0) return -1;
   if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1;
   if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1;
   if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1;
@@ -3678,6 +3678,7 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) {
   if (tDecodeI8(pCoder, &pSma->version) < 0) return -1;
   if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1;
   if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1;
+  if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1;
   if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1;
   if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1;
   if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1;
@@ -3911,7 +3912,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
 
   if (pReq->type == TSDB_CHILD_TABLE) {
     if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1;
-    if (tEncodeBinary(pCoder, pReq->ctb.pTag, kvRowLen(pReq->ctb.pTag)) < 0) return -1;
+    if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1;
   } else if (pReq->type == TSDB_NORMAL_TABLE) {
     if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1;
   } else {
@@ -3923,8 +3924,6 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
 }
 
 int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
-  uint32_t len;
-
   if (tStartDecode(pCoder) < 0) return -1;
 
   if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1;
@@ -3936,7 +3935,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
 
   if (pReq->type == TSDB_CHILD_TABLE) {
     if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1;
-    if (tDecodeBinary(pCoder, &pReq->ctb.pTag, &len) < 0) return -1;
+    if (tDecodeTag(pCoder, (STag **)&pReq->ctb.pTag) < 0) return -1;
   } else if (pReq->type == TSDB_NORMAL_TABLE) {
     if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1;
   } else {

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -219,9 +219,9 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER;
 
-  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -150,20 +150,26 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->tsdbCfg.minRows = pCreate->minRows;
   pCfg->tsdbCfg.maxRows = pCreate->maxRows;
   for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) {
-    memcpy(&pCfg->tsdbCfg.retentions[i], taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention));
+    SRetention *pRetention = &pCfg->tsdbCfg.retentions[i];
+    memcpy(pRetention, taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention));
+    if (i == 0) {
+      if ((pRetention->freq > 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
+    }
   }
+
   pCfg->walCfg.vgId = pCreate->vgId;
   pCfg->hashBegin = pCreate->hashBegin;
   pCfg->hashEnd = pCreate->hashEnd;
   pCfg->hashMethod = pCreate->hashMethod;
 
+  pCfg->standby = pCfg->standby;
   pCfg->syncCfg.myIndex = pCreate->selfIndex;
   pCfg->syncCfg.replicaNum = pCreate->replica;
   memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo));
   for (int i = 0; i < pCreate->replica; ++i) {
-    pCfg->syncCfg.nodeInfo[i].nodePort = pCreate->replicas[i].port;
+    SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
-    snprintf(pCfg->syncCfg.nodeInfo[i].nodeFqdn, sizeof(pCfg->syncCfg.nodeInfo[i].nodeFqdn), "%s",
+    pNode->nodePort = pCreate->replicas[i].port;
-             pCreate->replicas[i].fqdn);
+    tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
   }
 }
 
@@ -176,6 +182,8 @@ static void vmGenerateWrapperCfg(SVnodeMgmt *pMgmt, SCreateVnodeReq *pCreate, SW
 
 int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   SCreateVnodeReq createReq = {0};
+  SVnodeCfg       vnodeCfg = {0};
+  SWrapperCfg     wrapperCfg = {0};
   int32_t         code = -1;
   char            path[TSDB_FILENAME_LEN] = {0};
 
@@ -184,12 +192,9 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return -1;
   }
 
-  dDebug("vgId:%d, create vnode req is received, tsma:%d", createReq.vgId, createReq.isTsma);
+  dDebug("vgId:%d, create vnode req is received, tsma:%d standby:%d", createReq.vgId, createReq.isTsma,
-
+         createReq.standby);
-  SVnodeCfg vnodeCfg = {0};
   vmGenerateVnodeCfg(&createReq, &vnodeCfg);
-
-  SWrapperCfg wrapperCfg = {0};
   vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg);
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
@@ -218,9 +223,20 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
   if (code != 0) {
     dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
+    code = terrno;
     goto _OVER;
   }
 
+  if (createReq.isTsma) {
+    SMsgHead *smaMsg = createReq.pTsma;
+    uint32_t  contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead));
+    if (vnodeProcessCreateTSma(pImpl, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen) < 0) {
+      dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr());
+      code = terrno;
+      goto _OVER;
+    };
+  }
+
   code = vnodeStart(pImpl);
   if (code != 0) {
     dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr());
@@ -228,7 +244,10 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   code = vmWriteVnodeListToFile(pMgmt);
-  if (code != 0) goto _OVER;
+  if (code != 0) {
+    code = terrno;
+    goto _OVER;
+  }
 
 _OVER:
   if (code != 0) {
@@ -314,8 +333,9 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DISPATCH, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RECOVER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;
 
-  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
 

source/dnode/mnode/impl/inc/mndTrans.h

@@ -61,7 +61,7 @@ int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
 int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
 void    mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
 void    mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
-void    mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb);
+void    mndTransSetDbName(STrans *pTrans, const char *dbname);
 void    mndTransSetSerial(STrans *pTrans);
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -36,7 +36,7 @@ SArray *mndBuildDnodesArray(SMnode *pMnode);
 int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray);
 int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2);
 
-void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
+void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, bool standby);
 void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 

source/dnode/mnode/impl/src/mndDb.c

@@ -263,7 +263,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
   sdbRelease(pSdb, pDb);
 }
 
-static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) {
+static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool standby) {
   STransAction action = {0};
 
   SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
@@ -272,7 +272,7 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p
   mndReleaseDnode(pMnode, pDnode);
 
   int32_t contLen = 0;
-  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
+  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby);
   if (pReq == NULL) return -1;
 
   action.pCont = pReq;
@@ -288,7 +288,7 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p
   return 0;
 }
 
-static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
+static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) {
   STransAction action = {0};
   action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
@@ -298,7 +298,7 @@ static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD
 
   action.pCont = pReq;
   action.contLen = contLen;
-  action.msgType = TDMT_VND_ALTER_VNODE;
+  action.msgType = msgType;
 
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);
@@ -391,7 +391,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   }
 
   terrno = 0;
-  return TSDB_CODE_SUCCESS;
+  return terrno;
 }
 
 static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
@@ -472,7 +472,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
 
     for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
       SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
-      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid) != 0) {
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, false) != 0) {
         return -1;
       }
     }
@@ -556,7 +556,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
 
   mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db);
 
-  mndTransSetDbInfo(pTrans, &dbObj);
+  mndTransSetDbName(pTrans, dbObj.name);
   if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
   if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
   if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
@@ -694,29 +694,37 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
 static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pOld);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
-  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
+    sdbFreeRaw(pRedoRaw);
+    return -1;
+  }
 
+  sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
   return 0;
 }
 
 static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdbRaw *pCommitRaw = mndDbActionEncode(pNew);
   if (pCommitRaw == NULL) return -1;
-  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
+  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
-  if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
+    sdbFreeRaw(pCommitRaw);
+    return -1;
+  }
 
+  sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
   return 0;
 }
 
 static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
   if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
-    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup) != 0) {
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) {
       return -1;
     }
   } else {
     SVgObj newVgroup = {0};
     memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
+    mndTransSetSerial(pTrans);
+
     if (newVgroup.replica < pDb->cfg.replications) {
       mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
             pVgroup->vnodeGid[0].dnodeId);
@@ -726,9 +734,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj
         return -1;
       }
       newVgroup.replica = pDb->cfg.replications;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
-      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1;
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
-      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
     } else {
       mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
 
@@ -739,15 +747,18 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj
         return -1;
       }
       newVgroup.replica = pDb->cfg.replications;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
       if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
       if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
     }
 
     SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
     if (pVgRaw == NULL) return -1;
-    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
+    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
-    if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1;
+      sdbFreeRaw(pVgRaw);
+      return -1;
+    }
+    sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
   }
 
   return 0;
@@ -780,18 +791,16 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
 }
 
 static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *pNew) {
-  int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq);
-  if (pTrans == NULL) goto _OVER;
+  if (pTrans == NULL) return -1;
-
   mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name);
 
-  mndTransSetDbInfo(pTrans, pOld);
+  int32_t code = -1;
+  mndTransSetDbName(pTrans, pOld->name);
   if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
-
   code = 0;
 
 _OVER:
@@ -1047,7 +1056,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name);
-  mndTransSetDbInfo(pTrans, pDb);
+  mndTransSetDbName(pTrans, pDb->name);
 
   if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndSma.c

@@ -426,7 +426,7 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans,
   pVgroup->pTsma = pSmaReq;
 
   int32_t contLen = 0;
-  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
+  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, false);
   taosMemoryFreeClear(pSmaReq);
   if (pReq == NULL) return -1;
 
@@ -512,7 +512,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name);
-  mndTransSetDbInfo(pTrans, pDb);
+  mndTransSetDbName(pTrans, pDb->name);
   mndTransSetSerial(pTrans);
 
   if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
@@ -757,7 +757,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name);
-  mndTransSetDbInfo(pTrans, pDb);
+  mndTransSetDbName(pTrans, pDb->name);
 
   if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
   if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndStb.c

@@ -754,7 +754,7 @@ _OVER:
 }
 
 int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
-  mndTransSetDbInfo(pTrans, pDb);
+  mndTransSetDbName(pTrans, pDb->name);
   if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
@@ -1261,7 +1261,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name);
-  mndTransSetDbInfo(pTrans, pDb);
+  mndTransSetDbName(pTrans, pDb->name);
 
   if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
@@ -1407,7 +1407,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name);
-  mndTransSetDbInfo(pTrans, pDb);
+  mndTransSetDbName(pTrans, pDb->name);
 
   if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
   if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndTrans.c

@@ -619,8 +619,8 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *
   pTrans->paramLen = paramLen;
 }
 
-void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) {
+void mndTransSetDbName(STrans *pTrans, const char *dbname) {
-  memcpy(pTrans->dbname, pDb->name, TSDB_DB_FNAME_LEN);
+  memcpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN);
 }
 
 void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -51,9 +51,10 @@ int32_t mndInitVgroup(SMnode *pMnode) {
   };
 
   mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndProcessAlterVnodeRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndProcessAlterVnodeRsp);
   mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndProcessDropVnodeRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_VNODE_RSP, mndProcessCompactVnodeRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndProcessCompactVnodeRsp);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup);
@@ -188,10 +189,10 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
   sdbRelease(pSdb, pVgroup);
 }
 
-void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) {
+void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen,
+                             bool standby) {
   SCreateVnodeReq createReq = {0};
   createReq.vgId = pVgroup->vgId;
-  createReq.dnodeId = pDnode->id;
   memcpy(createReq.db, pDb->name, TSDB_DB_FNAME_LEN);
   createReq.dbUid = pDb->uid;
   createReq.vgVersion = pVgroup->version;
@@ -218,6 +219,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
   createReq.hashMethod = pDb->cfg.hashMethod;
   createReq.numOfRetensions = pDb->cfg.numOfRetensions;
   createReq.pRetensions = pDb->cfg.pRetensions;
+  createReq.standby = standby;
   createReq.isTsma = pVgroup->isTsma;
   createReq.pTsma = pVgroup->pTsma;
 
@@ -276,7 +278,6 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_
   alterReq.strict = pDb->cfg.strict;
   alterReq.cacheLastRow = pDb->cfg.cacheLastRow;
   alterReq.replica = pVgroup->replica;
-  alterReq.selfIndex = -1;
 
   for (int32_t v = 0; v < pVgroup->replica; ++v) {
     SReplica  *pReplica = &alterReq.replicas[v];
@@ -292,13 +293,6 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_
     mndReleaseDnode(pMnode, pVgidDnode);
   }
 
-#if 0
-  if (alterReq.selfIndex == -1) {
-    terrno = TSDB_CODE_MND_APP_ERROR;
-    return NULL;
-  }
-#endif
-
   int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq);
   if (contLen < 0) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -510,7 +504,7 @@ _OVER:
   taosArrayDestroy(pArray);
   return code;
 }
-
+//--->
 int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
   taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
@@ -538,7 +532,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
 
     SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos];
     pVgid->dnodeId = pDnode->id;
-    pVgid->role = TAOS_SYNC_STATE_FOLLOWER;
+    pVgid->role = TAOS_SYNC_STATE_ERROR;
     pDnode->numOfVnodes++;
 
     mInfo("db:%s, vgId:%d, vnode_index:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId);
@@ -549,16 +543,15 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
   terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
   return -1;
 }
-
+//--->
 int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) {
-  int32_t removedNum = 0;
-
   taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
     SDnodeObj *pDnode = taosArrayGet(pArray, i);
     mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes);
   }
 
+  int32_t removedNum = 0;
   for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) {
     SDnodeObj *pDnode = taosArrayGet(pArray, d);
 
@@ -664,6 +657,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
   int32_t numOfRows = 0;
   SVgObj *pVgroup = NULL;
   int32_t cols = 0;
+  int64_t curMs = taosGetTimestampMs();
 
   SDbObj *pDb = NULL;
   if (strlen(pShow->db) > 0) {
@@ -703,12 +697,15 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
       if (i < pVgroup->replica) {
         colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false);
 
+        bool       online = false;
+        SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId);
+        if (pDnode != NULL) {
+          online = mndIsDnodeOnline(pMnode, pDnode, curMs);
+          mndReleaseDnode(pMnode, pDnode);
+        }
+
         char        buf1[20] = {0};
-        SDnodeObj *pDnodeObj = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId);
+        const char *role = online ? syncStr(pVgroup->vnodeGid[i].role) : "OFFLINE";
-        ASSERT(pDnodeObj != NULL);
-        bool isOffLine = !mndIsDnodeOnline(pMnode, pDnodeObj, taosGetTimestampMs());
-        const char *role = isOffLine ? "OFFLINE" : syncStr(pVgroup->vnodeGid[i].role);
-        
         STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);
 
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);

source/dnode/mnode/impl/test/trans/trans2.cpp

@@ -128,7 +128,7 @@ class MndTestTrans2 : public ::testing::Test {
     mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1);
 
     if (pDb != NULL) {
-      mndTransSetDbInfo(pTrans, pDb);
+      mndTransSetDbName(pTrans, pDb->name);
     }
 
     int32_t code = mndTransPrepare(pMnode, pTrans);
@@ -201,7 +201,7 @@ class MndTestTrans2 : public ::testing::Test {
     }
 
     if (pDb != NULL) {
-      mndTransSetDbInfo(pTrans, pDb);
+      mndTransSetDbName(pTrans, pDb->name);
     }
 
     int32_t code = mndTransPrepare(pMnode, pTrans);

source/dnode/vnode/CMakeLists.txt

@@ -35,7 +35,6 @@ target_sources(
     "src/sma/smaTimeRange.c"
 
     # tsdb
-    # "src/tsdb/tsdbTDBImpl.c"
     "src/tsdb/tsdbCommit.c"
     "src/tsdb/tsdbCommit2.c"
     "src/tsdb/tsdbFile.c"
@@ -45,17 +44,18 @@ target_sources(
     "src/tsdb/tsdbMemTable2.c"
     "src/tsdb/tsdbRead.c"
     "src/tsdb/tsdbReadImpl.c"
-    # "src/tsdb/tsdbSma.c"
     "src/tsdb/tsdbWrite.c"
     "src/tsdb/tsdbSnapshot.c"
 
     # tq
     "src/tq/tq.c"
     "src/tq/tqExec.c"
-    "src/tq/tqCommit.c"
+    "src/tq/tqMeta.c"
+    "src/tq/tqRead.c"
     "src/tq/tqOffset.c"
     "src/tq/tqPush.c"
-    "src/tq/tqRead.c"
+    "src/tq/tqSink.c"
+    "src/tq/tqCommit.c"
 )
 target_include_directories(
     vnode

source/dnode/vnode/inc/vnode.h

@@ -68,6 +68,7 @@ void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
 int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever);
 int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
 int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
+int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 
 // meta
 typedef struct SMeta       SMeta;  // todo: remove
@@ -78,7 +79,7 @@ void        metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
 void        metaReaderClear(SMetaReader *pReader);
 int32_t     metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
 int32_t     metaReadNext(SMetaReader *pReader);
-const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid);
+const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal);
 
 typedef struct SMetaFltParam {
   tb_uid_t suid;
@@ -172,7 +173,9 @@ struct SVnodeCfg {
   bool     isHeap;
   bool     isWeak;
   int8_t   isTsma;
+  int8_t   isRsma;
   int8_t   hashMethod;
+  int8_t   standby;
   STsdbCfg tsdbCfg;
   SWalCfg  walCfg;
   SSyncCfg syncCfg;

source/dnode/vnode/src/inc/meta.h

@@ -117,7 +117,7 @@ typedef struct {
 } SSmaIdxKey;
 
 // metaTable ==================
-int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int8_t type, tb_uid_t uid,
+int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData,  int8_t type, tb_uid_t uid,
                         STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey);
 
 #ifndef META_REFACT

source/dnode/vnode/src/inc/sma.h

@@ -219,7 +219,7 @@ static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDisk
 void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
 
 int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
-int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version);
+int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version);
 // TODO: This is the basic params, and should wrap the params to a queryHandle.
 int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
 
@@ -227,4 +227,4 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
 }
 #endif
 
-#endif /*_TD_VNODE_SMA_H_*/
+#endif /*_TD_VNODE_SMA_H_*/

source/dnode/vnode/src/inc/tq.h

@@ -66,33 +66,27 @@ struct STqReadHandle {
 // tqPush
 
 typedef struct {
-  int64_t        consumerId;
+  STaosQueue* queue;
-  int32_t        epoch;
+  STaosQall*  qall;
-  int32_t        skipLogNum;
+  void*       qItem;
-  int64_t        reqOffset;
+} STqInputQ;
-  SRpcHandleInfo info;
-  SRWLatch       lock;
-} STqPushHandle;
 
-#if 0
 typedef struct {
-  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  // msg info
   int64_t consumerId;
+  int64_t reqOffset;
+  int64_t processedVer;
   int32_t epoch;
-  int8_t  subType;
+  int32_t skipLogNum;
-  // int8_t        withTbName;
+  // rpc info
-  // int8_t        withSchema;
+  int64_t        reqId;
-  // int8_t        withTag;
+  SRpcHandleInfo rpcInfo;
-  char*         qmsg;
+  // exec
-  SHashObj*     pDropTbUid;
+  int8_t    inputStatus;
-  STqPushHandle pushHandle;
+  int8_t    execStatus;
-  // SRWLatch        lock;
+  STqInputQ inputQ;
-  SWalReadHandle* pWalReader;
+  SRWLatch  lock;
-  // task number should be the same with fetch thread
+} STqPushHandle;
-  STqReadHandle* pExecReader[5];
-  qTaskInfo_t    task[5];
-} STqExec;
-#endif
 
 // tqExec
 
@@ -154,20 +148,21 @@ typedef struct {
 
 static STqMgmt tqMgmt = {0};
 
-// init once
+// tqRead
-int  tqInit();
-void tqCleanUp();
-
-// int32_t tEncodeSTqExec(SEncoder* pEncoder, const STqExec* pExec);
-// int32_t tDecodeSTqExec(SDecoder* pDecoder, STqExec* pExec);
-
-int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle);
-int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
-
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** pHeadWithCkSum);
 
+// tqExec
 int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId);
 
+// tqMeta
+int32_t tqMetaOpen(STQ* pTq);
+int32_t tqMetaClose(STQ* pTq);
+int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
+int32_t tqMetaDeleteHandle(STQ* pTq, const char* key);
+
+// tqSink
+void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
+
 // tqOffset
 STqOffsetStore* STqOffsetOpen(STqOffsetCfg*);
 void            STqOffsetClose(STqOffsetStore*);

source/dnode/vnode/src/inc/tsdb.h

@@ -32,14 +32,27 @@ extern "C" {
 #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
+typedef struct TSDBROW TSDBROW;
+typedef struct TSDBKEY TSDBKEY;
+typedef struct SDelOp  SDelOp;
+
+static int tsdbKeyCmprFn(const void *p1, const void *p2);
+
+// tsdbMemTable2.c ==============================================================================================
+typedef struct SMemTable SMemTable;
+
+int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable);
+void    tsdbMemTableDestroy2(SMemTable *pMemTable);
+
 // tsdbMemTable ================
+typedef struct STsdbRow      STsdbRow;
 typedef struct STbData       STbData;
 typedef struct STsdbMemTable STsdbMemTable;
 typedef struct SMergeInfo    SMergeInfo;
 typedef struct STable        STable;
 
 int  tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable);
-void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable);
+void tsdbMemTableDestroy(STsdbMemTable *pMemTable);
 int  tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead,
                            SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
 
@@ -845,6 +858,42 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
   return 0;
 }
 
+struct TSDBROW {
+  int64_t version;
+  STSRow2 tsRow;
+};
+
+struct TSDBKEY {
+  int64_t version;
+  TSKEY   ts;
+};
+
+struct SDelOp {
+  int64_t version;
+  TSKEY   sKey;  // included
+  TSKEY   eKey;  // included
+  SDelOp *pNext;
+};
+
+static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
+  TSDBKEY *pKey1 = (TSDBKEY *)p1;
+  TSDBKEY *pKey2 = (TSDBKEY *)p2;
+
+  if (pKey1->ts < pKey2->ts) {
+    return -1;
+  } else if (pKey1->ts > pKey2->ts) {
+    return 1;
+  }
+
+  if (pKey1->version < pKey2->version) {
+    return -1;
+  } else if (pKey1->version > pKey2->version) {
+    return 1;
+  }
+
+  return 0;
+}
+
 #endif
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/vnd.h

@@ -81,9 +81,10 @@ int32_t vnodeSyncCommit(SVnode* pVnode);
 int32_t vnodeAsyncCommit(SVnode* pVnode);
 
 // vnodeSync.c
-int32_t   vnodeSyncOpen(SVnode* pVnode, char* path);
+int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
-void      vnodeSyncStart(SVnode* pVnode);
+void    vnodeSyncStart(SVnode* pVnode);
-void      vnodeSyncClose(SVnode* pVnode);
+void    vnodeSyncClose(SVnode* pVnode);
+void    vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -125,6 +125,8 @@ int32_t      tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader);
 int32_t      tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData);
 
 // tq
+int     tqInit();
+void    tqCleanUp();
 STQ*    tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
 void    tqClose(STQ*);
 int     tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
@@ -145,11 +147,11 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t smaOpen(SVnode* pVnode);
 int32_t smaClose(SSma* pSma);
 
-int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version);
+int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version);
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
 
-int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq* pReq);
+int32_t tdProcessRSmaCreate(SVnode* pVnode, SVCreateStbReq* pReq);
 int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);
 int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
 int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
@@ -239,6 +241,8 @@ struct SVnode {
 #define VND_RSMA1(vnd)      ((vnd)->pSma->pRSmaTsdb1)
 #define VND_RSMA2(vnd)      ((vnd)->pSma->pRSmaTsdb2)
 #define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions)
+#define VND_IS_RSMA(v)      ((v)->config.isRsma == 1)
+#define VND_IS_TSMA(v)      ((v)->config.isTsma == 1)
 
 struct STbUidStore {
   tb_uid_t  suid;
@@ -271,11 +275,6 @@ struct SSma {
 #define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1)
 #define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2)
 
-static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) {
-  SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]);
-  return (pRetention->freq > 0 && pRetention->keep > 0);
-}
-
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 

source/dnode/vnode/src/meta/metaEntry.c

@@ -30,7 +30,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
     if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
     if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
-    if (tEncodeBinary(pCoder, pME->ctbEntry.pTags, kvRowLen(pME->ctbEntry.pTags)) < 0) return -1;
+    if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
@@ -47,7 +47,6 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
 }
 
 int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
-  uint32_t len;
   if (tStartDecode(pCoder) < 0) return -1;
 
   if (tDecodeI64(pCoder, &pME->version) < 0) return -1;
@@ -62,7 +61,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
     if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
     if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
-    if (tDecodeBinary(pCoder, &pME->ctbEntry.pTags, &len) < 0) return -1;  // (TODO)
+    if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1;  // (TODO)
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;

source/dnode/vnode/src/meta/metaQuery.c

@@ -573,10 +573,23 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) {
 
 #endif
 
-const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid) {
+const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) {
   ASSERT(pEntry->type == TSDB_CHILD_TABLE);
-  return tdGetKVRowValOfCol((const SKVRow)pEntry->ctbEntry.pTags, cid);
+  STag *tag = (STag *)pEntry->ctbEntry.pTags;
+  if (type == TSDB_DATA_TYPE_JSON){
+    if(tag->nTag == 0){
+      return NULL;
+    }
+    return tag;
+  }
+  bool find = tTagGet(tag, val);
+
+  if(!find){
+    return NULL;
+  }
+  return val;
 }
+
 typedef struct {
   SMeta *  pMeta;
   TBC *    pCur;
@@ -609,7 +622,13 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
   STagIdxKey *pKey = NULL;
   int32_t     nKey = 0;
 
-  ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, pCursor->type,
+  int32_t nTagData = 0;
+  if(IS_VAR_DATA_TYPE(param->type)){
+    nTagData = strlen(param->val);
+  }else{
+    nTagData = tDataTypes[param->type].bytes;
+  }
+  ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, nTagData, pCursor->type,
                             param->reverse ? INT64_MAX : INT64_MIN, &pKey, &nKey);
   if (ret != 0) {
     goto END;
@@ -651,4 +670,4 @@ END:
   taosMemoryFree(pCursor);
 
   return ret;
-}
+}

source/dnode/vnode/src/meta/metaTable.c

@@ -31,9 +31,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   int         vLen = 0;
   const void *pKey = NULL;
   const void *pVal = NULL;
-  void *      pBuf = NULL;
+  void       *pBuf = NULL;
   int32_t     szBuf = 0;
-  void *      p = NULL;
+  void       *p = NULL;
   SMetaReader mr = {0};
 
   // validate req
@@ -87,7 +87,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
   }
 
   // drop all child tables
-  TBC *   pCtbIdxc = NULL;
+  TBC    *pCtbIdxc = NULL;
   SArray *pArray = taosArrayInit(8, sizeof(tb_uid_t));
 
   tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
@@ -142,8 +142,8 @@ _exit:
 int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   SMetaEntry  oStbEntry = {0};
   SMetaEntry  nStbEntry = {0};
-  TBC *       pUidIdxc = NULL;
+  TBC        *pUidIdxc = NULL;
-  TBC *       pTbDbc = NULL;
+  TBC        *pTbDbc = NULL;
   const void *pData;
   int         nData;
   int64_t     oversion;
@@ -262,7 +262,7 @@ _err:
 }
 
 int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
-  void *   pData = NULL;
+  void    *pData = NULL;
   int      nData = 0;
   int      rc = 0;
   tb_uid_t uid;
@@ -288,7 +288,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
 }
 
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
-  void *     pData = NULL;
+  void      *pData = NULL;
   int        nData = 0;
   int        rc = 0;
   int64_t    version;
@@ -324,14 +324,14 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
 }
 
 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
-  void *          pVal = NULL;
+  void           *pVal = NULL;
   int             nVal = 0;
-  const void *    pData = NULL;
+  const void     *pData = NULL;
   int             nData = 0;
   int             ret = 0;
   tb_uid_t        uid;
   int64_t         oversion;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
   SMetaEntry      entry = {0};
   SSchemaWrapper *pSchema;
   int             c;
@@ -479,7 +479,7 @@ _err:
 static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
   SMetaEntry  ctbEntry = {0};
   SMetaEntry  stbEntry = {0};
-  void *      pVal = NULL;
+  void       *pVal = NULL;
   int         nVal = 0;
   int         ret;
   int         c;
@@ -510,7 +510,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   oversion = *(int64_t *)pData;
 
   // search table.db
-  TBC *    pTbDbc = NULL;
+  TBC     *pTbDbc = NULL;
   SDecoder dc1 = {0};
   SDecoder dc2 = {0};
 
@@ -534,7 +534,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   metaDecodeEntry(&dc2, &stbEntry);
 
   SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
   int32_t         iCol = 0;
   for (;;) {
     pColumn = NULL;
@@ -563,29 +563,39 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
     }
     memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
   } else {
-    SKVRowBuilder kvrb = {0};
+    const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags;
-    const SKVRow  pOldTag = (const SKVRow)ctbEntry.ctbEntry.pTags;
+    STag       *pNewTag = NULL;
-    SKVRow        pNewTag = NULL;
+    SArray     *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
-
+    if (!pTagArray) {
-    tdInitKVRowBuilder(&kvrb);
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
     for (int32_t i = 0; i < pTagSchema->nCols; i++) {
       SSchema *pCol = &pTagSchema->pSchema[i];
       if (iCol == i) {
-        tdAddColToKVRow(&kvrb, pCol->colId, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
+        STagVal val = {0};
+        val.type = pCol->type;
+        val.cid = pCol->colId;
+        if (IS_VAR_DATA_TYPE(pCol->type)) {
+          val.pData = pAlterTbReq->pTagVal;
+          val.nData = pAlterTbReq->nTagVal;
+        } else {
+          memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
+        }
+        taosArrayPush(pTagArray, &val);
       } else {
-        void *p = tdGetKVRowValOfCol(pOldTag, pCol->colId);
+        STagVal val = {0};
-        if (p) {
+        if (tTagGet(pOldTag, &val)) {
-          if (IS_VAR_DATA_TYPE(pCol->type)) {
+          taosArrayPush(pTagArray, &val);
-            tdAddColToKVRow(&kvrb, pCol->colId, p, varDataTLen(p));
-          } else {
-            tdAddColToKVRow(&kvrb, pCol->colId, p, pCol->bytes);
-          }
         }
       }
     }
-
+    if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) {
-    ctbEntry.ctbEntry.pTags = tdGetKVRowFromBuilder(&kvrb);
+      taosArrayDestroy(pTagArray);
-    tdDestroyKVRowBuilder(&kvrb);
+      goto _err;
+    }
+    ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag;
+    taosArrayDestroy(pTagArray);
   }
 
   // save to table.db
@@ -639,8 +649,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) {
 
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
-  void *   pKey = NULL;
+  void    *pKey = NULL;
-  void *   pVal = NULL;
+  void    *pVal = NULL;
   int      kLen = 0;
   int      vLen = 0;
   SEncoder coder = {0};
@@ -721,17 +731,17 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
   return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
 }
 
-int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid,
+int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,
                         STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) {
-  int32_t nTagData = 0;
+  // int32_t nTagData = 0;
 
-  if (pTagData) {
+  // if (pTagData) {
-    if (IS_VAR_DATA_TYPE(type)) {
+  //   if (IS_VAR_DATA_TYPE(type)) {
-      nTagData = varDataTLen(pTagData);
+  //     nTagData = varDataTLen(pTagData);
-    } else {
+  //   } else {
-      nTagData = tDataTypes[type].bytes;
+  //     nTagData = tDataTypes[type].bytes;
-    }
+  //   }
-  }
+  // }
   *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t);
 
   *ppTagIdxKey = (STagIdxKey *)taosMemoryMalloc(*nTagIdxKey);
@@ -755,14 +765,15 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
 }
 
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
-  void *         pData = NULL;
+  void          *pData = NULL;
   int            nData = 0;
   STbDbKey       tbDbKey = {0};
   SMetaEntry     stbEntry = {0};
-  STagIdxKey *   pTagIdxKey = NULL;
+  STagIdxKey    *pTagIdxKey = NULL;
   int32_t        nTagIdxKey;
   const SSchema *pTagColumn;       // = &stbEntry.stbEntry.schema.pSchema[0];
-  const void *   pTagData = NULL;  //
+  const void    *pTagData = NULL;  //
+  int32_t        nTagData = 0;
   SDecoder       dc = {0};
 
   // get super table
@@ -775,7 +786,21 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   metaDecodeEntry(&dc, &stbEntry);
 
   pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0];
-  pTagData = tdGetKVRowValOfCol((const SKVRow)pCtbEntry->ctbEntry.pTags, pTagColumn->colId);
+
+  STagVal tagVal = {.cid = pTagColumn->colId};
+  if (pTagColumn->type != TSDB_DATA_TYPE_JSON) {
+    tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal);
+    if (IS_VAR_DATA_TYPE(pTagColumn->type)) {
+      pTagData = tagVal.pData;
+      nTagData = (int32_t)tagVal.nData;
+    } else {
+      pTagData = &(tagVal.i64);
+      nTagData = tDataTypes[pTagColumn->type].bytes;
+    }
+  } else {
+    // pTagData = pCtbEntry->ctbEntry.pTags;
+    // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len;
+  }
 
   // update tag index
 #ifdef USE_INVERTED_INDEX
@@ -790,8 +815,8 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid);
   indexMultiTermDestroy(tmGroup);
 #else
-  if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, pTagColumn->type, pCtbEntry->uid,
+  if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
-                          &pTagIdxKey, &nTagIdxKey) < 0) {
+                          pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
     return -1;
   }
   tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
@@ -804,7 +829,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
 
 static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   SEncoder              coder = {0};
-  void *                pVal = NULL;
+  void                 *pVal = NULL;
   int                   vLen = 0;
   int                   rcode = 0;
   SSkmDbKey             skmDbKey = {0};

source/dnode/vnode/src/sma/sma.c

@@ -15,7 +15,6 @@
 
 #include "sma.h"
 
-
 // TODO: Who is responsible for resource allocate and release?
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) {
   int32_t code = TSDB_CODE_SUCCESS;
@@ -37,7 +36,7 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) {
   return code;
 }
 
-int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version) {
+int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version) {
   int32_t code = TSDB_CODE_SUCCESS;
   if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) {
     smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno));

source/dnode/vnode/src/sma/smaOpen.c

@@ -104,7 +104,7 @@ int32_t smaOpen(SVnode *pVnode) {
   taosThreadMutexInit(&pSma->mutex, NULL);
   pSma->locked = false;
 
-  if (vnodeIsRollup(pVnode)) {
+  if (VND_IS_RSMA(pVnode)) {
     STsdbKeepCfg keepCfg = {0};
     for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
       if (i == TSDB_RETENTION_L0) {

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -932,7 +932,7 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde
  * @param msg SSubmitReq
  * @return int32_t
  */
-int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) {
+int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) {
   // no time-range-sma, just return success
   if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) {
     smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma));

source/dnode/vnode/src/tq/tq.c

@@ -14,7 +14,6 @@
  */
 
 #include "tq.h"
-#include "tdbInt.h"
 
 int32_t tqInit() {
   int8_t old;
@@ -47,51 +46,6 @@ void tqCleanUp() {
   }
 }
 
-int tqExecKeyCompare(const void* pKey1, int32_t kLen1, const void* pKey2, int32_t kLen2) {
-  return strcmp(pKey1, pKey2);
-}
-
-int32_t tqStoreHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
-  int32_t code;
-  int32_t vlen;
-  tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code);
-  ASSERT(code == 0);
-
-  void* buf = taosMemoryCalloc(1, vlen);
-  if (buf == NULL) {
-    ASSERT(0);
-  }
-
-  SEncoder encoder;
-  tEncoderInit(&encoder, buf, vlen);
-
-  if (tEncodeSTqHandle(&encoder, pHandle) < 0) {
-    ASSERT(0);
-  }
-
-  TXN txn;
-
-  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbBegin(pTq->pMetaStore, &txn) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, &txn) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbCommit(pTq->pMetaStore, &txn) < 0) {
-    ASSERT(0);
-  }
-
-  tEncoderClear(&encoder);
-  taosMemoryFree(buf);
-  return 0;
-}
-
 STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
   STQ* pTq = taosMemoryMalloc(sizeof(STQ));
   if (pTq == NULL) {
@@ -108,60 +62,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
 
   pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
 
-  if (tdbOpen(path, 16 * 1024, 1, &pTq->pMetaStore) < 0) {
+  if (tqMetaOpen(pTq) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbTbOpen("handles", -1, -1, tqExecKeyCompare, pTq->pMetaStore, &pTq->pExecStore) < 0) {
-    ASSERT(0);
-  }
-
-  TXN txn;
-
-  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
-    ASSERT(0);
-  }
-
-  TBC* pCur;
-  if (tdbTbcOpen(pTq->pExecStore, &pCur, &txn) < 0) {
-    ASSERT(0);
-  }
-
-  void* pKey;
-  int   kLen;
-  void* pVal;
-  int   vLen;
-
-  tdbTbcMoveToFirst(pCur);
-  SDecoder decoder;
-
-  while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
-    STqHandle handle;
-    tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
-    tDecodeSTqHandle(&decoder, &handle);
-    handle.pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
-    for (int32_t i = 0; i < 5; i++) {
-      handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
-    }
-    if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-      for (int32_t i = 0; i < 5; i++) {
-        SReadHandle reader = {
-            .reader = handle.execHandle.pExecReader[i],
-            .meta = pTq->pVnode->pMeta,
-            .pMsgCb = &pTq->pVnode->msgCb,
-        };
-        handle.execHandle.exec.execCol.task[i] =
-            qCreateStreamExecTaskInfo(handle.execHandle.exec.execCol.qmsg, &reader);
-        ASSERT(handle.execHandle.exec.execCol.task[i]);
-      }
-    } else {
-      handle.execHandle.exec.execDb.pFilterOutTbUid =
-          taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-    }
-    taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle));
-  }
-
-  if (tdbTxnClose(&txn) < 0) {
     ASSERT(0);
   }
 
@@ -174,179 +75,16 @@ void tqClose(STQ* pTq) {
     taosHashCleanup(pTq->handles);
     taosHashCleanup(pTq->pStreamTasks);
     taosHashCleanup(pTq->pushMgr);
-    tdbClose(pTq->pMetaStore);
+    tqMetaClose(pTq);
     taosMemoryFree(pTq);
   }
   // TODO
 }
 
-int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) {
-  if (tStartEncode(pEncoder) < 0) return -1;
-  if (tEncodeCStr(pEncoder, pHandle->subKey) < 0) return -1;
-  if (tEncodeI64(pEncoder, pHandle->consumerId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pHandle->epoch) < 0) return -1;
-  if (tEncodeI8(pEncoder, pHandle->execHandle.subType) < 0) return -1;
-  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    if (tEncodeCStr(pEncoder, pHandle->execHandle.exec.execCol.qmsg) < 0) return -1;
-  }
-  tEndEncode(pEncoder);
-  return pEncoder->pos;
-}
-
-int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-  if (tDecodeCStrTo(pDecoder, pHandle->subKey) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pHandle->consumerId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pHandle->epoch) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pHandle->execHandle.subType) < 0) return -1;
-  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.exec.execCol.qmsg) < 0) return -1;
-  }
-  tEndDecode(pDecoder);
-  return 0;
-}
-
-int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
-  void* pIter = NULL;
-  while (1) {
-    pIter = taosHashIterate(pTq->handles, pIter);
-    if (pIter == NULL) break;
-    STqHandle* pExec = (STqHandle*)pIter;
-    if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-      for (int32_t i = 0; i < 5; i++) {
-        int32_t code = qUpdateQualifiedTableId(pExec->execHandle.exec.execCol.task[i], tbUidList, isAdd);
-        ASSERT(code == 0);
-      }
-    } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) {
-      if (!isAdd) {
-        int32_t sz = taosArrayGetSize(tbUidList);
-        for (int32_t i = 0; i < sz; i++) {
-          int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
-          taosHashPut(pExec->execHandle.exec.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0);
-        }
-      }
-    } else {
-      // tq update id
-    }
-  }
-  while (1) {
-    pIter = taosHashIterate(pTq->pStreamTasks, pIter);
-    if (pIter == NULL) break;
-    SStreamTask* pTask = (SStreamTask*)pIter;
-    if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
-      int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
-      ASSERT(code == 0);
-    }
-  }
-  return 0;
-}
-
-int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) {
-  if (msgType != TDMT_VND_SUBMIT) return 0;
-  void*       pIter = NULL;
-  STqHandle*  pHandle = NULL;
-  SSubmitReq* pReq = (SSubmitReq*)msg;
-  int32_t     workerId = 4;
-  int64_t     fetchOffset = ver;
-
-  while (1) {
-    pIter = taosHashIterate(pTq->pushMgr, pIter);
-    if (pIter == NULL) break;
-    pHandle = *(STqHandle**)pIter;
-
-    taosWLockLatch(&pHandle->pushHandle.lock);
-
-    /*SRpcHandleInfo* pInfo = atomic_load_ptr(&pHandle->pushHandle.pInfo);*/
-    /*ASSERT(pInfo);*/
-
-    SMqDataBlkRsp rsp = {0};
-    rsp.reqOffset = pHandle->pushHandle.reqOffset;
-    rsp.blockData = taosArrayInit(0, sizeof(void*));
-    rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
-
-    if (msgType == TDMT_VND_SUBMIT) {
-      tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
-    } else {
-      // TODO
-      ASSERT(0);
-    }
-
-    if (rsp.blockNum == 0) {
-      taosWUnLockLatch(&pHandle->pushHandle.lock);
-      continue;
-    }
-
-    ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
-    ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
-
-    rsp.rspOffset = fetchOffset;
-
-    int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
-    void*   buf = rpcMallocCont(tlen);
-    if (buf == NULL) {
-      // todo free
-      return -1;
-    }
-
-    ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
-    ((SMqRspHead*)buf)->epoch = pHandle->pushHandle.epoch;
-    ((SMqRspHead*)buf)->consumerId = pHandle->pushHandle.consumerId;
-
-    void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-    tEncodeSMqDataBlkRsp(&abuf, &rsp);
-
-    SRpcMsg resp = {
-        .info = pHandle->pushHandle.info,
-        .pCont = buf,
-        .contLen = tlen,
-        .code = 0,
-    };
-    tmsgSendRsp(&resp);
-
-    /*atomic_store_ptr(&pHandle->pushHandle.pInfo, NULL);*/
-    memset(&pHandle->pushHandle.info, 0, sizeof(SRpcHandleInfo));
-    taosWUnLockLatch(&pHandle->pushHandle.lock);
-
-    tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
-            TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum,
-            rsp.reqOffset, rsp.rspOffset);
-
-    // TODO destroy
-    taosArrayDestroy(rsp.blockData);
-    taosArrayDestroy(rsp.blockDataLen);
-  }
-
-  return 0;
-}
-
-int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
-  if (msgType == TDMT_VND_SUBMIT) {
-    if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0;
-
-    if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
-      // TODO handle sma error
-    }
-    void* data = taosMemoryMalloc(msgLen);
-    if (data == NULL) {
-      return -1;
-    }
-    memcpy(data, msg, msgLen);
-
-    tqProcessStreamTrigger(pTq, data);
-  }
-
-  return 0;
-}
-
-int tqCommit(STQ* pTq) {
-  // do nothing
-  return 0;
-}
-
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
-  int64_t     waitTime = pReq->timeout;
+  int64_t     timeout = pReq->timeout;
   int32_t     reqEpoch = pReq->epoch;
   int64_t     fetchOffset;
 
@@ -389,20 +127,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
     rsp.withSchema = false;
-
     rsp.withTag = false;
   } else {
     rsp.withSchema = true;
-    rsp.blockSchema = taosArrayInit(0, sizeof(void*));
-
     rsp.withTag = false;
+    rsp.blockSchema = taosArrayInit(0, sizeof(void*));
   }
 
   while (1) {
     consumerEpoch = atomic_load_32(&pHandle->epoch);
     if (consumerEpoch > reqEpoch) {
-      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
+      tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
-              consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
+             consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
 
@@ -413,27 +149,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
     SWalReadHead* pHead = &pHeadWithCkSum->head;
 
-#if 0
-      // add to pushMgr
-      taosWLockLatch(&pExec->pushHandle.lock);
-
-      pExec->pushHandle.consumerId = consumerId;
-      pExec->pushHandle.epoch = reqEpoch;
-      pExec->pushHandle.reqOffset = rsp.reqOffset;
-      pExec->pushHandle.skipLogNum = rsp.skipLogNum;
-      pExec->pushHandle.handle = pMsg;
-
-      taosWUnLockLatch(&pExec->pushHandle.lock);
-
-      // TODO add timer
-
-      // TODO: the pointer will always be valid?
-      taosHashPut(pTq->pushMgr, &consumerId, sizeof(int64_t), &pExec, sizeof(void*));
-      taosArrayDestroy(rsp.blockData);
-      taosArrayDestroy(rsp.blockDataLen);
-      return 0;
-#endif
-
     tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
             TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
 
@@ -508,24 +223,9 @@ int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
   int32_t code = taosHashRemove(pTq->handles, pReq->subKey, strlen(pReq->subKey));
   ASSERT(code == 0);
 
-  TXN txn;
+  if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) {
-
-  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
     ASSERT(0);
   }
-
-  if (tdbBegin(pTq->pMetaStore, &txn) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbTbDelete(pTq->pExecStore, pReq->subKey, (int)strlen(pReq->subKey), &txn) < 0) {
-    /*ASSERT(0);*/
-  }
-
-  if (tdbCommit(pTq->pMetaStore, &txn) < 0) {
-    ASSERT(0);
-  }
-
   return 0;
 }
 
@@ -583,30 +283,12 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
     atomic_add_fetch_32(&pHandle->epoch, 1);
   }
 
-  if (tqStoreHandle(pTq, req.subKey, pHandle) < 0) {
+  if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
     // TODO
   }
   return 0;
 }
 
-void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
-  const SArray* pRes = (const SArray*)data;
-  SVnode*       pVnode = (SVnode*)vnode;
-
-  ASSERT(pTask->tbSink.pTSchema);
-  SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid,
-                                     pTask->tbSink.stbFullName, pVnode->config.vgId);
-  /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/
-  // build write msg
-  SRpcMsg msg = {
-      .msgType = TDMT_VND_SUBMIT,
-      .pCont = pReq,
-      .contLen = ntohl(pReq->length),
-  };
-
-  ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0);
-}
-
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
   SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
   if (pTask == NULL) {
@@ -697,9 +379,11 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) {
         continue;
       }
 
-      streamDataSubmitRefInc(pSubmit);
+      SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit);
-      SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
+      if (pSubmitClone == NULL) {
-      memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit));
+        atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
+        continue;
+      }
       taosWriteQitem(pTask->inputQ, pSubmitClone);
 
       int8_t execStatus = atomic_load_8(&pTask->status);

source/dnode/vnode/src/tq/tqCommit.c

@@ -14,3 +14,8 @@
  */
 
 #include "tq.h"
+
+int tqCommit(STQ* pTq) {
+  // do nothing
+  return 0;
+}

source/dnode/vnode/src/tq/tqMeta.c

@@ -12,3 +12,163 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include "tdbInt.h"
+#include "tq.h"
+
+static int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pHandle->subKey) < 0) return -1;
+  if (tEncodeI64(pEncoder, pHandle->consumerId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pHandle->epoch) < 0) return -1;
+  if (tEncodeI8(pEncoder, pHandle->execHandle.subType) < 0) return -1;
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+    if (tEncodeCStr(pEncoder, pHandle->execHandle.exec.execCol.qmsg) < 0) return -1;
+  }
+  tEndEncode(pEncoder);
+  return pEncoder->pos;
+}
+
+static int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pHandle->subKey) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pHandle->consumerId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pHandle->epoch) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pHandle->execHandle.subType) < 0) return -1;
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+    if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.exec.execCol.qmsg) < 0) return -1;
+  }
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int tqExecKeyCompare(const void* pKey1, int32_t kLen1, const void* pKey2, int32_t kLen2) {
+  return strcmp(pKey1, pKey2);
+}
+
+int32_t tqMetaOpen(STQ* pTq) {
+  if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaStore) < 0) {
+    ASSERT(0);
+  }
+
+  if (tdbTbOpen("handles", -1, -1, tqExecKeyCompare, pTq->pMetaStore, &pTq->pExecStore) < 0) {
+    ASSERT(0);
+  }
+
+  TXN txn;
+
+  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
+    ASSERT(0);
+  }
+
+  TBC* pCur;
+  if (tdbTbcOpen(pTq->pExecStore, &pCur, &txn) < 0) {
+    ASSERT(0);
+  }
+
+  void* pKey;
+  int   kLen;
+  void* pVal;
+  int   vLen;
+
+  tdbTbcMoveToFirst(pCur);
+  SDecoder decoder;
+
+  while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
+    STqHandle handle;
+    tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
+    tDecodeSTqHandle(&decoder, &handle);
+    handle.pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
+    for (int32_t i = 0; i < 5; i++) {
+      handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
+    }
+    if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+      for (int32_t i = 0; i < 5; i++) {
+        SReadHandle reader = {
+            .reader = handle.execHandle.pExecReader[i],
+            .meta = pTq->pVnode->pMeta,
+            .pMsgCb = &pTq->pVnode->msgCb,
+        };
+        handle.execHandle.exec.execCol.task[i] =
+            qCreateStreamExecTaskInfo(handle.execHandle.exec.execCol.qmsg, &reader);
+        ASSERT(handle.execHandle.exec.execCol.task[i]);
+      }
+    } else {
+      handle.execHandle.exec.execDb.pFilterOutTbUid =
+          taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+    }
+    taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle));
+  }
+
+  if (tdbTxnClose(&txn) < 0) {
+    ASSERT(0);
+  }
+  return 0;
+}
+
+int32_t tqMetaClose(STQ* pTq) {
+  tdbClose(pTq->pMetaStore);
+  return 0;
+}
+
+int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
+  int32_t code;
+  int32_t vlen;
+  tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code);
+  ASSERT(code == 0);
+
+  void* buf = taosMemoryCalloc(1, vlen);
+  if (buf == NULL) {
+    ASSERT(0);
+  }
+
+  SEncoder encoder;
+  tEncoderInit(&encoder, buf, vlen);
+
+  if (tEncodeSTqHandle(&encoder, pHandle) < 0) {
+    ASSERT(0);
+  }
+
+  TXN txn;
+
+  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+    ASSERT(0);
+  }
+
+  if (tdbBegin(pTq->pMetaStore, &txn) < 0) {
+    ASSERT(0);
+  }
+
+  if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, &txn) < 0) {
+    ASSERT(0);
+  }
+
+  if (tdbCommit(pTq->pMetaStore, &txn) < 0) {
+    ASSERT(0);
+  }
+
+  tEncoderClear(&encoder);
+  taosMemoryFree(buf);
+  return 0;
+}
+
+int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
+  TXN txn;
+
+  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+    ASSERT(0);
+  }
+
+  if (tdbBegin(pTq->pMetaStore, &txn) < 0) {
+    ASSERT(0);
+  }
+
+  if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), &txn) < 0) {
+    /*ASSERT(0);*/
+  }
+
+  if (tdbCommit(pTq->pMetaStore, &txn) < 0) {
+    ASSERT(0);
+  }
+
+  return 0;
+}

source/dnode/vnode/src/tq/tqPush.c

@@ -12,3 +12,185 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
+#include "tq.h"
+
+int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) {
+  // 1. guard and set status executing
+  // 2. check processedVer
+  // 2.1. if not missed, get msg from queue
+  // 2.2. if missed, scan wal
+  //
+  // 3. exec, after each success, update processed ver
+  // first run
+  // set exec status closing
+  // second run
+  // set exec status idle
+  //
+  // 4. if get result
+  // 4.1 set exec input status blocked and exec status idle
+  // 4.2 rpc send
+  // 4.3 clear rpc info
+  return 0;
+}
+
+int32_t tqOpenPushHandle(STQ* pTq, STqHandle* pHandle) {
+  memset(&pHandle->pushHandle, 0, sizeof(STqPushHandle));
+  pHandle->pushHandle.inputQ.queue = taosOpenQueue();
+  pHandle->pushHandle.inputQ.qall = taosAllocateQall();
+  if (pHandle->pushHandle.inputQ.queue == NULL || pHandle->pushHandle.inputQ.qall == NULL) {
+    if (pHandle->pushHandle.inputQ.queue) {
+      taosCloseQueue(pHandle->pushHandle.inputQ.queue);
+    }
+    if (pHandle->pushHandle.inputQ.qall) {
+      taosFreeQall(pHandle->pushHandle.inputQ.qall);
+    }
+    return -1;
+  }
+  return 0;
+}
+
+void tqPreparePush(STQ* pTq, STqHandle* pHandle, int64_t reqId, const SRpcHandleInfo* pInfo, int64_t processedVer) {
+  memcpy(&pHandle->pushHandle.rpcInfo, pInfo, sizeof(SRpcHandleInfo));
+  atomic_store_64(&pHandle->pushHandle.reqId, reqId);
+  atomic_store_64(&pHandle->pushHandle.processedVer, processedVer);
+  atomic_store_8(&pHandle->pushHandle.inputStatus, TASK_INPUT_STATUS__NORMAL);
+  // set timeout timer
+}
+
+int32_t tqEnqueue(STqHandle* pHandle, SStreamDataSubmit* pSubmit) {
+  int8_t inputStatus = atomic_load_8(&pHandle->pushHandle.inputStatus);
+  if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
+    SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit);
+    if (pSubmitClone == NULL) {
+      return -1;
+    }
+    taosWriteQitem(pHandle->pushHandle.inputQ.queue, pSubmitClone);
+    return 0;
+  }
+  return -1;
+}
+
+int32_t tqSendExecReq(STQ* pTq, STqHandle* pHandle) {
+  //
+  return 0;
+}
+
+int32_t tqEnqueueAll(STQ* pTq, SSubmitReq* pReq) {
+  void*              pIter = NULL;
+  SStreamDataSubmit* pSubmit = streamDataSubmitNew(pReq);
+  if (pSubmit == NULL) {
+    return -1;
+  }
+
+  while (1) {
+    pIter = taosHashIterate(pTq->handles, pIter);
+    if (pIter == NULL) break;
+    STqHandle* pHandle = (STqHandle*)pIter;
+    if (tqEnqueue(pHandle, pSubmit) < 0) {
+      continue;
+    }
+    int8_t execStatus = atomic_load_8(&pHandle->pushHandle.execStatus);
+    if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
+      tqSendExecReq(pTq, pHandle);
+    }
+  }
+
+  streamDataSubmitRefDec(pSubmit);
+
+  return 0;
+}
+
+int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) {
+  if (msgType != TDMT_VND_SUBMIT) return 0;
+  void*       pIter = NULL;
+  STqHandle*  pHandle = NULL;
+  SSubmitReq* pReq = (SSubmitReq*)msg;
+  int32_t     workerId = 4;
+  int64_t     fetchOffset = ver;
+
+  while (1) {
+    pIter = taosHashIterate(pTq->pushMgr, pIter);
+    if (pIter == NULL) break;
+    pHandle = *(STqHandle**)pIter;
+
+    taosWLockLatch(&pHandle->pushHandle.lock);
+
+    SMqDataBlkRsp rsp = {0};
+    rsp.reqOffset = pHandle->pushHandle.reqOffset;
+    rsp.blockData = taosArrayInit(0, sizeof(void*));
+    rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
+
+    if (msgType == TDMT_VND_SUBMIT) {
+      tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
+    } else {
+      // TODO
+      ASSERT(0);
+    }
+
+    if (rsp.blockNum == 0) {
+      taosWUnLockLatch(&pHandle->pushHandle.lock);
+      continue;
+    }
+
+    ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
+    ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
+
+    rsp.rspOffset = fetchOffset;
+
+    int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
+    void*   buf = rpcMallocCont(tlen);
+    if (buf == NULL) {
+      // todo free
+      return -1;
+    }
+
+    ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
+    ((SMqRspHead*)buf)->epoch = pHandle->pushHandle.epoch;
+    ((SMqRspHead*)buf)->consumerId = pHandle->pushHandle.consumerId;
+
+    void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+    tEncodeSMqDataBlkRsp(&abuf, &rsp);
+
+    SRpcMsg resp = {
+        .info = pHandle->pushHandle.rpcInfo,
+        .pCont = buf,
+        .contLen = tlen,
+        .code = 0,
+    };
+    tmsgSendRsp(&resp);
+
+    memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo));
+    taosWUnLockLatch(&pHandle->pushHandle.lock);
+
+    tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+            TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum,
+            rsp.reqOffset, rsp.rspOffset);
+
+    // TODO destroy
+    taosArrayDestroy(rsp.blockData);
+    taosArrayDestroy(rsp.blockDataLen);
+  }
+
+  return 0;
+}
+
+int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
+  if (msgType == TDMT_VND_SUBMIT) {
+    if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0;
+
+    if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
+      // TODO handle sma error
+    }
+    void* data = taosMemoryMalloc(msgLen);
+    if (data == NULL) {
+      return -1;
+    }
+    memcpy(data, msg, msgLen);
+
+    tqProcessStreamTrigger(pTq, data);
+  }
+
+  return 0;
+}
+

source/dnode/vnode/src/tq/tqRead.c

@@ -298,3 +298,38 @@ int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList)
 
   return 0;
 }
+
+int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
+  void* pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pTq->handles, pIter);
+    if (pIter == NULL) break;
+    STqHandle* pExec = (STqHandle*)pIter;
+    if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+      for (int32_t i = 0; i < 5; i++) {
+        int32_t code = qUpdateQualifiedTableId(pExec->execHandle.exec.execCol.task[i], tbUidList, isAdd);
+        ASSERT(code == 0);
+      }
+    } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) {
+      if (!isAdd) {
+        int32_t sz = taosArrayGetSize(tbUidList);
+        for (int32_t i = 0; i < sz; i++) {
+          int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
+          taosHashPut(pExec->execHandle.exec.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0);
+        }
+      }
+    } else {
+      // tq update id
+    }
+  }
+  while (1) {
+    pIter = taosHashIterate(pTq->pStreamTasks, pIter);
+    if (pIter == NULL) break;
+    SStreamTask* pTask = (SStreamTask*)pIter;
+    if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+      int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
+      ASSERT(code == 0);
+    }
+  }
+  return 0;
+}

source/dnode/vnode/src/tq/tqSink.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tq.h"
+
+void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
+  const SArray* pRes = (const SArray*)data;
+  SVnode*       pVnode = (SVnode*)vnode;
+
+  ASSERT(pTask->tbSink.pTSchema);
+  SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid,
+                                     pTask->tbSink.stbFullName, pVnode->config.vgId);
+  /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/
+  // build write msg
+  SRpcMsg msg = {
+      .msgType = TDMT_VND_SUBMIT,
+      .pCont = pReq,
+      .contLen = ntohl(pReq->length),
+  };
+
+  ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0);
+}

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -238,7 +238,7 @@ static void tsdbStartCommit(STsdb *pRepo) {
 
 static void tsdbEndCommit(STsdb *pTsdb, int eno) {
   tsdbEndFSTxn(pTsdb);
-  tsdbMemTableDestroy(pTsdb, pTsdb->imem);
+  tsdbMemTableDestroy(pTsdb->imem);
   pTsdb->imem = NULL;
   tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
 }

source/dnode/vnode/src/tsdb/tsdbDelete.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -60,7 +60,7 @@ int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) {
   return 0;
 }
 
-void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) {
+void tsdbMemTableDestroy(STsdbMemTable *pMemTable) {
   if (pMemTable) {
     taosHashCleanup(pMemTable->pHashIdx);
     SSkipListIterator *pIter = tSkipListCreateIter(pMemTable->pSlIdx);
@@ -142,69 +142,6 @@ int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter
       } else {
         fKey = tdGetKey(filterKeys[filterIter]);
       }
-#if 0
-    } else if (fKey > rowKey) {
-      if (isRowDel) {
-        pMergeInfo->rowsDeleteFailed++;
-      } else {
-        if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
-        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
-
-        pMergeInfo->rowsInserted++;
-        pMergeInfo->nOperations++;
-        pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
-        pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
-        tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
-      }
-
-      tSkipListIterNext(pIter);
-      row = tsdbNextIterRow(pIter);
-      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
-        rowKey = INT64_MAX;
-        isRowDel = false;
-      } else {
-        rowKey = TD_ROW_KEY(row);
-        isRowDel = TD_ROW_IS_DELETED(row);
-      }
-    } else {
-      if (isRowDel) {
-        ASSERT(!keepDup);
-        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
-        pMergeInfo->rowsDeleteSucceed++;
-        pMergeInfo->nOperations++;
-        tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
-      } else {
-        if (keepDup) {
-          if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
-          pMergeInfo->rowsUpdated++;
-          pMergeInfo->nOperations++;
-          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
-          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
-          tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
-        } else {
-          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
-          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
-        }
-      }
-
-      tSkipListIterNext(pIter);
-      row = tsdbNextIterRow(pIter);
-      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
-        rowKey = INT64_MAX;
-        isRowDel = false;
-      } else {
-        rowKey = TD_ROW_KEY(row);
-        isRowDel = TD_ROW_IS_DELETED(row);
-      }
-
-      filterIter++;
-      if (filterIter >= nFilterKeys) {
-        fKey = INT64_MAX;
-      } else {
-        fKey = tdGetKey(filterKeys[filterIter]);
-      }
-    }
-#endif
 #if 1
     } else if (fKey > rowKey) {
       if (isRowDel) {
@@ -321,7 +258,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo
     terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     return -1;
   }
-  if(pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name);
+  if (pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name);
 
   if (mr.me.type == TSDB_NORMAL_TABLE) {
     sverNew = mr.me.ntbEntry.schemaRow.version;

source/dnode/vnode/src/tsdb/tsdbMemTable2.c

@@ -15,52 +15,308 @@
 
 #include "tsdb.h"
 
-typedef struct SMemTable          SMemTable;
+typedef struct SMemData         SMemData;
-typedef struct SMemData           SMemData;
+typedef struct SMemSkipList     SMemSkipList;
-typedef struct SMemSkipList       SMemSkipList;
+typedef struct SMemSkipListNode SMemSkipListNode;
-typedef struct SMemSkipListNode   SMemSkipListNode;
-typedef struct SMemSkipListCurosr SMemSkipListCurosr;
-
-#define SL_MAX_LEVEL 5
-
-struct SMemTable {
-  STsdb              *pTsdb;
-  TSKEY               minKey;
-  TSKEY               maxKey;
-  int64_t             minVer;
-  int64_t             maxVer;
-  int64_t             nRows;
-  int32_t             nHash;
-  int32_t             nBucket;
-  SMemData          **pBuckets;
-  SMemSkipListCurosr *pSlc;
-};
 
 struct SMemSkipListNode {
   int8_t            level;
-  SMemSkipListNode *forwards[1];  // Windows does not allow 0
+  SMemSkipListNode *forwards[0];
 };
 
 struct SMemSkipList {
-  uint32_t         seed;
+  uint32_t          seed;
-  int8_t           maxLevel;
+  int32_t           size;
-  int8_t           level;
+  int8_t            maxLevel;
-  int32_t          size;
+  int8_t            level;
-  SMemSkipListNode pHead[1];  // Windows does not allow 0
+  SMemSkipListNode *pHead;
+  SMemSkipListNode *pTail;
 };
 
 struct SMemData {
-  SMemData    *pHashNext;
   tb_uid_t     suid;
   tb_uid_t     uid;
-  TSKEY        minKey;
+  TSDBKEY      minKey;
-  TSKEY        maxKey;
+  TSDBKEY      maxKey;
-  int64_t      minVer;
+  SDelOp      *delOpHead;
-  int64_t      maxVer;
+  SDelOp      *delOpTail;
-  int64_t      nRows;
   SMemSkipList sl;
 };
 
+struct SMemTable {
+  STsdb  *pTsdb;
+  int32_t nRef;
+  TSDBKEY minKey;
+  TSDBKEY maxKey;
+  int64_t nRows;
+  SArray *pArray;  // SArray<SMemData>
+};
+
+#define SL_NODE_SIZE(l)        (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
+#define SL_NODE_HALF_SIZE(l)   (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l))
+#define SL_NODE_FORWARD(n, l)  ((n)->forwards[l])
+#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
+#define SL_NODE_DATA(n)        (&SL_NODE_BACKWARD(n, (n)->level))
+
+#define SL_HEAD_FORWARD(sl, l)  SL_NODE_FORWARD((sl)->pHead, l)
+#define SL_TAIL_BACKWARD(sl, l) SL_NODE_FORWARD((sl)->pTail, l)
+
+static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData);
+static int     memDataPCmprFn(const void *p1, const void *p2);
+static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
+static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
+static int8_t  tsdbMemSkipListRandLevel(SMemSkipList *pSl);
+
+// SMemTable ==============================================
+int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) {
+  int32_t    code = 0;
+  SMemTable *pMemTable = NULL;
+
+  pMemTable = (SMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
+  if (pMemTable == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pMemTable->pTsdb = pTsdb;
+  pMemTable->nRef = 1;
+  pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
+  pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
+  pMemTable->nRows = 0;
+  pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *));
+  if (pMemTable->pArray == NULL) {
+    taosMemoryFree(pMemTable);
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  *ppMemTable = pMemTable;
+  return code;
+
+_err:
+  *ppMemTable = NULL;
+  return code;
+}
+
+void tsdbMemTableDestroy2(SMemTable *pMemTable) {
+  taosArrayDestroyEx(pMemTable->pArray, NULL /*TODO*/);
+  taosMemoryFree(pMemTable);
+}
+
+int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) {
+  int32_t    code = 0;
+  SMemTable *pMemTable = (SMemTable *)pTsdb->mem;  // TODO
+  SMemData  *pMemData;
+  TSDBROW    row = {.version = version};
+
+  ASSERT(pMemTable);
+
+  {
+    // check if table exists (todo)
+  }
+
+  code = tsdbGetOrCreateMemData(pMemTable, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData);
+  if (code) {
+    tsdbError("vgId:%d failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+    goto _err;
+  }
+
+  // do insert
+  int32_t           nt;
+  uint8_t          *pt;
+  int32_t           n = 0;
+  uint8_t          *p = pSubmitBlk->pData;
+  SVBufPool        *pPool = pTsdb->pVnode->inUse;
+  int8_t            level;
+  SMemSkipListNode *pNode;
+  while (n < pSubmitBlk->nData) {
+    nt = tGetTSRow(p + n, &row.tsRow);
+    n += nt;
+
+    ASSERT(n <= pSubmitBlk->nData);
+
+    // build the node
+    level = tsdbMemSkipListRandLevel(&pMemData->sl);
+    pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + nt + sizeof(version));
+    if (pNode == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+    pNode->level = level;
+    tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), &row);
+
+    // put the node (todo)
+
+    // set info
+    if (tsdbKeyCmprFn(&row, &pMemData->minKey) < 0) pMemData->minKey = *(TSDBKEY *)&row;
+    if (tsdbKeyCmprFn(&row, &pMemData->maxKey) > 0) pMemData->maxKey = *(TSDBKEY *)&row;
+  }
+
+  if (tsdbKeyCmprFn(&pMemTable->minKey, &pMemData->minKey) < 0) pMemTable->minKey = pMemData->minKey;
+  if (tsdbKeyCmprFn(&pMemTable->maxKey, &pMemData->maxKey) > 0) pMemTable->maxKey = pMemData->maxKey;
+
+  return code;
+
+_err:
+  return code;
+}
+
+int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
+  int32_t    code = 0;
+  SMemTable *pMemTable = (SMemTable *)pTsdb->mem;  // TODO
+  SMemData  *pMemData;
+  SVBufPool *pPool = pTsdb->pVnode->inUse;
+
+  ASSERT(pMemTable);
+
+  {
+    // check if table exists (todo)
+  }
+
+  code = tsdbGetOrCreateMemData(pMemTable, suid, uid, &pMemData);
+  if (code) {
+    goto _err;
+  }
+
+  // do delete
+  SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp));
+  if (pDelOp == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pDelOp->version = version;
+  pDelOp->sKey = sKey;
+  pDelOp->eKey = eKey;
+  pDelOp->pNext = NULL;
+  if (pMemData->delOpHead == NULL) {
+    ASSERT(pMemData->delOpTail == NULL);
+    pMemData->delOpHead = pMemData->delOpTail = pDelOp;
+  } else {
+    pMemData->delOpTail->pNext = pDelOp;
+    pMemData->delOpTail = pDelOp;
+  }
+
+  {
+    // update the state of pMemTable, pMemData, last and lastrow (todo)
+  }
+
+  tsdbDebug("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
+            " since %s",
+            TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
+  return code;
+
+_err:
+  tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
+            " since %s",
+            TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) {
+  int32_t    code = 0;
+  int32_t    idx = 0;
+  SMemData  *pMemDataT = &(SMemData){.suid = suid, .uid = uid};
+  SMemData  *pMemData = NULL;
+  SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
+  int8_t     maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
+
+  // get
+  idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE);
+  if (idx >= 0) {
+    pMemData = (SMemData *)taosArrayGet(pMemTable->pArray, idx);
+    if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit;
+  }
+
+  // create
+  pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2);
+  if (pMemData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pMemData->suid = suid;
+  pMemData->uid = uid;
+  pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
+  pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
+  pMemData->delOpHead = pMemData->delOpTail = NULL;
+  pMemData->sl.seed = taosRand();
+  pMemData->sl.size = 0;
+  pMemData->sl.maxLevel = maxLevel;
+  pMemData->sl.level = 0;
+  pMemData->sl.pHead = (SMemSkipListNode *)&pMemData[1];
+  pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_HALF_SIZE(maxLevel));
+
+  for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
+    SL_HEAD_FORWARD(&pMemData->sl, iLevel) = pMemData->sl.pTail;
+    SL_TAIL_BACKWARD(&pMemData->sl, iLevel) = pMemData->sl.pHead;
+  }
+
+  if (idx < 0) idx = 0;
+  if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+_exit:
+  *ppMemData = pMemData;
+  return code;
+
+_err:
+  *ppMemData = NULL;
+  return code;
+}
+
+static int memDataPCmprFn(const void *p1, const void *p2) {
+  SMemData *pMemData1 = *(SMemData **)p1;
+  SMemData *pMemData2 = *(SMemData **)p2;
+
+  if (pMemData1->suid < pMemData2->suid) {
+    return -1;
+  } else if (pMemData1->suid > pMemData2->suid) {
+    return 1;
+  }
+
+  if (pMemData1->uid < pMemData2->uid) {
+    return -1;
+  } else if (pMemData1->uid > pMemData2->uid) {
+    return 1;
+  }
+
+  return 0;
+}
+
+static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) {
+  int32_t n = 0;
+
+  n += tPutI64(p ? p + n : p, pRow->version);
+  n += tPutTSRow(p ? p + n : p, &pRow->tsRow);
+
+  return n;
+}
+
+static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) {
+  int32_t n = 0;
+
+  n += tGetI64(p + n, &pRow->version);
+  n += tGetTSRow(p + n, &pRow->tsRow);
+
+  return n;
+}
+
+static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
+  int8_t         level = 1;
+  int8_t         tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
+  const uint32_t factor = 4;
+
+  while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
+    level++;
+  }
+
+  return level;
+}
+
+#if 0  //====================================================================================
+
+#define SL_MAX_LEVEL 5
+
 struct SMemSkipListCurosr {
   SMemSkipList     *pSl;
   SMemSkipListNode *pNodes[SL_MAX_LEVEL];
@@ -74,12 +330,6 @@ typedef struct {
 
 #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET))
 
-#define SL_NODE_SIZE(l)        (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
-#define SL_NODE_HALF_SIZE(l)   (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l))
-#define SL_NODE_FORWARD(n, l)  ((n)->forwards[l])
-#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
-#define SL_NODE_DATA(n)        (&SL_NODE_BACKWARD(n, (n)->level))
-
 #define SL_HEAD_NODE(sl)            ((sl)->pHead)
 #define SL_TAIL_NODE(sl)            ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel))
 #define SL_HEAD_NODE_FORWARD(n, l)  SL_NODE_FORWARD(n, l)
@@ -99,50 +349,7 @@ static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc);
 static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc);
 static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow);
 
-// SMemTable
+// SMemTable ========================
-int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
-  SMemTable *pMemTb = NULL;
-
-  pMemTb = taosMemoryCalloc(1, sizeof(*pMemTb));
-  if (pMemTb == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pMemTb->pTsdb = pTsdb;
-  pMemTb->minKey = TSKEY_MAX;
-  pMemTb->maxKey = TSKEY_MIN;
-  pMemTb->minVer = -1;
-  pMemTb->maxVer = -1;
-  pMemTb->nRows = 0;
-  pMemTb->nHash = 0;
-  pMemTb->nBucket = 1024;
-  pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets));
-  if (pMemTb->pBuckets == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFree(pMemTb);
-    return -1;
-  }
-  if (tsdbMemSkipListCursorCreate(pTsdb->pVnode->config.tsdbCfg.slLevel, &pMemTb->pSlc) < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFree(pMemTb->pBuckets);
-    taosMemoryFree(pMemTb);
-  }
-
-  *ppMemTb = pMemTb;
-  return 0;
-}
-
-int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) {
-  if (pMemTb) {
-    // loop to destroy the contents (todo)
-    tsdbMemSkipListCursorDestroy(pMemTb->pSlc);
-    taosMemoryFree(pMemTb->pBuckets);
-    taosMemoryFree(pMemTb);
-  }
-  return 0;
-}
-
 int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) {
   SMemData  *pMemData;
   STsdb     *pTsdb = pMemTb->pTsdb;
@@ -253,18 +460,6 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p
   return 0;
 }
 
-static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
-  int8_t         level = 1;
-  int8_t         tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
-  const uint32_t factor = 4;
-
-  while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
-    level++;
-  }
-
-  return level;
-}
-
 static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) {
   if (tEncodeI64(pEncoder, pRow->version) < 0) return -1;
   if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1;
@@ -377,4 +572,5 @@ static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipLis
   }
 
   return pNode;
-}
+}
+#endif

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -333,7 +333,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData
 }
 
 static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) {
-  if (vnodeIsRollup(pVnode)) {
+  if (VND_IS_RSMA(pVnode)) {
     int     level = 0;
     int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
 

source/dnode/vnode/src/tsdb/tsdbTDBImpl.c

@@ -1,128 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define ALLOW_FORBID_FUNC
-
-#include "tsdb.h"
-
-int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) {
-  int ret = 0;
-
-  if (path == NULL) return -1;
-
-  ret = tdbOpen(path, 4096, 256, ppEnv);  // use as param
-
-  if (ret != 0) {
-    tsdbError("Failed to create tsdb db env, ret = %d", ret);
-    return -1;
-  }
-
-  return 0;
-}
-
-int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); }
-
-static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
-  const SSmaKey *pKey1 = (const SSmaKey *)arg1;
-  const SSmaKey *pKey2 = (const SSmaKey *)arg2;
-
-  ASSERT(len1 == len2 && len1 == sizeof(SSmaKey));
-
-  if (pKey1->skey < pKey2->skey) {
-    return -1;
-  } else if (pKey1->skey > pKey2->skey) {
-    return 1;
-  }
-  if (pKey1->groupId < pKey2->groupId) {
-    return -1;
-  } else if (pKey1->groupId > pKey2->groupId) {
-    return 1;
-  }
-
-  return 0;
-}
-
-static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) {
-  int           ret;
-  tdb_cmpr_fn_t compFunc;
-
-  // Create a database
-  compFunc = tsdbSmaKeyCmpr;
-  ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
-
-  return 0;
-}
-
-static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); }
-
-int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) {
-  // TEnv is shared by a group of SDBFile
-  if (!pEnv || !pDBF) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    return -1;
-  }
-
-  // Open DBF
-  if (tsdbOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
-    terrno = TSDB_CODE_TDB_INIT_FAILED;
-    tsdbCloseDBDb(pDBF->pDB);
-    return -1;
-  }
-
-  return 0;
-}
-
-int32_t tsdbCloseDBF(SDBFile *pDBF) {
-  int32_t ret = 0;
-  if (pDBF->pDB) {
-    ret = tsdbCloseDBDb(pDBF->pDB);
-    pDBF->pDB = NULL;
-  }
-  taosMemoryFreeClear(pDBF->path);
-  return ret;
-}
-
-int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
-  int32_t ret;
-
-  ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
-  if (ret < 0) {
-    tsdbError("Failed to create insert sma data into db, ret = %d", ret);
-    return -1;
-  }
-
-  return 0;
-}
-
-void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) {
-  void *pVal = NULL;
-  int   ret;
-
-  ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
-
-  if (ret < 0) {
-    tsdbError("Failed to get sma data from db, ret = %d", ret);
-    return NULL;
-  }
-
-  ASSERT(*valLen >= 0);
-
-  // TODO: lock?
-  // TODO: Would the key/value be destoryed during return the data?
-  // TODO: How about the key is updated while value length is changed? The original value buffer would be freed
-  // automatically?
-
-  return pVal;
-}

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -57,6 +57,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
@@ -133,6 +134,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if(code < 0) return -1;
   tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code);
   if(code < 0) return -1;
+  tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code);
+  if(code < 0) return -1;
   tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code);
   if(code < 0) return -1;
   tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code);

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -230,7 +230,7 @@ int vnodeCommit(SVnode *pVnode) {
     return -1;
   }
 
-  if(vnodeIsRollup(pVnode)) {
+  if (VND_IS_RSMA(pVnode)) {
     if (tsdbCommit(VND_RSMA0(pVnode)) < 0) {
       ASSERT(0);
       return -1;
@@ -250,7 +250,6 @@ int vnodeCommit(SVnode *pVnode) {
     }
   }
 
-  
   if (tqCommit(pVnode->pTq) < 0) {
     ASSERT(0);
     return -1;

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -97,7 +97,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   }
 
   // open tsdb
-  if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL) < 0) {
+  if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL) < 0) {
     vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
@@ -189,4 +189,4 @@ void vnodeStop(SVnode *pVnode) {}
 
 int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
 
-void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }
+void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -22,7 +22,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
 static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
+static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 
 int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
   SDecoder dc = {0};
@@ -88,6 +88,9 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
       }
 
     } break;
+    case TDMT_VND_ALTER_REPLICA: {
+      vnodeSyncAlter(pVnode, pMsg);
+    } break;
     default:
       break;
   }
@@ -154,7 +157,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
       }
     } break;
-    case TDMT_VND_ALTER_VNODE:
+    case TDMT_VND_ALTER_CONFIG:
       break;
     default:
       ASSERT(0);
@@ -776,28 +779,30 @@ _exit:
 
   // TODO: the partial success scenario and the error case
   // TODO: refactor
-  if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) &&
+  if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
-      (pRsp->code == TSDB_CODE_SUCCESS)) {
     tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
   }
 
   return 0;
 }
 
-static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
+static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVCreateTSmaReq req = {0};
   SDecoder        coder;
 
-  pRsp->msgType = TDMT_VND_CREATE_SMA_RSP;
+  if (pRsp) {
-  pRsp->code = TSDB_CODE_SUCCESS;
+    pRsp->msgType = TDMT_VND_CREATE_SMA_RSP;
-  pRsp->pCont = NULL;
+    pRsp->code = TSDB_CODE_SUCCESS;
-  pRsp->contLen = 0;
+    pRsp->pCont = NULL;
+    pRsp->contLen = 0;
+  }
 
   // decode and process req
   tDecoderInit(&coder, pReq, len);
 
   if (tDecodeSVCreateTSmaReq(&coder, &req) < 0) {
-    pRsp->code = terrno;
+    terrno = TSDB_CODE_MSG_DECODE_ERROR;
+    if (pRsp) pRsp->code = terrno;
     goto _err;
   }
 
@@ -805,18 +810,30 @@ static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq
   req.timezoneInt = tsTimezone;
 
   if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) {
-    pRsp->code = terrno;
+    if (pRsp) pRsp->code = terrno;
     goto _err;
   }
 
   tDecoderClear(&coder);
-  vDebug("vgId:%d success to create tsma %s:%" PRIi64 " for table %" PRIi64, TD_VID(pVnode), req.indexName,
+  vDebug("vgId:%d success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode),
-         req.indexUid, req.tableUid);
+         req.indexName, req.indexUid, version, req.tableUid);
   return 0;
 
 _err:
   tDecoderClear(&coder);
-  vError("vgId:%d failed to create tsma %s:%" PRIi64 " for table %" PRIi64 " since %s", TD_VID(pVnode), req.indexName,
+  vError("vgId:%d failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s",
-         req.indexUid, req.tableUid, terrstr(terrno));
+         TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr(terrno));
   return -1;
 }
+
+/**
+ * @brief specific for smaDstVnode
+ *
+ * @param pVnode
+ * @param pCont
+ * @param contLen
+ * @return int32_t
+ */
+int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) {
+  return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL);
+}

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -27,6 +27,7 @@ static int32_t   vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
 int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
   SSyncInfo syncInfo = {
       .vgId = pVnode->config.vgId,
+      .isStandBy = pVnode->config.standby,
       .syncCfg = pVnode->config.syncCfg,
       .pWal = pVnode->pWal,
       .msgcb = NULL,
@@ -49,28 +50,72 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
   return 0;
 }
 
+void vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) {
+  SAlterVnodeReq req = {0};
+  if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    vError("vgId:%d, failed to alter replica since %s", TD_VID(pVnode), terrstr());
+    SRpcMsg rsp = {.info = pMsg->info, .code = terrno};
+    tmsgSendRsp(&rsp);
+  }
+
+  vInfo("vgId:%d, start to alter vnode replica to %d", TD_VID(pVnode), req.replica);
+  SSyncCfg cfg = {.replicaNum = req.replica, .myIndex = req.selfIndex};
+  for (int32_t r = 0; r < req.replica; ++r) {
+    SNodeInfo *pNode = &cfg.nodeInfo[r];
+    tstrncpy(pNode->nodeFqdn, req.replicas[r].fqdn, sizeof(pNode->nodeFqdn));
+    pNode->nodePort = req.replicas[r].port;
+    vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort);
+  }
+
+  if (syncReconfig(pVnode->sync, &cfg) != 0) {
+    vError("vgId:%d, failed to propose sync reconfig since %s", TD_VID(pVnode), terrstr());
+    SRpcMsg rsp = {.info = pMsg->info, .code = terrno};
+    tmsgSendRsp(&rsp);
+  }
+}
+
 void vnodeSyncStart(SVnode *pVnode) {
   syncSetMsgCb(pVnode->sync, &pVnode->msgCb);
-  syncStart(pVnode->sync);
+  if (pVnode->config.standby) {
+    syncStartStandBy(pVnode->sync);
+  } else {
+    syncStart(pVnode->sync);
+  }
 }
 
 void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); }
 
-int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { 
+int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
   int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
   if (code != 0) {
     rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
   }
   return code;
 }
 
-int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return tmsgSendReq(pEpSet, pMsg); }
+int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
+  int32_t code = tmsgSendReq(pEpSet, pMsg);
+  if (code != 0) {
+    rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
+  }
+  return code;
+}
 
 int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
   vnodeGetSnapshot(pFsm->data, pSnapshot);
   return 0;
 }
 
+void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
+  SVnode *pVnode = pFsm->data;
+  vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode));
+
+  // todo rpc response here
+}
+
 void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   SyncIndex beginIndex = SYNC_INDEX_INVALID;
   if (pFsm->FpGetSnapshot != NULL) {
@@ -87,20 +132,12 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
         pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
     syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 
-    SVnode       *pVnode = (SVnode *)(pFsm->data);
+    SVnode       *pVnode = pFsm->data;
     SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
     SRpcMsg       applyMsg;
     syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
     syncApplyMsgDestroy(pSyncApplyMsg);
 
-    /*
-        SRpcMsg applyMsg;
-        applyMsg = *pMsg;
-        applyMsg.pCont = rpcMallocCont(applyMsg.contLen);
-        assert(applyMsg.contLen == pMsg->contLen);
-        memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen);
-    */
-
     // recover handle for response
     SRpcMsg saveRpcMsg;
     int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
@@ -142,14 +179,13 @@ void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta
 
 SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
-  memset(pFsm, 0, sizeof(*pFsm));
   pFsm->data = pVnode;
   pFsm->FpCommitCb = vnodeSyncCommitMsg;
   pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg;
   pFsm->FpRollBackCb = vnodeSyncRollBackMsg;
   pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
   pFsm->FpRestoreFinishCb = NULL;
-  pFsm->FpReConfigCb = NULL;
+  pFsm->FpReConfigCb = vnodeSyncReconfig;
 
   return pFsm;
 }

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -368,7 +368,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   SDiskCfg pDisks = {0};
   pDisks.level = 0;
   pDisks.primary = 1;
-  strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN);
+  strncpy(pDisks.dir, TD_DATA_DIR_PATH, TSDB_FILENAME_LEN);
   int32_t numOfDisks = 1;
   pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);
   EXPECT_NE(pTsdb->pTfs, nullptr);

source/libs/catalog/test/catalogTests.cpp

@@ -137,7 +137,7 @@ void ctgTestInitLogFile() {
 
   tsAsyncLog = 0;
   qDebugFlag = 159;
-  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsLogDir, TD_LOG_DIR_PATH);
 
   ctgdEnableDebug("api");
   ctgdEnableDebug("meta");

source/libs/executor/src/scanoperator.c

@@ -313,29 +313,23 @@ void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_
     if (fmIsScanPseudoColumnFunc(functionId)) {
       setTbNameColData(pHandle->meta, pBlock, pColInfoData, functionId);
     } else {  // these are tags
-      const char* p = NULL;
+      STagVal tagVal = {0};
-      if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
+      tagVal.cid = pExpr->base.pParam[0].pCol->colId;
-        const uint8_t* tmp = mr.me.ctbEntry.pTags;
+      const char *p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal);
 
-        char* data = taosMemoryCalloc(kvRowLen(tmp) + 1, 1);
+      char *data = NULL;
-        if (data == NULL) {
+      if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL){
-          metaReaderClear(&mr);
+        data = tTagValToData((const STagVal *)p, false);
-          qError("doTagScan calloc error:%d", kvRowLen(tmp) + 1);
+      }else {
-          return;
+        data = (char*)p;
-        }
-
-        *data = TSDB_DATA_TYPE_JSON;
-        memcpy(data + 1, tmp, kvRowLen(tmp));
-        p = data;
-      } else {
-        p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId);
       }
 
       for (int32_t i = 0; i < pBlock->info.rows; ++i) {
-        colDataAppend(pColInfoData, i, p, (p == NULL));
+        colDataAppend(pColInfoData, i, data, (data == NULL));
       }
-      if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
+      if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL &&
-        taosMemoryFree((void*)p);
+          IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){
+        taosMemoryFree(data);
       }
     }
   }
@@ -1589,22 +1583,21 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
         STR_TO_VARSTR(str, mr.me.name);
         colDataAppend(pDst, count, str, false);
       } else {  // it is a tag value
-        if (pDst->info.type == TSDB_DATA_TYPE_JSON) {
+        STagVal val = {0};
-          const uint8_t* tmp = mr.me.ctbEntry.pTags;
+        val.cid = pExprInfo[j].base.pParam[0].pCol->colId;
-          // TODO opt perf by realloc memory
+        const char* p = metaGetTableTagVal(&mr.me, pDst->info.type, &val);
-          char* data = taosMemoryCalloc(kvRowLen(tmp) + 1, 1);
-          if (data == NULL) {
-            qError("%s failed to malloc memory, size:%d", GET_TASKID(pTaskInfo), kvRowLen(tmp) + 1);
-            longjmp(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
-          }
 
-          *data = TSDB_DATA_TYPE_JSON;
+        char *data = NULL;
-          memcpy(data + 1, tmp, kvRowLen(tmp));
+        if(pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL){
-          colDataAppend(pDst, count, data, false);
+          data = tTagValToData((const STagVal *)p, false);
+        }else {
+          data = (char*)p;
+        }
+        colDataAppend(pDst, count, data, (data == NULL));
+
+        if(pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL
+            && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data != NULL){
           taosMemoryFree(data);
-        } else {
-          const char* p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId);
-          colDataAppend(pDst, count, p, (p == NULL));
         }
       }
     }
@@ -1634,8 +1627,8 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
 }
 
 SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput,
-                                         SSDataBlock* pResBlock, SArray* pColMatchInfo,
+                                         SSDataBlock* pResBlock, SArray* pColMatchInfo, STableListInfo* pTableListInfo,
-                                         STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
+                                         SExecTaskInfo* pTaskInfo) {
   STagScanInfo*  pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
   SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {

source/libs/parser/inc/parUtil.h

@@ -58,7 +58,7 @@ int32_t       getNumOfColumns(const STableMeta* pTableMeta);
 int32_t       getNumOfTags(const STableMeta* pTableMeta);
 STableComInfo getTableInfo(const STableMeta* pTableMeta);
 STableMeta*   tableMetaDup(const STableMeta* pTableMeta);
-int32_t       parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId);
+int32_t       parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf);
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
 

source/libs/parser/inc/sql.y

@@ -15,11 +15,15 @@
 #include <assert.h>
 #include <stdbool.h>
 
+#define ALLOW_FORBID_FUNC
+
 #include "functionMgt.h"
 #include "nodes.h"
 #include "parToken.h"
 #include "ttokendef.h"
 #include "parAst.h"
+
+#define YYSTACKDEPTH 0
 }
 
 %syntax_error {

source/libs/parser/src/parInsert.c

@@ -54,7 +54,6 @@ typedef struct SInsertParseContext {
   SMsgBuf            msg;                 // input
   STableMeta*        pTableMeta;          // each table
   SParsedDataColInfo tags;                // each table
-  SKVRowBuilder      tagsBuilder;         // each table
   SVCreateTbReq      createTblReq;        // each table
   SHashObj*          pVgroupsHashObj;     // global
   SHashObj*          pTableBlockHashObj;  // global
@@ -73,9 +72,10 @@ static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
 static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
 
 typedef struct SKvParam {
-  SKVRowBuilder* builder;
+  int16_t  pos;
-  SSchema*       schema;
+  SArray*  pTagVals;
-  char           buf[TSDB_MAX_TAGS_LEN];
+  SSchema* schema;
+  char     buf[TSDB_MAX_TAGS_LEN];
 } SKvParam;
 
 typedef struct SMemParam {
@@ -446,7 +446,7 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) {
+static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) {
   if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER &&
        pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL &&
        pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT &&
@@ -492,7 +492,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
   uint64_t uv;
   char*    endptr = NULL;
 
-  int32_t code = checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf);
+  int32_t code = checkAndTrimValue(pToken, tmpTokenBuf, pMsgBuf);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -641,14 +641,12 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
     case TSDB_DATA_TYPE_NCHAR: {
       return func(pMsgBuf, pToken->z, pToken->n, param);
     }
-
     case TSDB_DATA_TYPE_JSON: {
       if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
         return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z);
       }
       return func(pMsgBuf, pToken->z, pToken->n, param);
     }
-
     case TSDB_DATA_TYPE_TIMESTAMP: {
       int64_t tmpVal;
       if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
@@ -773,99 +771,282 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) {
+static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid) {
-  SKvParam* pa = (SKvParam*)param;
-
-  int8_t  type = pa->schema->type;
-  int16_t colId = pa->schema->colId;
-
-  if (TSDB_DATA_TYPE_JSON == type) {
-    return parseJsontoTagData(value, pa->builder, pMsgBuf, colId);
-  }
-
-  if (value == NULL) {  // it is a null data
-    // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset,
-    // pa->colIdx);
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (TSDB_DATA_TYPE_BINARY == type) {
-    STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
-    tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
-  } else if (TSDB_DATA_TYPE_NCHAR == type) {
-    // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
-    int32_t output = 0;
-    if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
-      if (errno == E2BIG) {
-        return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name);
-      }
-
-      char buf[512] = {0};
-      snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
-      return buildSyntaxErrMsg(pMsgBuf, buf, value);
-    }
-
-    varDataSetLen(pa->buf, output);
-    tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
-  } else {
-    tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]);
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, SKVRow row, int64_t suid) {
   pTbReq->type = TD_CHILD_TABLE;
   pTbReq->name = strdup(tname);
   pTbReq->ctb.suid = suid;
-  pTbReq->ctb.pTag = row;
+  pTbReq->ctb.pTag = (uint8_t*)pTag;
+
+  return;
+}
+
+static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val,
+                             SMsgBuf* pMsgBuf) {
+  int64_t  iv;
+  uint64_t uv;
+  char*    endptr = NULL;
+
+  if (isNullStr(pToken)) {
+    if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
+      return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z);
+    }
+
+    return TSDB_CODE_SUCCESS;
+  }
+
+  val->cid = pSchema->colId;
+  val->type = pSchema->type;
+
+  switch (pSchema->type) {
+    case TSDB_DATA_TYPE_BOOL: {
+      if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) {
+        if (strncmp(pToken->z, "true", pToken->n) == 0) {
+          *(int8_t*)(&val->i64) = TRUE_VALUE;
+        } else if (strncmp(pToken->z, "false", pToken->n) == 0) {
+          *(int8_t*)(&val->i64) = FALSE_VALUE;
+        } else {
+          return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
+        }
+      } else if (pToken->type == TK_NK_INTEGER) {
+        *(int8_t*)(&val->i64) = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE);
+      } else if (pToken->type == TK_NK_FLOAT) {
+        *(int8_t*)(&val->i64) = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE);
+      } else {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_TINYINT: {
+      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z);
+      } else if (!IS_VALID_TINYINT(iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z);
+      }
+
+      *(int8_t*)(&val->i64) = iv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_UTINYINT: {
+      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z);
+      } else if (!IS_VALID_UTINYINT(uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z);
+      }
+      *(uint8_t*)(&val->i64) = uv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_SMALLINT: {
+      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z);
+      } else if (!IS_VALID_SMALLINT(iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z);
+      }
+      *(int16_t*)(&val->i64) = iv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_USMALLINT: {
+      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z);
+      } else if (!IS_VALID_USMALLINT(uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z);
+      }
+      *(uint16_t*)(&val->i64) = uv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_INT: {
+      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z);
+      } else if (!IS_VALID_INT(iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z);
+      }
+      *(int32_t*)(&val->i64) = iv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_UINT: {
+      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z);
+      } else if (!IS_VALID_UINT(uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z);
+      }
+      *(uint32_t*)(&val->i64) = uv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_BIGINT: {
+      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
+      } else if (!IS_VALID_BIGINT(iv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z);
+      }
+
+      val->i64 = iv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_UBIGINT: {
+      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z);
+      } else if (!IS_VALID_UBIGINT(uv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z);
+      }
+      *(uint64_t*)(&val->i64) = uv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_FLOAT: {
+      double dv;
+      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+        return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
+      }
+      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
+          isnan(dv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
+      }
+      *(float*)(&val->i64) = dv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double dv;
+      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+        return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
+      }
+      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
+        return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
+      }
+
+      *(double*)(&val->i64) = dv;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_BINARY: {
+      // Too long values will raise the invalid sql error message
+      if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {
+        return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
+      }
+      val->pData = pToken->z;
+      val->nData = pToken->n;
+      break;
+    }
+
+    case TSDB_DATA_TYPE_NCHAR: {
+      int32_t output = 0;
+      void*   p = taosMemoryCalloc(1, pToken->n * TSDB_NCHAR_SIZE);
+      if (p == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
+        if (errno == E2BIG) {
+          taosMemoryFree(p);
+          return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
+        }
+        char buf[512] = {0};
+        snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
+        taosMemoryFree(p);
+        return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z);
+      }
+      val->pData = p;
+      val->nData = output;
+      break;
+    }
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z);
+      }
+
+      val->i64 = iv;
+      break;
+    }
+  }
 
   return TSDB_CODE_SUCCESS;
 }
 
 // pSql -> tag1_value, ...)
 static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) {
-  if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) {
+  int32_t code = TSDB_CODE_SUCCESS;
-    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal));
-  }
+  SToken  sToken;
-
+  bool    isParseBindParam = false;
-  SKvParam param = {.builder = &pCxt->tagsBuilder};
+  bool    isJson = false;
-  SToken   sToken;
+  STag*   pTag = NULL;
-  bool     isParseBindParam = false;
-  char     tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0};  // used for deleting Escape character: \\, \', \"
   for (int i = 0; i < pCxt->tags.numOfBound; ++i) {
     NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
 
     if (sToken.type == TK_NK_QUESTION) {
       isParseBindParam = true;
       if (NULL == pCxt->pStmtCb) {
-        return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
+        code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
+        goto end;
       }
 
       continue;
     }
 
     if (isParseBindParam) {
-      return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values");
+      code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values");
+      goto end;
     }
 
     SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]];
-    param.schema = pTagSchema;
+    char*    tmpTokenBuf = taosMemoryCalloc(1, sToken.n);  // this can be optimize with parse column
-    CHECK_CODE(
+    code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg);
-        parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, &param, &pCxt->msg));
+    if (code != TSDB_CODE_SUCCESS) {
+      taosMemoryFree(tmpTokenBuf);
+      goto end;
+    }
+    if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
+      if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+        code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z);
+        taosMemoryFree(tmpTokenBuf);
+        goto end;
+      }
+      code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg);
+      taosMemoryFree(tmpTokenBuf);
+      if (code != TSDB_CODE_SUCCESS) {
+        goto end;
+      }
+      isJson = true;
+    } else {
+      STagVal val = {0};
+      code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg);
+      if (TSDB_CODE_SUCCESS != code) {
+        taosMemoryFree(tmpTokenBuf);
+        goto end;
+      }
+      if (pTagSchema->type != TSDB_DATA_TYPE_BINARY) {
+        taosMemoryFree(tmpTokenBuf);
+      }
+      taosArrayPush(pTagVals, &val);
+    }
   }
 
   if (isParseBindParam) {
-    return TSDB_CODE_SUCCESS;
+    code = TSDB_CODE_SUCCESS;
+    goto end;
   }
 
-  SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder);
+  if (!isJson && (code = tTagNew(pTagVals, 1, false, &pTag)) != TSDB_CODE_SUCCESS) {
-  if (NULL == row) {
+    goto end;
-    return buildInvalidOperationMsg(&pCxt->msg, "out of memory");
   }
-  tdSortKVRowByColIdx(row);
 
-  return buildCreateTbReq(&pCxt->createTblReq, tName, row, pCxt->pTableMeta->suid);
+  buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid);
+
+end:
+  for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
+    STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);
+    if (IS_VAR_DATA_TYPE(p->type)) {
+      taosMemoryFree(p->pData);
+    }
+  }
+  taosArrayDestroy(pTagVals);
+  return code;
 }
 
 static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) {
@@ -1082,7 +1263,6 @@ void destroyCreateSubTbReq(SVCreateTbReq* pReq) {
 static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
   taosMemoryFreeClear(pCxt->pTableMeta);
   destroyBoundColumnInfo(&pCxt->tags);
-  tdDestroyKVRowBuilder(&pCxt->tagsBuilder);
   destroyCreateSubTbReq(&pCxt->createTblReq);
 }
 
@@ -1538,46 +1718,93 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN
     return TSDB_CODE_QRY_APP_ERROR;
   }
 
-  SKVRowBuilder tagBuilder;
+  SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
-  if (tdInitKVRowBuilder(&tagBuilder) < 0) {
+  if (!pTagArray) {
-    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return buildInvalidOperationMsg(&pBuf, "out of memory");
   }
 
+  int32_t  code = TSDB_CODE_SUCCESS;
   SSchema* pSchema = pDataBlock->pTableMeta->schema;
-  SKvParam param = {.builder = &tagBuilder};
+
+  bool  isJson = false;
+  STag* pTag = NULL;
 
   for (int c = 0; c < tags->numOfBound; ++c) {
-    SSchema* pTagSchema = &pSchema[tags->boundColumns[c]];
-    param.schema = pTagSchema;
-
     if (bind[c].is_null && bind[c].is_null[0]) {
-      KvRowAppend(&pBuf, NULL, 0, &param);
       continue;
     }
 
-    int32_t colLen = pTagSchema->bytes;
+    SSchema* pTagSchema = &pSchema[tags->boundColumns[c]];
+    int32_t  colLen = pTagSchema->bytes;
     if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
       colLen = bind[c].length[0];
     }
+    if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
+      if (colLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+        code = buildSyntaxErrMsg(&pBuf, "json string too long than 4095", bind[c].buffer);
+        goto end;
+      }
 
-    CHECK_CODE(KvRowAppend(&pBuf, (char*)bind[c].buffer, colLen, &param));
+      isJson = true;
+      char* tmp = taosMemoryCalloc(1, colLen + 1);
+      memcpy(tmp, bind[c].buffer, colLen);
+      code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf);
+      taosMemoryFree(tmp);
+      if (code != TSDB_CODE_SUCCESS) {
+        goto end;
+      }
+    } else {
+      STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
+      if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) {
+        val.pData = (uint8_t*)bind[c].buffer;
+        val.nData = colLen;
+      } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {
+        int32_t output = 0;
+        void*   p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE);
+        if (p == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto end;
+        }
+        if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
+          if (errno == E2BIG) {
+            taosMemoryFree(p);
+            code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
+            goto end;
+          }
+          char buf[512] = {0};
+          snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
+          taosMemoryFree(p);
+          code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer);
+          goto end;
+        }
+        val.pData = p;
+        val.nData = output;
+      } else {
+        memcpy(&val.i64, bind[c].buffer, colLen);
+      }
+      taosArrayPush(pTagArray, &val);
+    }
   }
 
-  SKVRow row = tdGetKVRowFromBuilder(&tagBuilder);
+  if (!isJson && (code = tTagNew(pTagArray, 1, false, &pTag)) != TSDB_CODE_SUCCESS) {
-  if (NULL == row) {
+    goto end;
-    tdDestroyKVRowBuilder(&tagBuilder);
-    return buildInvalidOperationMsg(&pBuf, "out of memory");
   }
-  tdSortKVRowByColIdx(row);
 
   SVCreateTbReq tbReq = {0};
-  CHECK_CODE(buildCreateTbReq(&tbReq, tName, row, suid));
+  buildCreateTbReq(&tbReq, tName, pTag, suid);
-  CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq));
+  code = buildCreateTbMsg(pDataBlock, &tbReq);
-
   destroyCreateSubTbReq(&tbReq);
-  tdDestroyKVRowBuilder(&tagBuilder);
 
-  return TSDB_CODE_SUCCESS;
+end:
+  for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
+    STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
+    if (p->type == TSDB_DATA_TYPE_NCHAR) {
+      taosMemoryFree(p->pData);
+    }
+  }
+  taosArrayDestroy(pTagArray);
+
+  return code;
 }
 
 int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
@@ -1746,7 +1973,8 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, uint8_t timePrec) {
+int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields,
+                         uint8_t timePrec) {
   if (fields) {
     *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD));
     if (NULL == *fields) {
@@ -1757,7 +1985,7 @@ int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_
     if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) {
       (*fields)[0].precision = timePrec;
     }
-    
+
     for (int32_t i = 0; i < boundInfo->numOfBound; ++i) {
       schema = &pSchema[boundInfo->boundColumns[i]];
       strcpy((*fields)[i].name, schema->name);
@@ -1803,7 +2031,8 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel
     return TSDB_CODE_SUCCESS;
   }
 
-  CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields, pDataBlock->pTableMeta->tableInfo.precision));
+  CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields,
+                              pDataBlock->pTableMeta->tableInfo.precision));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1812,7 +2041,6 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel
 
 typedef struct SmlExecTableHandle {
   SParsedDataColInfo tags;          // each table
-  SKVRowBuilder      tagsBuilder;   // each table
   SVCreateTbReq      createTblReq;  // each table
 } SmlExecTableHandle;
 
@@ -1824,7 +2052,6 @@ typedef struct SmlExecHandle {
 
 static void smlDestroyTableHandle(void* pHandle) {
   SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle;
-  tdDestroyKVRowBuilder(&handle->tagsBuilder);
   destroyBoundColumnInfo(&handle->tags);
   destroyCreateSubTbReq(&handle->createTblReq);
 }
@@ -1900,30 +2127,68 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema,
+/**
-                              SKVRow* row, SMsgBuf* msg) {
+ * @brief No json tag for schemaless
-  if (tdInitKVRowBuilder(tagsBuilder) < 0) {
+ *
+ * @param cols
+ * @param tags
+ * @param pSchema
+ * @param ppTag
+ * @param msg
+ * @return int32_t
+ */
+static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SMsgBuf* msg) {
+  SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
+  if (!pTagArray) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  SKvParam param = {.builder = tagsBuilder};
+  int32_t code = TSDB_CODE_SUCCESS;
   for (int i = 0; i < tags->numOfBound; ++i) {
     SSchema* pTagSchema = &pSchema[tags->boundColumns[i]];
-    param.schema = pTagSchema;
+    SSmlKv*  kv = taosArrayGetP(cols, i);
-    SSmlKv* kv = taosArrayGetP(cols, i);
+
-    if (IS_VAR_DATA_TYPE(kv->type)) {
+    STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
-      KvRowAppend(msg, kv->value, kv->length, &param);
+    if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) {
+      val.pData = (uint8_t*)kv->value;
+      val.nData = kv->length;
+    } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {
+      int32_t output = 0;
+      void*   p = taosMemoryCalloc(1, pTagSchema->bytes - VARSTR_HEADER_SIZE);
+      if (p == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto end;
+      }
+      if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), pTagSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
+        if (errno == E2BIG) {
+          taosMemoryFree(p);
+          code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
+          goto end;
+        }
+        char buf[512] = {0};
+        snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
+        taosMemoryFree(p);
+        code = buildSyntaxErrMsg(msg, buf, kv->value);
+        goto end;
+      }
+      val.pData = p;
+      val.nData = output;
     } else {
-      KvRowAppend(msg, &(kv->value), kv->length, &param);
+      memcpy(&val.i64, &(kv->value), kv->length);
     }
+    taosArrayPush(pTagArray, &val);
   }
 
-  *row = tdGetKVRowFromBuilder(tagsBuilder);
+  code = tTagNew(pTagArray, 1, false, ppTag);
-  if (*row == NULL) {
+end:
-    return TSDB_CODE_OUT_OF_MEMORY;
+  for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
+    STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
+    if (p->type == TSDB_DATA_TYPE_NCHAR) {
+      taosMemoryFree(p->pData);
+    }
   }
-  tdSortKVRowByColIdx(*row);
+  taosArrayDestroy(pTagArray);
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
@@ -1939,14 +2204,13 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
     buildInvalidOperationMsg(&pBuf, "bound tags error");
     return ret;
   }
-  SKVRow row = NULL;
+  STag* pTag = NULL;
-  ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema,
+  ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &pBuf);
-                       &row, &pBuf);
   if (ret != TSDB_CODE_SUCCESS) {
     return ret;
   }
 
-  buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, row, pTableMeta->suid);
+  buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid);
 
   STableDataBlocks* pDataBlock = NULL;
   ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid),

source/libs/parser/src/parTranslater.c

@@ -4167,8 +4167,8 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) {
   return code;
 }
 
-static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, SKVRow row,
+static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt,
-                                     uint64_t suid, SVgroupInfo* pVgInfo) {
+                                     const STag* pTag, uint64_t suid, SVgroupInfo* pVgInfo) {
   char  dbFName[TSDB_DB_FNAME_LEN] = {0};
   SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId};
   strcpy(name.dbname, pStmt->dbName);
@@ -4178,7 +4178,7 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S
   req.type = TD_CHILD_TABLE;
   req.name = strdup(pStmt->tableName);
   req.ctb.suid = suid;
-  req.ctb.pTag = row;
+  req.ctb.pTag = (uint8_t*)pTag;
   if (pStmt->ignoreExists) {
     req.flags |= TD_CREATE_IF_NOT_EXISTS;
   }
@@ -4198,24 +4198,6 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S
   }
 }
 
-static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
-                             SKVRowBuilder* pBuilder) {
-  if (pSchema->type == TSDB_DATA_TYPE_JSON) {
-    if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
-      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
-    }
-
-    return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
-  }
-
-  if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) {
-    tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal),
-                    IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) {
   int32_t code = getFuncInfo(pCxt, pFunc);
   if (TSDB_CODE_SUCCESS == code) {
@@ -4243,15 +4225,22 @@ static int32_t translateTagVal(STranslateContext* pCxt, uint8_t precision, SSche
 }
 
 static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,
-                                     SKVRowBuilder* pBuilder) {
+                                     STag** ppTag) {
   int32_t numOfTags = getNumOfTags(pSuperTableMeta);
   if (LIST_LENGTH(pStmt->pValsOfTags) != LIST_LENGTH(pStmt->pSpecificTags) ||
       numOfTags < LIST_LENGTH(pStmt->pValsOfTags)) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TAGS_NOT_MATCHED);
   }
 
+  SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal));
+  if (!pTagArray) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+  int32_t  code = TSDB_CODE_SUCCESS;
+  int16_t  nTags = 0, nBufPos = 0;
   SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta);
-  SNode *  pTag, *pNode;
+  SNode *  pTag = NULL, *pNode = NULL;
+  bool isJson = false;
   FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) {
     SColumnNode* pCol = (SColumnNode*)pTag;
     SSchema*     pSchema = NULL;
@@ -4262,56 +4251,125 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla
       }
     }
     if (NULL == pSchema) {
-      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName);
+      code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName);
+      goto end;
     }
     SValueNode* pVal = NULL;
-    int32_t     code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal);
+    code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal);
-    if (TSDB_CODE_SUCCESS == code) {
-      if (NULL == pVal) {
-        pVal = (SValueNode*)pNode;
-      } else {
-        REPLACE_LIST2_NODE(pVal);
-      }
-    }
-    if (TSDB_CODE_SUCCESS == code) {
-      code = addValToKVRow(pCxt, pVal, pSchema, pBuilder);
-    }
     if (TSDB_CODE_SUCCESS != code) {
-      return code;
+      goto end;
+    }
+
+    if (NULL == pVal) {
+      pVal = (SValueNode*)pNode;
+    } else {
+      REPLACE_LIST2_NODE(pVal);
+    }
+    if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
+      if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+        code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
+        goto end;
+      }
+
+      isJson = true;
+      code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf);
+      if(code != TSDB_CODE_SUCCESS){
+        goto end;
+      }
+    }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) {
+      void* nodeVal = nodesGetValueFromNode(pVal);
+      STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
+      if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
+        val.pData = varDataVal(nodeVal);
+        val.nData = varDataLen(nodeVal);
+      } else {
+        memcpy(&val.i64, nodeVal, pTagSchema->bytes);
+      }
+      taosArrayPush(pTagArray, &val);
     }
   }
 
+  if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag);
+
+end:
+  if(isJson){
+    for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
+      STagVal *p = (STagVal *)taosArrayGet(pTagArray, i);
+      if(IS_VAR_DATA_TYPE(p->type)){
+        taosMemoryFree(p->pData);
+      }
+    }
+  }
+  taosArrayDestroy(pTagArray);
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,
-                                    SKVRowBuilder* pBuilder) {
+                                    STag** ppTag) {
   if (getNumOfTags(pSuperTableMeta) != LIST_LENGTH(pStmt->pValsOfTags)) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TAGS_NOT_MATCHED);
   }
 
-  SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta);
+  SSchema* pTagSchemas = getTableTagSchema(pSuperTableMeta);
   SNode*   pNode;
+  int32_t  code = TSDB_CODE_SUCCESS;
   int32_t  index = 0;
+  SArray*  pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal));
+  if (!pTagArray) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+
+  bool isJson = false;
   FOREACH(pNode, pStmt->pValsOfTags) {
     SValueNode* pVal = NULL;
-    int32_t     code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema + index, pNode, &pVal);
+    SSchema*    pTagSchema = pTagSchemas + index;
-    if (TSDB_CODE_SUCCESS == code) {
+    code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema, pNode, &pVal);
-      if (NULL == pVal) {
-        pVal = (SValueNode*)pNode;
-      } else {
-        REPLACE_NODE(pVal);
-      }
-    }
-    if (TSDB_CODE_SUCCESS == code) {
-      code = addValToKVRow(pCxt, pVal, pTagSchema + index++, pBuilder);
-    }
     if (TSDB_CODE_SUCCESS != code) {
-      return code;
+      goto end;
+    }
+    if (NULL == pVal) {
+      pVal = (SValueNode*)pNode;
+    } else {
+      REPLACE_NODE(pVal);
+    }
+    if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
+      if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+        code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
+        goto end;
+      }
+
+      isJson = true;
+      code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf);
+      if(code != TSDB_CODE_SUCCESS){
+        goto end;
+      }
+    }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) {
+      char* tmpVal = nodesGetValueFromNode(pVal);
+      STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
+      if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
+        val.pData = varDataVal(tmpVal);
+        val.nData = varDataLen(tmpVal);
+      } else {
+        memcpy(&val.i64, tmpVal, pTagSchema->bytes);
+      }
+      taosArrayPush(pTagArray, &val);
+    }
+    ++index;
+  }
+  if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag);
+
+end:
+  if(isJson){
+    for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
+      STagVal *p = (STagVal *)taosArrayGet(pTagArray, i);
+      if(IS_VAR_DATA_TYPE(p->type)){
+        taosMemoryFree(p->pData);
+      }
     }
   }
 
-  return TSDB_CODE_SUCCESS;
+  taosArrayDestroy(pTagArray);
+  return code;
 }
 
 static int32_t checkCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt) {
@@ -4328,26 +4386,13 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
     code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
   }
 
-  SKVRowBuilder kvRowBuilder = {0};
+  STag* pTag = NULL;
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tdInitKVRowBuilder(&kvRowBuilder);
-  }
 
   if (TSDB_CODE_SUCCESS == code) {
     if (NULL != pStmt->pSpecificTags) {
-      code = buildKVRowForBindTags(pCxt, pStmt, pSuperTableMeta, &kvRowBuilder);
+      code = buildKVRowForBindTags(pCxt, pStmt, pSuperTableMeta, &pTag);
     } else {
-      code = buildKVRowForAllTags(pCxt, pStmt, pSuperTableMeta, &kvRowBuilder);
+      code = buildKVRowForAllTags(pCxt, pStmt, pSuperTableMeta, &pTag);
-    }
-  }
-
-  SKVRow row = NULL;
-  if (TSDB_CODE_SUCCESS == code) {
-    row = tdGetKVRowFromBuilder(&kvRowBuilder);
-    if (NULL == row) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-    } else {
-      tdSortKVRowByColIdx(row);
     }
   }
 
@@ -4356,11 +4401,10 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
     code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, row, pSuperTableMeta->uid, &info);
+    addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, pTag, pSuperTableMeta->uid, &info);
   }
 
   taosMemoryFreeClear(pSuperTableMeta);
-  tdDestroyKVRowBuilder(&kvRowBuilder);
   return code;
 }
 
@@ -4585,37 +4629,41 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
 
   pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
   if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) {
-    SKVRowBuilder kvRowBuilder = {0};
-    int32_t       code = tdInitKVRowBuilder(&kvRowBuilder);
-
-    if (TSDB_CODE_SUCCESS != code) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
     if (pStmt->pVal->literal &&
         strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
       return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal);
     }
-
+    SArray *pTagVals = taosArrayInit(1, sizeof(STagVal));
-    code = parseJsontoTagData(pStmt->pVal->literal, &kvRowBuilder, &pCxt->msgBuf, pSchema->colId);
+    int32_t code = TSDB_CODE_SUCCESS;
-    if (TSDB_CODE_SUCCESS != code) {
+    STag* pTag = NULL;
+    do{
+      code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf);
+      if (TSDB_CODE_SUCCESS != code) {
+        break;
+      }
+    }while(0);
+    for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
+      STagVal *p = (STagVal *)taosArrayGet(pTagVals, i);
+      if(IS_VAR_DATA_TYPE(p->type)){
+        taosMemoryFree(p->pData);
+      }
+    }
+    taosArrayDestroy(pTagVals);
+    if (code != TSDB_CODE_SUCCESS){
       return code;
     }
-
+    pReq->nTagVal = pTag->len;
-    SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+    pReq->pTagVal = (uint8_t *)pTag;
-    if (NULL == row) {
+    pStmt->pVal->datum.p = (char*)pTag;  // for free
-      tdDestroyKVRowBuilder(&kvRowBuilder);
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    pReq->nTagVal = kvRowLen(row);
-    pReq->pTagVal = row;
-    pStmt->pVal->datum.p = row;  // for free
-    tdDestroyKVRowBuilder(&kvRowBuilder);
   } else {
     pReq->nTagVal = pStmt->pVal->node.resType.bytes;
-    if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) {
-      pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE;
-    }
     pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal);
+
+    // data and length are seperated for new tag format STagVal
+    if (IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type)) {
+      pReq->nTagVal = varDataLen(pReq->pTagVal);
+      pReq->pTagVal = varDataVal(pReq->pTagVal);
+    }
   }
 
   return TSDB_CODE_SUCCESS;

source/libs/parser/src/parUtil.c

@@ -322,33 +322,35 @@ static bool isValidateTag(char* input) {
   return true;
 }
 
-int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId) {
+int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf) {
+  int32_t   retCode = TSDB_CODE_SUCCESS;
+  cJSON* root = NULL;
+  SHashObj* keyHash = NULL;
+  int32_t size = 0;
   // set json NULL data
-  uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
-  int32_t jsonIndex = startColId + 1;
   if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) {
-    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
+    retCode = TSDB_CODE_SUCCESS;
-    return TSDB_CODE_SUCCESS;
+    goto end;
   }
 
   // set json real data
-  cJSON* root = cJSON_Parse(json);
+  root = cJSON_Parse(json);
   if (root == NULL) {
-    return buildSyntaxErrMsg(pMsgBuf, "json parse error", json);
+    retCode = buildSyntaxErrMsg(pMsgBuf, "json parse error", json);
+    goto end;
   }
 
-  int32_t size = cJSON_GetArraySize(root);
+  size = cJSON_GetArraySize(root);
   if (!cJSON_IsObject(root)) {
-    return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json);
+    retCode = buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json);
+    goto end;
   }
 
-  int32_t   retCode = 0;
+  keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
-  char*     tagKV = NULL;
-  SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
   for (int32_t i = 0; i < size; i++) {
     cJSON* item = cJSON_GetArrayItem(root, i);
     if (!item) {
-      qError("json inner error:%d", i);
+      uError("json inner error:%d", i);
       retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
       goto end;
     }
@@ -360,85 +362,60 @@ int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBu
     }
     size_t keyLen = strlen(jsonKey);
     if (keyLen > TSDB_MAX_JSON_KEY_LEN) {
-      qError("json key too long error");
+      uError("json key too long error");
       retCode = buildSyntaxErrMsg(pMsgBuf, "json key too long, more than 256", jsonKey);
       goto end;
     }
     if (keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL) {
       continue;
     }
-    // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: DOUBLE_BYTES
+    STagVal val = {0};
-    tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES, 1);
+    val.pKey = jsonKey;
-    if (!tagKV) {
+    taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
-      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto end;
-    }
-    strncpy(varDataVal(tagKV), jsonKey, keyLen);
-    varDataSetLen(tagKV, keyLen);
-    if (taosHashGetSize(keyHash) == 0) {
-      uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON;
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);  // add json type
-    }
-    taosHashPut(keyHash, jsonKey, keyLen, &keyLen,
-                CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
 
     if (item->type == cJSON_String) {  // add json value  format: type|data
       char*   jsonValue = item->valuestring;
       int32_t valLen = (int32_t)strlen(jsonValue);
-      int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
+      char*   tmp = taosMemoryCalloc(1, valLen * TSDB_NCHAR_SIZE);
-      char*   tmp = taosMemoryRealloc(tagKV, totalLen);
       if (!tmp) {
         retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
         goto end;
       }
-      tagKV = tmp;
+      val.type = TSDB_DATA_TYPE_NCHAR;
-      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp,
-      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
-      *valueType = TSDB_DATA_TYPE_NCHAR;
-      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
                                        (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
-        qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue,
+        uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue,
                strerror(errno));
         retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
         goto end;
       }
-
+      val.nData = valLen;
-      varDataSetLen(valueData, valLen);
+      val.pData = tmp;
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
     } else if (item->type == cJSON_Number) {
       if (!isfinite(item->valuedouble)) {
-        qError("json value is invalidate");
+        uError("json value is invalidate");
         retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
         goto end;
       }
-      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      val.type = TSDB_DATA_TYPE_DOUBLE;
-      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *((double*)&(val.i64)) = item->valuedouble;
-      *valueType = TSDB_DATA_TYPE_DOUBLE;
-      *((double*)valueData) = item->valuedouble;
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES);
     } else if (item->type == cJSON_True || item->type == cJSON_False) {
-      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      val.type = TSDB_DATA_TYPE_BOOL;
-      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *((char*)&(val.i64)) = (char)(item->valueint);
-      *valueType = TSDB_DATA_TYPE_BOOL;
-      *valueData = (char)(item->valueint);
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
     } else if (item->type == cJSON_NULL) {
-      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      val.type = TSDB_DATA_TYPE_NULL;
-      *valueType = TSDB_DATA_TYPE_NULL;
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
     } else {
       retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
       goto end;
     }
-  }
+    taosArrayPush(pTagVals, &val);
-
-  if (taosHashGetSize(keyHash) == 0) {  // set json NULL true
-    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
   }
 
 end:
-  taosMemoryFree(tagKV);
   taosHashCleanup(keyHash);
+  if(retCode == TSDB_CODE_SUCCESS){
+    tTagNew(pTagVals, 1, true, ppTag);
+  }
   cJSON_Delete(root);
   return retCode;
 }

source/libs/parser/src/parser.c

@@ -76,28 +76,8 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
   int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes);
   pVal->node.resType.type = pParam->buffer_type;
   pVal->node.resType.bytes = inputSize;
+  
   switch (pParam->buffer_type) {
-    case TSDB_DATA_TYPE_BOOL:
-      pVal->datum.b = *((bool*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_TINYINT:
-      pVal->datum.i = *((int8_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_SMALLINT:
-      pVal->datum.i = *((int16_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_INT:
-      pVal->datum.i = *((int32_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_BIGINT:
-      pVal->datum.i = *((int64_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_FLOAT:
-      pVal->datum.d = *((float*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      pVal->datum.d = *((double*)pParam->buffer);
-      break;
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:
       pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
@@ -124,28 +104,13 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
       pVal->node.resType.bytes = output + VARSTR_HEADER_SIZE;
       break;
     }
-    case TSDB_DATA_TYPE_TIMESTAMP:
+    default: {
-      pVal->datum.i = *((int64_t*)pParam->buffer);
+      int32_t code = nodesSetValueNodeValue(pVal, pParam->buffer);
-      break;
+      if (code) {
-    case TSDB_DATA_TYPE_UTINYINT:
+        return code;
-      pVal->datum.u = *((uint8_t*)pParam->buffer);
+      }
-      break;
-    case TSDB_DATA_TYPE_USMALLINT:
-      pVal->datum.u = *((uint16_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_UINT:
-      pVal->datum.u = *((uint32_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_UBIGINT:
-      pVal->datum.u = *((uint64_t*)pParam->buffer);
-      break;
-    case TSDB_DATA_TYPE_JSON:
-    case TSDB_DATA_TYPE_DECIMAL:
-    case TSDB_DATA_TYPE_BLOB:
-    case TSDB_DATA_TYPE_MEDIUMBLOB:
-      // todo
-    default:
       break;
+    }
   }
   pVal->translate = true;
   return TSDB_CODE_SUCCESS;

source/libs/qworker/test/qworkerTests.cpp

@@ -108,7 +108,7 @@ void qwtInitLogFile() {
 
   tsAsyncLog = 0;
   qDebugFlag = 159;
-  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsLogDir, TD_LOG_DIR_PATH);
 
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);

source/libs/scalar/src/sclvector.c

@@ -922,23 +922,13 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
   }
 }
 
-char *getJsonValue(char *json, char *key){      //todo
+STagVal getJsonValue(char *json, char *key, bool *isExist) {
-  json++;     // jump type
+  STagVal val = {.pKey = key};
-  int16_t cols = kvRowNCols(json);
+  bool find = tTagGet(((const STag *)json), &val);  // json value is null and not exist is different
-  for (int i = 0; i < cols; ++i) {
+  if(isExist){
-    SColIdx *pColIdx = kvRowColIdxAt(json, i);
+    *isExist = find;
-    char *data = kvRowColVal(json, pColIdx);
-    if(i == 0){
-      if(*data == TSDB_DATA_TYPE_NULL) {
-        return NULL;
-      }
-      continue;
-    }
-    if(memcmp(key, data, varDataTLen(data)) == 0){
-      return data + varDataTLen(data);
-    }
   }
-  return NULL;
+  return val;
 }
 
 void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
@@ -950,6 +940,8 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
   pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
 
   char *pRightData = colDataGetVarData(pRight->columnData, 0);
+  char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1);
+  memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData));
   for (; i >= 0 && i < pLeft->numOfRows; i += step) {
     if (colDataIsNull_var(pLeft->columnData, i)) {
       colDataSetNull_var(pOutputCol, i);
@@ -957,14 +949,15 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
       continue;
     }
     char *pLeftData = colDataGetVarData(pLeft->columnData, i);
-    char *value = getJsonValue(pLeftData, pRightData);
+    bool isExist = false;
-    if (!value) {
+    STagVal value = getJsonValue(pLeftData, jsonKey, &isExist);
-      colDataSetNull_var(pOutputCol, i);
+    char *data = isExist ? tTagValToData(&value, true) : NULL;
-      pOutputCol->hasNull = true;
+    colDataAppend(pOutputCol, i, data, data == NULL);
-      continue;
+    if(isExist && IS_VAR_DATA_TYPE(value.type) && data){
+      taosMemoryFree(data);
     }
-    colDataAppend(pOutputCol, i, value, false);
   }
+  taosMemoryFree(jsonKey);
 }
 
 void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {

source/libs/scalar/test/filter/filterTests.cpp

@@ -60,7 +60,7 @@ void flttInitLogFile() {
 
   tsAsyncLog = 0;
   qDebugFlag = 159;
-  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsLogDir, TD_LOG_DIR_PATH);
 
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);

source/libs/scalar/test/scalar/CMakeLists.txt

@@ -17,9 +17,7 @@ TARGET_INCLUDE_DIRECTORIES(
         PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
         PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
 )
-if(NOT TD_WINDOWS)
+add_test(
-        add_test(
+        NAME scalarTest
-                NAME scalarTest
+        COMMAND scalarTest
-                COMMAND scalarTest
+)
-        )
-endif(NOT TD_WINDOWS)

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -74,7 +74,7 @@ void scltInitLogFile() {
 
   tsAsyncLog = 0;
   qDebugFlag = 159;
-  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsLogDir, TD_LOG_DIR_PATH);
 
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);
@@ -217,7 +217,7 @@ void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode
   SOperatorNode *onode = (SOperatorNode *)node;
   onode->node.resType.type = resType;
   onode->node.resType.bytes = tDataTypes[resType].bytes;
-  
+
   onode->opType = opType;
   onode->pLeft = pLeft;
   onode->pRight = pRight;
@@ -1035,7 +1035,7 @@ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
 
   SNode *pLeft = NULL, *pRight = NULL;
   scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar);
-  scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, kvRowLen(json), 1, json);
+  scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, ((STag*)json)->len, 1, json);
   scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
 }
 
@@ -1111,17 +1111,9 @@ TEST(columnTest, json_column_arith_op) {
 
   char rightv[256] = {0};
   memcpy(rightv, rightvTmp, strlen(rightvTmp));
-  SKVRowBuilder kvRowBuilder;
+  SArray *tags = taosArrayInit(1, sizeof(STagVal));
-  tdInitKVRowBuilder(&kvRowBuilder);
+  STag* row = NULL;
-  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  parseJsontoTagData(rightv, tags, &row, NULL);
-  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
-  char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1);
-  if(tmp == NULL){
-    ASSERT_TRUE(0);
-  }
-  memmove(tmp+1, tmp, kvRowLen(tmp));
-  *tmp = TSDB_DATA_TYPE_JSON;
-  row = tmp;
 
   const int32_t len = 8;
   EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
@@ -1175,7 +1167,7 @@ TEST(columnTest, json_column_arith_op) {
     makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
   }
 
-  tdDestroyKVRowBuilder(&kvRowBuilder);
+  taosArrayDestroy(tags);
   taosMemoryFree(row);
 }
 
@@ -1195,17 +1187,9 @@ TEST(columnTest, json_column_logic_op) {
 
   char rightv[256] = {0};
   memcpy(rightv, rightvTmp, strlen(rightvTmp));
-  SKVRowBuilder kvRowBuilder;
+  SArray *tags = taosArrayInit(1, sizeof(STagVal));
-  tdInitKVRowBuilder(&kvRowBuilder);
+  STag* row = NULL;
-  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  parseJsontoTagData(rightv, tags, &row, NULL);
-  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
-  char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1);
-  if(tmp == NULL){
-    ASSERT_TRUE(0);
-  }
-  memmove(tmp+1, tmp, kvRowLen(tmp));
-  *tmp = TSDB_DATA_TYPE_JSON;
-  row = tmp;
 
   const int32_t len = 9;
   const int32_t len1 = 4;
@@ -1305,7 +1289,7 @@ TEST(columnTest, json_column_logic_op) {
     taosMemoryFree(rightData);
   }
 
-  tdDestroyKVRowBuilder(&kvRowBuilder);
+  taosArrayDestroy(tags);
   taosMemoryFree(row);
 }
 

source/libs/scheduler/test/schedulerTests.cpp

@@ -79,7 +79,7 @@ void schtInitLogFile() {
 
   tsAsyncLog = 0;
   qDebugFlag = 159;
-  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsLogDir, TD_LOG_DIR_PATH);
 
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);

source/libs/stream/src/tstream.c

@@ -35,6 +35,16 @@ void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
   return (void*)buf;
 }
 
+SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) {
+  SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
+  if (pSubmitClone == NULL) {
+    return NULL;
+  }
+  streamDataSubmitRefInc(pSubmit);
+  memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit));
+  return pSubmitClone;
+}
+
 static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
   SStreamDispatchReq req = {
       .streamId = pTask->streamId,
@@ -207,7 +217,6 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
   if (pRes == NULL) return -1;
   while (1) {
     int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
-    void*  exec = pTask->exec.executor;
     if (execStatus == TASK_STATUS__IDLE) {
       // first run, from qall, handle failure from last exec
       pRes = streamExecForQall(pTask, pRes);

source/os/src/osEnv.c

@@ -70,11 +70,11 @@ void osDefaultInit() {
 
 #elif defined(_TD_DARWIN_64)
   if (configDir[0] == 0) {
-    strcpy(configDir, "/tmp/taosd");
+    strcpy(configDir, "/usr/local/etc/taos");
   }
   strcpy(tsDataDir, "/usr/local/var/lib/taos");
   strcpy(tsLogDir, "/usr/local/var/log/taos");
-  strcpy(tsTempDir, "/usr/local/etc/taos");
+  strcpy(tsTempDir, "/tmp/taosd");
   strcpy(tsOsName, "Darwin");
 
 #else

source/util/src/terror.c

@@ -75,6 +75,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY,                      "Cannot add duplicate
 TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY,                   "Retry needed")
 TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE,      "Out of memory in rpc queue")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP,            "Invalid timestamp format")
+TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR,             "Msg decode error")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY,                "Ref out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL,                     "too many Ref Objs")

tests/pytest/util/dnodes.py

@@ -397,6 +397,7 @@ class TDDnode:
     def stop(self):
         if (not self.remoteIP == ""):
             self.remoteExec(self.cfgDict, "tdDnodes.stop(%d)"%self.index)
+            tdLog.info("stop dnode%d"%self.index)
             return
         if self.valgrind == 0:
             toBeKilled = "taosd"

tests/script/tsim/db/alter_replica_13.sim

@@ -0,0 +1,124 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+
+print =============== step1: create dnodes
+sql create dnode $hostname port 7200
+
+$loop_cnt = 0
+step1:
+	$loop_cnt = $loop_cnt + 1
+	sleep 1000
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create database
+sql create database db vgroups 1
+sql show databases
+if $rows != 3 then 
+  return -1
+endi
+if $data(db)[4] != 1 then 
+  return -1
+endi
+
+sql show dnodes
+if $data(2)[2] != 1 then
+  return -1
+endi
+
+# vnodes
+sql show dnodes
+if $data(2)[2] != 1 then
+  return -1
+endi
+
+# v1_dnode
+sql show db.vgroups
+if $data(2)[3] != 2 then
+  return -1
+endi
+
+sql_error alter database db replica 3
+sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd"
+sql create table db.ctb using db.stb tags(101, "102")
+sql insert into db.ctb values(now, 1, "2")
+sql select * from db.stb
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step3: create dnodes
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+
+$loop_cnt = 0
+step3:
+	$loop_cnt = $loop_cnt + 1
+	sleep 1000
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> rows: $rows 
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data24 $data35
+if $rows != 4 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step3
+endi
+if $data(2)[4] != ready then
+  goto step3
+endi
+if $data(3)[4] != ready then
+  goto step3
+endi
+if $data(4)[4] != ready then
+  goto step3
+endi
+
+return
+print ============= step4: alter database
+sql alter database db replica 3
+if $rows != 3 then 
+  return -1
+endi
+if $data(db)[4] != 3 then 
+  return -1
+endi
+
+sql select * from db.stb
+if $rows != 1 then
+  return -1
+endi
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/system-test/0-others/taosShell.py

@@ -84,6 +84,12 @@ class TDTestCase:
     #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\
     #                 'serverPort': 7080, 'firstEp': 'trd02:7080'}
     hostname = socket.gethostname()
+    if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+        try:
+            config = eval(tdDnodes.dnodes[0].remoteIP)
+            hostname = config["host"]
+        except Exception:
+            hostname = tdDnodes.dnodes[0].remoteIP
     serverPort = '7080'
     rpcDebugFlagVal = '143'
     clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}

tests/system-test/0-others/taosShellError.py

@@ -86,6 +86,12 @@ class TDTestCase:
     #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\
     #                 'serverPort': 7080, 'firstEp': 'trd02:7080'}
     hostname = socket.gethostname()
+    if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+        try:
+            config = eval(tdDnodes.dnodes[0].remoteIP)
+            hostname = config["host"]
+        except Exception:
+            hostname = tdDnodes.dnodes[0].remoteIP
     serverPort = '7080'
     rpcDebugFlagVal = '143'
     clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}

tests/system-test/0-others/taosShellNetChk.py

@@ -86,6 +86,12 @@ class TDTestCase:
     #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\
     #                 'serverPort': 7080, 'firstEp': 'trd02:7080'}
     hostname = socket.gethostname()
+    if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+        try:
+            config = eval(tdDnodes.dnodes[0].remoteIP )
+            hostname = config["host"]
+        except Exception:
+            hostname = tdDnodes.dnodes[0].remoteIP
     serverPort = '7080'
     rpcDebugFlagVal = '143'
     clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
@@ -196,7 +202,7 @@ class TDTestCase:
         pktNum = '10'
         role   = 'client'
         if platform.system().lower() == 'windows':
-            taosCmd = buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c']
+            taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c']
             taosCmd = taosCmd.replace('\\','\\\\')
         else:
             taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c']