other: merge 3.0

cmake/cmake.define

@@ -1,5 +1,15 @@
 cmake_minimum_required(VERSION 3.16)
 
+if (NOT DEFINED TD_GRANT)
+  SET(TD_GRANT FALSE)
+endif()
+if (NOT DEFINED TD_USB_DONGLE)
+  SET(TD_USB_DONGLE FALSE)
+endif()
+IF (TD_GRANT)
+  ADD_DEFINITIONS(-D_GRANT)
+ENDIF ()
+
 IF ("${BUILD_TOOLS}" STREQUAL "")
   IF (TD_LINUX)
     IF (TD_ARM_32)

example/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_executable(tmq "")
 add_executable(tstream "")
+add_executable(demoapi "")
 
 target_sources(tmq
     PRIVATE
@@ -10,6 +11,12 @@ target_sources(tstream
     PRIVATE
     "src/tstream.c"
 )
+
+target_sources(demoapi
+    PRIVATE
+    "src/demoapi.c"
+)
+
 target_link_libraries(tmq
     taos
 )
@@ -18,6 +25,10 @@ target_link_libraries(tstream
     taos
 )
 
+target_link_libraries(demoapi
+    taos
+)
+
 target_include_directories(tmq
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
@@ -26,5 +37,11 @@ target_include_directories(tstream
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
+target_include_directories(demoapi
+    PUBLIC "${TD_SOURCE_DIR}/include/client"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
 SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq)
 SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream)
+SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi)

example/src/demoapi.c

@@ -0,0 +1,197 @@
+// C api call sequence demo
+// to compile: gcc -o apidemo apidemo.c -ltaos
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <argp.h>
+
+#include "taos.h"
+
+#define debugPrint(fmt, ...) \
+    do { if (g_args.debug_print || g_args.verbose_print) \
+      fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0)
+
+#define warnPrint(fmt, ...) \
+    do { fprintf(stderr, "\033[33m"); \
+        fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \
+        fprintf(stderr, "\033[0m"); } while(0)
+
+#define errorPrint(fmt, ...) \
+    do { fprintf(stderr, "\033[31m"); \
+        fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \
+        fprintf(stderr, "\033[0m"); } while(0)
+
+#define okPrint(fmt, ...) \
+    do { fprintf(stderr, "\033[32m"); \
+        fprintf(stderr, "OK: "fmt, __VA_ARGS__); \
+        fprintf(stderr, "\033[0m"); } while(0)
+
+int64_t g_num_of_tb = 2;
+int64_t g_num_of_rec = 2;
+
+static struct argp_option options[] = {
+    {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."},
+    {"records", 'n', "NUMBER", 0,
+     "Number of records for each table, default is 10000."},
+    {0}};
+
+static error_t parse_opt(int key, char *arg, struct argp_state *state) {
+    switch (key) {
+        case 't':
+            g_num_of_tb = atoll(arg);
+            break;
+
+        case 'n':
+            g_num_of_rec = atoll(arg);
+            break;
+    }
+
+    return 0;
+}
+
+static struct argp argp = {options, parse_opt, "", ""};
+
+static void prepare_data(TAOS* taos) {
+    TAOS_RES *res;
+    res = taos_query(taos, "drop database if exists test;");
+    taos_free_result(res);
+    usleep(100000);
+
+    res = taos_query(taos, "create database test;");
+    taos_free_result(res);
+    usleep(100000);
+    taos_select_db(taos, "test");
+
+    res = taos_query(taos, "create table meters(ts timestamp, f float, n int, b binary(20)) tags(area int, localtion binary(20));");
+    taos_free_result(res);
+
+    char command[1024] = {0};
+    for (int64_t i = 0; i < g_num_of_tb; i ++) {
+        sprintf(command, "create table t%"PRId64" using meters tags(%"PRId64", '%s');",
+                i, i, (i%2)?"beijing":"shanghai");
+        res = taos_query(taos,  command);
+        taos_free_result(res);
+
+        int64_t j = 0;
+        int64_t total = 0;
+        int64_t affected;
+        for (; j < g_num_of_rec -1; j ++) {
+            sprintf(command, "insert into t%"PRId64" values(%" PRId64 ", %f, %"PRId64", '%c%d')",
+                    i, 1650000000000+j, (float)j, j, 'a'+(int)j%10, rand());
+            res = taos_query(taos,  command);
+            if ((res) && (0 == taos_errno(res))) {
+                affected = taos_affected_rows(res);
+                total += affected;
+            } else {
+                errorPrint("%s() LN%d: %s\n",
+                        __func__, __LINE__, taos_errstr(res));
+            }
+            taos_free_result(res);
+        }
+        sprintf(command, "insert into t%"PRId64" values(%" PRId64 ", NULL, NULL, NULL)",
+                i, 1650000000000+j+1);
+        res = taos_query(taos,  command);
+        if ((res) && (0 == taos_errno(res))) {
+            affected = taos_affected_rows(res);
+            total += affected;
+        } else {
+            errorPrint("%s() LN%d: %s\n",
+                    __func__, __LINE__, taos_errstr(res));
+        }
+        taos_free_result(res);
+
+        printf("insert %"PRId64" records into t%"PRId64", total affected rows: %"PRId64"\n", j, i, total);
+    }
+}
+
+static int print_result(TAOS_RES* res, int block) {
+    int64_t num_rows = 0;
+    TAOS_ROW    row = NULL;
+    int         num_fields = taos_num_fields(res);
+    TAOS_FIELD* fields = taos_fetch_fields(res);
+
+    if (block) {
+        warnPrint("%s() LN%d, call taos_fetch_block()\n", __func__, __LINE__);
+        int rows = 0;
+        while ((rows = taos_fetch_block(res, &row))) {
+            num_rows += rows;
+        }
+    } else {
+        warnPrint("%s() LN%d, call taos_fetch_rows()\n", __func__, __LINE__);
+        while ((row = taos_fetch_row(res))) {
+            char temp[256] = {0};
+            taos_print_row(temp, row, fields, num_fields);
+            puts(temp);
+            num_rows ++;
+        }
+    }
+
+    return num_rows;
+}
+
+static void verify_query(TAOS* taos) {
+    // TODO: select count(tbname) from stable once stable query work
+    char command[1024] = {0};
+
+    for (int64_t i = 0; i < g_num_of_tb; i++) {
+        sprintf(command, "select * from t%"PRId64"", i);
+        TAOS_RES* res = taos_query(taos, command);
+
+        if (res) {
+            if (0 == taos_errno(res)) {
+                int field_count = taos_field_count(res);
+                printf("field_count: %d\n", field_count);
+                int* lengths = taos_fetch_lengths(res);
+                if (lengths) {
+                for (int c = 0; c < field_count; c++) {
+                    printf("length of column %d is %d\n", c, lengths[c]);
+                }
+                } else {
+                    errorPrint("%s() LN%d: t%"PRId64"'s lengths is NULL\n",
+                            __func__, __LINE__, i);
+                }
+
+               int64_t rows = print_result(res, i % 2);
+               printf("rows is: %"PRId64"\n", rows);
+
+            } else {
+                errorPrint("%s() LN%d: %s\n",
+                        __func__, __LINE__, taos_errstr(res));
+            }
+        } else {
+            errorPrint("%s() LN%d: %s\n",
+                    __func__, __LINE__, taos_errstr(res));
+        }
+    }
+}
+
+int main(int argc, char *argv[]) {
+    const char* host = "127.0.0.1";
+    const char* user = "root";
+    const char* passwd = "taosdata";
+
+    argp_parse(&argp, argc, argv, 0, 0, NULL);
+    TAOS* taos = taos_connect(host, user, passwd, "", 0);
+    if (taos == NULL) {
+        printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));
+        exit(1);
+    }
+
+    const char* info = taos_get_server_info(taos);
+    printf("server info: %s\n", info);
+    info = taos_get_client_info(taos);
+    printf("client info: %s\n", info);
+
+    prepare_data(taos);
+
+    verify_query(taos);
+
+    taos_close(taos);
+    printf("done\n");
+
+    return 0;
+}
+

example/src/tmq.c

@@ -19,8 +19,8 @@
 #include <time.h>
 #include "taos.h"
 
-static int  running = 1;
+static int running = 1;
-static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }
+/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/
 
 int32_t init_env() {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -166,11 +166,11 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   int32_t cnt = 0;
   /*clock_t startTime = clock();*/
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500);
     if (tmqmessage) {
       cnt++;
-      printf("get data\n");
+      /*printf("get data\n");*/
-      msg_process(tmqmessage);
+      /*msg_process(tmqmessage);*/
       tmq_message_destroy(tmqmessage);
       /*} else {*/
       /*break;*/
@@ -198,9 +198,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   }
 
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1000);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000);
     if (tmqmessage) {
-      msg_process(tmqmessage);
+      /*msg_process(tmqmessage);*/
       tmq_message_destroy(tmqmessage);
 
       if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
@@ -226,10 +226,10 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
   int32_t skipLogNum = 0;
   clock_t startTime = clock();
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500);
     if (tmqmessage) {
       batchCnt++;
-      skipLogNum += tmqGetSkipLogNum(tmqmessage);
+      /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/
       /*msg_process(tmqmessage);*/
       tmq_message_destroy(tmqmessage);
     } else {

include/client/taos.h

@@ -30,7 +30,7 @@ typedef void **TAOS_ROW;
 #if 0
 typedef void   TAOS_STREAM;
 #endif
-typedef void   TAOS_SUB;
+typedef void TAOS_SUB;
 
 // Data type definition
 #define TSDB_DATA_TYPE_NULL       0   // 1 bytes
@@ -138,13 +138,13 @@ typedef enum {
 #define RET_MSG_LENGTH 1024
 typedef struct setConfRet {
   SET_CONF_RET_CODE retCode;
-  char   retMsg[RET_MSG_LENGTH];
+  char              retMsg[RET_MSG_LENGTH];
 } setConfRet;
 
-DLL_EXPORT void  taos_cleanup(void);
+DLL_EXPORT void       taos_cleanup(void);
-DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
+DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
-DLL_EXPORT setConfRet   taos_set_config(const char *config);
+DLL_EXPORT setConfRet taos_set_config(const char *config);
-DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
+DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
@@ -152,34 +152,34 @@ DLL_EXPORT void  taos_close(TAOS *taos);
 
 const char *taos_data_type(int type);
 
-DLL_EXPORT TAOS_STMT  *taos_stmt_init(TAOS *taos);
+DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
-DLL_EXPORT int         taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
+DLL_EXPORT int        taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-DLL_EXPORT int         taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags);
+DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags);
-DLL_EXPORT int         taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
+DLL_EXPORT int        taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
-DLL_EXPORT int         taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
+DLL_EXPORT int        taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
 
-DLL_EXPORT int         taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
+DLL_EXPORT int       taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
-DLL_EXPORT int         taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
+DLL_EXPORT int       taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
-DLL_EXPORT int         taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
+DLL_EXPORT int       taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int         taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind);
+DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind);
-DLL_EXPORT int         taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int         taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
+DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
-DLL_EXPORT int         taos_stmt_add_batch(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
-DLL_EXPORT int         taos_stmt_execute(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
-DLL_EXPORT TAOS_RES   *taos_stmt_use_result(TAOS_STMT *stmt);
+DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
-DLL_EXPORT int         taos_stmt_close(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
-DLL_EXPORT char       *taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
-DLL_EXPORT int         taos_stmt_affected_rows(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
 
-DLL_EXPORT TAOS_RES   *taos_query(TAOS *taos, const char *sql);
+DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
-DLL_EXPORT TAOS_RES   *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
+DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
 
-DLL_EXPORT TAOS_ROW    taos_fetch_row(TAOS_RES *res);
+DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
-DLL_EXPORT int         taos_result_precision(TAOS_RES *res);  // get the time precision of result
+DLL_EXPORT int      taos_result_precision(TAOS_RES *res);  // get the time precision of result
-DLL_EXPORT void        taos_free_result(TAOS_RES *res);
+DLL_EXPORT void     taos_free_result(TAOS_RES *res);
-DLL_EXPORT int         taos_field_count(TAOS_RES *res);
+DLL_EXPORT int      taos_field_count(TAOS_RES *res);
-DLL_EXPORT int         taos_num_fields(TAOS_RES *res);
+DLL_EXPORT int      taos_num_fields(TAOS_RES *res);
-DLL_EXPORT int         taos_affected_rows(TAOS_RES *res);
+DLL_EXPORT int      taos_affected_rows(TAOS_RES *res);
 
 DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
 DLL_EXPORT int         taos_select_db(TAOS *taos, const char *db);
@@ -188,14 +188,14 @@ DLL_EXPORT void        taos_stop_query(TAOS_RES *res);
 DLL_EXPORT bool        taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
 DLL_EXPORT bool        taos_is_update_query(TAOS_RES *res);
 DLL_EXPORT int         taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
-DLL_EXPORT int         taos_fetch_block_s(TAOS_RES *res, int* numOfRows, TAOS_ROW *rows);
+DLL_EXPORT int         taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
-DLL_EXPORT int         taos_fetch_raw_block(TAOS_RES *res, int* numOfRows, void** pData);
+DLL_EXPORT int         taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
 DLL_EXPORT int        *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
 DLL_EXPORT int         taos_validate_sql(TAOS *taos, const char *sql);
 DLL_EXPORT void        taos_reset_current_db(TAOS *taos);
 
-DLL_EXPORT int        *taos_fetch_lengths(TAOS_RES *res);
+DLL_EXPORT int      *taos_fetch_lengths(TAOS_RES *res);
-DLL_EXPORT TAOS_ROW   *taos_result_block(TAOS_RES *res);
+DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
 
 DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
 DLL_EXPORT const char *taos_get_client_info();
@@ -237,9 +237,9 @@ typedef struct tmq_t                   tmq_t;
 typedef struct tmq_topic_vgroup_t      tmq_topic_vgroup_t;
 typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t;
 
-typedef struct tmq_conf_t    tmq_conf_t;
+typedef struct tmq_conf_t tmq_conf_t;
-typedef struct tmq_list_t    tmq_list_t;
+typedef struct tmq_list_t tmq_list_t;
-typedef struct tmq_message_t tmq_message_t;
+// typedef struct tmq_message_t tmq_message_t;
 
 typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param));
 
@@ -259,7 +259,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
+DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups);
@@ -268,8 +268,8 @@ DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** v
 DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
-#endif
 DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
+#endif
 /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
 
 enum tmq_conf_res_t {
@@ -285,21 +285,24 @@ DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb);
 
+#if 0
 // temporary used function for demo only
 void    tmqShowMsg(tmq_message_t *tmq_message);
 int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
+#endif
 
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 
+DLL_EXPORT char   *tmq_get_topic_name(TAOS_RES *res);
+DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
+#if 0
 DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
-DLL_EXPORT char       *tmq_get_topic_name(tmq_message_t *message);
-DLL_EXPORT int32_t     tmq_get_vgroup_id(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
 DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
 DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
-DLL_EXPORT void        tmq_message_destroy(tmq_message_t *tmq_message);
+#endif
-
+DLL_EXPORT void tmq_message_destroy(TAOS_RES *res);
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
 #if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
@@ -308,7 +311,7 @@ DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *
 DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
 
 /* ------------------------------ TMQ END -------------------------------- */
-#if 1 // Shuduo: temporary enable for app build
+#if 1  // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
 #endif
 

include/common/tdatablock.h

@@ -199,11 +199,11 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
 
-void    colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
+void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
-void    blockDataCleanup(SSDataBlock* pDataBlock);
+void blockDataCleanup(SSDataBlock* pDataBlock);
-
-size_t  blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 
+size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
+void*  blockDataDestroy(SSDataBlock* pBlock);
 
 int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
 
@@ -211,6 +211,10 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
 
 void blockDebugShowData(const SArray* dataBlocks);
 
+static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
+  return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock);
+}
+
 static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,
                                                  int8_t compressed) {
   int32_t colSize = colDataGetLength(pColRes, numOfRows);

include/common/tmsg.h

@@ -485,7 +485,7 @@ typedef struct {
   char    intervalUnit;
   char    slidingUnit;
   char
-          offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
+      offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
   int8_t  precision;
   int64_t interval;
   int64_t sliding;
@@ -2021,6 +2021,7 @@ typedef struct {
 static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI8(buf, pSchema->type);
+  tlen += taosEncodeFixedI8(buf, pSchema->index);
   tlen += taosEncodeFixedI32(buf, pSchema->bytes);
   tlen += taosEncodeFixedI16(buf, pSchema->colId);
   tlen += taosEncodeString(buf, pSchema->name);
@@ -2029,6 +2030,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema
 
 static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) {
   buf = taosDecodeFixedI8(buf, &pSchema->type);
+  buf = taosDecodeFixedI8(buf, &pSchema->index);
   buf = taosDecodeFixedI32(buf, &pSchema->bytes);
   buf = taosDecodeFixedI16(buf, &pSchema->colId);
   buf = taosDecodeStringTo(buf, pSchema->name);
@@ -2037,6 +2039,7 @@ static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) {
 
 static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
   if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1;
+  if (tEncodeI8(pEncoder, pSchema->index) < 0) return -1;
   if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1;
   if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1;
   if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1;
@@ -2045,6 +2048,7 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch
 
 static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) {
   if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pSchema->index) < 0) return -1;
   if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1;
   if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1;
@@ -2385,6 +2389,53 @@ typedef struct {
   SArray*         pBlockData;  // SArray<SSDataBlock>
 } SMqPollRsp;
 
+typedef struct {
+  SMqRspHead head;
+  int64_t    reqOffset;
+  int64_t    rspOffset;
+  int32_t    skipLogNum;
+  int32_t    dataLen;
+  SArray*    blockPos;   // beginning pos for each SRetrieveTableRsp
+  void*      blockData;  // serialized batched SRetrieveTableRsp
+} SMqPollRspV2;
+
+static FORCE_INLINE int32_t tEncodeSMqPollRspV2(void** buf, const SMqPollRspV2* pRsp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
+  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
+  tlen += taosEncodeFixedI32(buf, pRsp->dataLen);
+  if (pRsp->dataLen != 0) {
+    int32_t sz = taosArrayGetSize(pRsp->blockPos);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      int32_t blockPos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
+      tlen += taosEncodeFixedI32(buf, blockPos);
+    }
+    tlen += taosEncodeBinary(buf, pRsp->blockData, pRsp->dataLen);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRsp) {
+  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
+  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
+  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
+  buf = taosDecodeFixedI32(buf, &pRsp->dataLen);
+  if (pRsp->dataLen != 0) {
+    int32_t sz;
+    buf = taosDecodeFixedI32(buf, &sz);
+    pRsp->blockPos = taosArrayInit(sz, sizeof(int32_t));
+    for (int32_t i = 0; i < sz; i++) {
+      int32_t blockPos;
+      buf = taosDecodeFixedI32(buf, &blockPos);
+      taosArrayPush(pRsp->blockPos, &blockPos);
+    }
+    buf = taosDecodeBinary(buf, &pRsp->blockData, pRsp->dataLen);
+  }
+  return (void*)buf;
+}
+
 typedef struct {
   SMqRspHead head;
   char       cgroup[TSDB_CGROUP_LEN];

include/common/ttime.h

@@ -64,6 +64,7 @@ char getPrecisionUnit(int32_t precision);
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
+int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal);
 
 void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
 

include/common/ttokendef.h

@@ -140,88 +140,89 @@
 #define TK_APPS                           122
 #define TK_CONNECTIONS                    123
 #define TK_LICENCE                        124
-#define TK_QUERIES                        125
+#define TK_GRANTS                         125
-#define TK_SCORES                         126
+#define TK_QUERIES                        126
-#define TK_TOPICS                         127
+#define TK_SCORES                         127
-#define TK_VARIABLES                      128
+#define TK_TOPICS                         128
-#define TK_BNODES                         129
+#define TK_VARIABLES                      129
-#define TK_SNODES                         130
+#define TK_BNODES                         130
-#define TK_LIKE                           131
+#define TK_SNODES                         131
-#define TK_INDEX                          132
+#define TK_LIKE                           132
-#define TK_FULLTEXT                       133
+#define TK_INDEX                          133
-#define TK_FUNCTION                       134
+#define TK_FULLTEXT                       134
-#define TK_INTERVAL                       135
+#define TK_FUNCTION                       135
-#define TK_TOPIC                          136
+#define TK_INTERVAL                       136
-#define TK_AS                             137
+#define TK_TOPIC                          137
-#define TK_DESC                           138
+#define TK_AS                             138
-#define TK_DESCRIBE                       139
+#define TK_DESC                           139
-#define TK_RESET                          140
+#define TK_DESCRIBE                       140
-#define TK_QUERY                          141
+#define TK_RESET                          141
-#define TK_EXPLAIN                        142
+#define TK_QUERY                          142
-#define TK_ANALYZE                        143
+#define TK_EXPLAIN                        143
-#define TK_VERBOSE                        144
+#define TK_ANALYZE                        144
-#define TK_NK_BOOL                        145
+#define TK_VERBOSE                        145
-#define TK_RATIO                          146
+#define TK_NK_BOOL                        146
-#define TK_COMPACT                        147
+#define TK_RATIO                          147
-#define TK_VNODES                         148
+#define TK_COMPACT                        148
-#define TK_IN                             149
+#define TK_VNODES                         149
-#define TK_OUTPUTTYPE                     150
+#define TK_IN                             150
-#define TK_AGGREGATE                      151
+#define TK_OUTPUTTYPE                     151
-#define TK_BUFSIZE                        152
+#define TK_AGGREGATE                      152
-#define TK_STREAM                         153
+#define TK_BUFSIZE                        153
-#define TK_INTO                           154
+#define TK_STREAM                         154
-#define TK_KILL                           155
+#define TK_INTO                           155
-#define TK_CONNECTION                     156
+#define TK_KILL                           156
-#define TK_MERGE                          157
+#define TK_CONNECTION                     157
-#define TK_VGROUP                         158
+#define TK_MERGE                          158
-#define TK_REDISTRIBUTE                   159
+#define TK_VGROUP                         159
-#define TK_SPLIT                          160
+#define TK_REDISTRIBUTE                   160
-#define TK_SYNCDB                         161
+#define TK_SPLIT                          161
-#define TK_NULL                           162
+#define TK_SYNCDB                         162
-#define TK_FIRST                          163
+#define TK_NULL                           163
-#define TK_LAST                           164
+#define TK_FIRST                          164
-#define TK_NOW                            165
+#define TK_LAST                           165
-#define TK_ROWTS                          166
+#define TK_NOW                            166
-#define TK_TBNAME                         167
+#define TK_ROWTS                          167
-#define TK_QSTARTTS                       168
+#define TK_TBNAME                         168
-#define TK_QENDTS                         169
+#define TK_QSTARTTS                       169
-#define TK_WSTARTTS                       170
+#define TK_QENDTS                         170
-#define TK_WENDTS                         171
+#define TK_WSTARTTS                       171
-#define TK_WDURATION                      172
+#define TK_WENDTS                         172
-#define TK_BETWEEN                        173
+#define TK_WDURATION                      173
-#define TK_IS                             174
+#define TK_BETWEEN                        174
-#define TK_NK_LT                          175
+#define TK_IS                             175
-#define TK_NK_GT                          176
+#define TK_NK_LT                          176
-#define TK_NK_LE                          177
+#define TK_NK_GT                          177
-#define TK_NK_GE                          178
+#define TK_NK_LE                          178
-#define TK_NK_NE                          179
+#define TK_NK_GE                          179
-#define TK_MATCH                          180
+#define TK_NK_NE                          180
-#define TK_NMATCH                         181
+#define TK_MATCH                          181
-#define TK_JOIN                           182
+#define TK_NMATCH                         182
-#define TK_INNER                          183
+#define TK_JOIN                           183
-#define TK_SELECT                         184
+#define TK_INNER                          184
-#define TK_DISTINCT                       185
+#define TK_SELECT                         185
-#define TK_WHERE                          186
+#define TK_DISTINCT                       186
-#define TK_PARTITION                      187
+#define TK_WHERE                          187
-#define TK_BY                             188
+#define TK_PARTITION                      188
-#define TK_SESSION                        189
+#define TK_BY                             189
-#define TK_STATE_WINDOW                   190
+#define TK_SESSION                        190
-#define TK_SLIDING                        191
+#define TK_STATE_WINDOW                   191
-#define TK_FILL                           192
+#define TK_SLIDING                        192
-#define TK_VALUE                          193
+#define TK_FILL                           193
-#define TK_NONE                           194
+#define TK_VALUE                          194
-#define TK_PREV                           195
+#define TK_NONE                           195
-#define TK_LINEAR                         196
+#define TK_PREV                           196
-#define TK_NEXT                           197
+#define TK_LINEAR                         197
-#define TK_GROUP                          198
+#define TK_NEXT                           198
-#define TK_HAVING                         199
+#define TK_GROUP                          199
-#define TK_ORDER                          200
+#define TK_HAVING                         200
-#define TK_SLIMIT                         201
+#define TK_ORDER                          201
-#define TK_SOFFSET                        202
+#define TK_SLIMIT                         202
-#define TK_LIMIT                          203
+#define TK_SOFFSET                        203
-#define TK_OFFSET                         204
+#define TK_LIMIT                          204
-#define TK_ASC                            205
+#define TK_OFFSET                         205
-#define TK_NULLS                          206
+#define TK_ASC                            206
+#define TK_NULLS                          207
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301

include/libs/function/functionMgt.h

@@ -85,8 +85,8 @@ typedef enum EFunctionType {
   // conversion function
   FUNCTION_TYPE_CAST = 2000,
   FUNCTION_TYPE_TO_ISO8601,
+  FUNCTION_TYPE_TO_UNIXTIMESTAMP,
   FUNCTION_TYPE_TO_JSON,
-  FUNCTION_TYPE_UNIXTIMESTAMP,
 
   // date and time function
   FUNCTION_TYPE_NOW = 2500,

include/libs/scalar/scalar.h

@@ -31,8 +31,8 @@ pNode will be freed in API;
 */
 int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
 
-/* 
+/*
-pDst need to freed in caller 
+pDst need to freed in caller
 */
 int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst);
 
@@ -73,6 +73,12 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 /* Conversion functions */
 int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
+/* Time related functions */
+int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
 bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 
 int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

include/os/osDir.h

@@ -38,6 +38,7 @@ typedef struct TdDirEntry *TdDirEntryPtr;
 void    taosRemoveDir(const char *dirname);
 bool    taosDirExist(char *dirname);
 int32_t taosMkDir(const char *dirname);
+int32_t taosMulMkDir(const char *dirname);
 void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
 int32_t taosRealPath(char *dirname, int32_t maxlen);

include/os/osFile.h

@@ -79,7 +79,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
 int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
 void    taosFprintfFile(TdFilePtr pFile, const char *format, ...);
 
-int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
+int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf);
 
 int32_t taosEOFFile(TdFilePtr pFile);
  

include/os/osSystem.h

@@ -29,6 +29,13 @@ extern "C" {
     #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
 #endif
 
+typedef struct TdCmd *TdCmdPtr;
+
+TdCmdPtr taosOpenCmd(const char *cmd);
+int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf);
+int32_t taosEOFCmd(TdCmdPtr pCmd);
+int64_t taosCloseCmd(TdCmdPtr *ppCmd);
+
 void* taosLoadDll(const char* filename);
 void* taosLoadSym(void* handle, char* name);
 void  taosCloseDll(void* handle);

include/util/tdef.h

@@ -94,6 +94,11 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_TIME_PRECISION_MICRO_STR "us"
 #define TSDB_TIME_PRECISION_NANO_STR  "ns"
 
+#define TSDB_TIME_PRECISION_SEC_DIGITS   10
+#define TSDB_TIME_PRECISION_MILLI_DIGITS 13
+#define TSDB_TIME_PRECISION_MICRO_DIGITS 16
+#define TSDB_TIME_PRECISION_NANO_DIGITS  19
+
 #define TSDB_INFORMATION_SCHEMA_DB            "information_schema"
 #define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
 #define TSDB_INS_TABLE_DNODES                 "dnodes"

packaging/install.sh

@@ -157,7 +157,7 @@ function install_main_path() {
     ${csudo} mkdir -p ${install_main_dir}/cfg
     ${csudo} mkdir -p ${install_main_dir}/bin
     ${csudo} mkdir -p ${install_main_dir}/connector
-    ${csudo} mkdir -p ${install_main_dir}/driver
+    ${csudo} mkdir -p ${install_main_dir}/lib
     ${csudo} mkdir -p ${install_main_dir}/examples
     ${csudo} mkdir -p ${install_main_dir}/include
     ${csudo} mkdir -p ${install_main_dir}/init.d
@@ -198,6 +198,10 @@ function install_lib() {
     # Remove links
     ${csudo} rm -f ${lib_link_dir}/libtaos.*         || :
     ${csudo} rm -f ${lib64_link_dir}/libtaos.*       || :
+    ${csudo} rm -f ${lib_link_dir}/libtdb.*         || :
+    ${csudo} rm -f ${lib64_link_dir}/libtdb.*       || :
+
+    ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/*
 
     ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1
     ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so

source/client/inc/clientInt.h

@@ -194,12 +194,12 @@ enum {
 #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY)
 #define TD_RES_TMQ(res)   (*(int8_t*)res == RES_TYPE__TMQ)
 
-typedef struct SMqRspObj {
+typedef struct {
   int8_t  resType;
   char*   topic;
-  void*   vg;
   SArray* res;  // SArray<SReqResultInfo>
   int32_t resIter;
+  int32_t vgId;
 } SMqRspObj;
 
 typedef struct SRequestObj {

source/client/src/clientMain.c

@@ -465,7 +465,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
       return 0;
     }
 
-    doFetchRow(pRequest, false, false);
+    doFetchRow(pRequest, false, true);
 
     SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
 

source/client/src/tmq.c

@@ -27,11 +27,22 @@
 struct tmq_message_t {
   SMqPollRsp msg;
   char*      topic;
-  void*      vg;
   SArray*    res;  // SArray<SReqResultInfo>
+  int32_t    vgId;
   int32_t    resIter;
 };
 
+typedef struct {
+  int8_t  tmqRspType;
+  int32_t epoch;
+} SMqRspWrapper;
+
+typedef struct {
+  int8_t           tmqRspType;
+  int32_t          epoch;
+  SMqCMGetSubEpRsp msg;
+} SMqAskEpRspWrapper;
+
 struct tmq_list_t {
   SArray container;
 };
@@ -118,6 +129,14 @@ typedef struct {
   TAOS_FIELD* fields;
 } SMqClientTopic;
 
+typedef struct {
+  int8_t          tmqRspType;
+  int32_t         epoch;
+  SMqClientVg*    vgHandle;
+  SMqClientTopic* topicHandle;
+  SMqPollRspV2    msg;
+} SMqPollRspWrapper;
+
 typedef struct {
   tmq_t*         tmq;
   tsem_t         rspSem;
@@ -133,10 +152,10 @@ typedef struct {
 typedef struct {
   tmq_t*          tmq;
   SMqClientVg*    pVg;
+  SMqClientTopic* pTopic;
   int32_t         epoch;
   int32_t         vgId;
   tsem_t          rspSem;
-  tmq_message_t** msg;
   int32_t         sync;
 } SMqPollCbParam;
 
@@ -244,7 +263,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
 }
 
 void tmqClearUnhandleMsg(tmq_t* tmq) {
-  tmq_message_t* msg = NULL;
+  SMqRspWrapper* msg = NULL;
   while (1) {
     taosGetQitem(tmq->qall, (void**)&msg);
     if (msg)
@@ -777,7 +796,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
 
   return buf;
 }
-
+#if 0
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return 0;
   SMqPollRsp* pRsp = &tmq_message->msg;
@@ -827,11 +846,13 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
     }
   }
 }
+#endif
 
 int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   /*printf("recv poll\n");*/
   SMqPollCbParam* pParam = (SMqPollCbParam*)param;
   SMqClientVg*    pVg = pParam->pVg;
+  SMqClientTopic* pTopic = pParam->pTopic;
   tmq_t*          tmq = pParam->tmq;
   if (code != 0) {
     tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code);
@@ -874,18 +895,22 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
 #endif
 
   /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
-  tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
+  /*tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));*/
-  if (pRsp == NULL) {
+  SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper));
+  if (pRspWrapper == NULL) {
     tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
     goto CREATE_MSG_FAIL;
   }
-  memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
+  pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP;
-  tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg);
+  pRspWrapper->vgHandle = pVg;
-  /*pRsp->iter.curBlock = 0;*/
+  pRspWrapper->topicHandle = pTopic;
-  /*pRsp->iter.curRow = 0;*/
+  /*memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));*/
+  memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
+  tDecodeSMqPollRspV2(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
   // TODO: alloc mem
   /*pRsp->*/
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
+
 #if 0
   if (pRsp->msg.numOfTopics == 0) {
     /*printf("no data\n");*/
@@ -894,11 +919,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 #endif
 
-  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId,
+  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId,
-           pRsp->msg.reqOffset, pRsp->msg.rspOffset);
+           pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
 
-  pRsp->vg = pParam->pVg;
+  taosWriteQitem(tmq->mqueue, pRspWrapper);
-  taosWriteQitem(tmq->mqueue, pRsp);
   atomic_add_fetch_32(&tmq->readyRequest, 1);
   /*tsem_post(&tmq->rspSem);*/
   return 0;
@@ -1015,16 +1039,19 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     }
     tDeleteSMqCMGetSubEpRsp(&rsp);
   } else {
-    SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));
+    /*SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));*/
-    if (pRsp == NULL) {
+    SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper));
+    if (pWrapper == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       code = -1;
       goto END;
     }
-    memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
+    pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
-    tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp);
+    pWrapper->epoch = head->epoch;
+    memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
+    tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
 
-    taosWriteQitem(tmq->mqueue, pRsp);
+    taosWriteQitem(tmq->mqueue, pWrapper);
     /*tsem_post(&tmq->rspSem);*/
   }
 
@@ -1152,6 +1179,25 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
   return pReq;
 }
 
+SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
+  SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
+  pRspObj->resType = RES_TYPE__TMQ;
+  pRspObj->topic = strdup(pWrapper->topicHandle->topicName);
+  pRspObj->resIter = -1;
+  pRspObj->vgId = pWrapper->vgHandle->vgId;
+  SMqPollRspV2* pRsp = &pWrapper->msg;
+  int32_t       blockNum = taosArrayGetSize(pRsp->blockPos);
+  pRspObj->res = taosArrayInit(blockNum, sizeof(SReqResultInfo));
+  for (int32_t i = 0; i < blockNum; i++) {
+    int32_t            pos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
+    SRetrieveTableRsp* pRetrieve = POINTER_SHIFT(pRsp->blockData, pos);
+    SReqResultInfo     resInfo;
+    setQueryResultFromRsp(&resInfo, pRetrieve, true);
+    taosArrayPush(pRspObj->res, &resInfo);
+  }
+  return pRspObj;
+}
+
 #if 0
 tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
   tmq_message_t* msg = NULL;
@@ -1258,6 +1304,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       }
       pParam->tmq = tmq;
       pParam->pVg = pVg;
+      pParam->pTopic = pTopic;
       pParam->vgId = pVg->vgId;
       pParam->epoch = tmq->epoch;
       pParam->sync = 0;
@@ -1296,13 +1343,13 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
   return 0;
 }
 
-// return
+int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) {
-int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
+  if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) {
-  if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
     /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
-    if (rspHead->epoch > atomic_load_32(&tmq->epoch)) {
+    if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) {
-      SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead;
+      SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper;
-      tmqUpdateEp(tmq, rspHead->epoch, rspMsg);
+      SMqCMGetSubEpRsp*   rspMsg = &pEpRspWrapper->msg;
+      tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg);
       /*tmqClearUnhandleMsg(tmq);*/
       *pReset = true;
     } else {
@@ -1314,41 +1361,43 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
   return 0;
 }
 
-tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
+SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
   while (1) {
-    SMqRspHead* rspHead = NULL;
+    SMqRspWrapper* rspWrapper = NULL;
-    taosGetQitem(tmq->qall, (void**)&rspHead);
+    taosGetQitem(tmq->qall, (void**)&rspWrapper);
-    if (rspHead == NULL) {
+    if (rspWrapper == NULL) {
       taosReadAllQitems(tmq->mqueue, tmq->qall);
-      taosGetQitem(tmq->qall, (void**)&rspHead);
+      taosGetQitem(tmq->qall, (void**)&rspWrapper);
-      if (rspHead == NULL) return NULL;
+      if (rspWrapper == NULL) return NULL;
     }
 
-    if (rspHead->mqMsgType == TMQ_MSG_TYPE__POLL_RSP) {
+    if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
-      tmq_message_t* rspMsg = (tmq_message_t*)rspHead;
+      SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
       atomic_sub_fetch_32(&tmq->readyRequest, 1);
       /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
-      if (rspMsg->msg.head.epoch == atomic_load_32(&tmq->epoch)) {
+      if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) {
         /*printf("epoch match\n");*/
-        SMqClientVg* pVg = rspMsg->vg;
+        SMqClientVg* pVg = pollRspWrapper->vgHandle;
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
-        pVg->currentOffset = rspMsg->msg.rspOffset;
+        pVg->currentOffset = pollRspWrapper->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        if (rspMsg->msg.numOfTopics == 0) {
+        if (pollRspWrapper->msg.dataLen == 0) {
-          taosFreeQitem(rspMsg);
+          taosFreeQitem(pollRspWrapper);
-          rspHead = NULL;
+          rspWrapper = NULL;
           continue;
         }
-        return rspMsg;
+        // build msg
+        SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
+        return pRsp;
       } else {
         /*printf("epoch mismatch\n");*/
-        taosFreeQitem(rspMsg);
+        taosFreeQitem(pollRspWrapper);
       }
     } else {
       /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
       bool reset = false;
-      tmqHandleNoPollRsp(tmq, rspHead, &reset);
+      tmqHandleNoPollRsp(tmq, rspWrapper, &reset);
-      taosFreeQitem(rspHead);
+      taosFreeQitem(rspWrapper);
       if (pollIfReset && reset) {
         tscDebug("consumer %ld reset and repoll", tmq->consumerId);
         tmqPollImpl(tmq, blockingTime);
@@ -1382,17 +1431,17 @@ tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
 }
 #endif
 
-tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
+TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
-  tmq_message_t* rspMsg;
+  SMqRspObj* rspObj;
-  int64_t        startTime = taosGetTimestampMs();
+  int64_t    startTime = taosGetTimestampMs();
 
   // TODO: put into another thread or delayed queue
   int64_t status = atomic_load_64(&tmq->status);
   tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
 
-  rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
+  rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
-  if (rspMsg) {
+  if (rspObj) {
-    return rspMsg;
+    return (TAOS_RES*)rspObj;
   }
 
   while (1) {
@@ -1402,9 +1451,9 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
     /*tsem_wait(&tmq->rspSem);*/
 
-    rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
+    rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
-    if (rspMsg) {
+    if (rspObj) {
-      return rspMsg;
+      return (TAOS_RES*)rspObj;
     }
     if (blocking_time != 0) {
       int64_t endTime = taosGetTimestampMs();
@@ -1546,6 +1595,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
 }
 #endif
 
+#if 0
 void tmq_message_destroy(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return;
   SMqPollRsp* pRsp = &tmq_message->msg;
@@ -1553,6 +1603,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) {
   /*taosMemoryFree(tmq_message);*/
   taosFreeQitem(tmq_message);
 }
+#endif
 
 tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; }
 
@@ -1563,4 +1614,27 @@ const char* tmq_err2str(tmq_resp_err_t err) {
   return "fail";
 }
 
-char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; }
+char* tmq_get_topic_name(TAOS_RES* res) {
+  if (TD_RES_TMQ(res)) {
+    SMqRspObj* pRspObj = (SMqRspObj*)res;
+    return pRspObj->topic;
+  } else {
+    return NULL;
+  }
+}
+
+int32_t tmq_get_vgroup_id(TAOS_RES* res) {
+  if (TD_RES_TMQ(res)) {
+    SMqRspObj* pRspObj = (SMqRspObj*)res;
+    return pRspObj->vgId;
+  } else {
+    return -1;
+  }
+}
+
+void tmq_message_destroy(TAOS_RES* res) {
+  if (res == NULL) return;
+  if (TD_RES_TMQ(res)) {
+    SMqRspObj* pRspObj = (SMqRspObj*)res;
+  }
+}

source/common/src/tglobal.c

@@ -616,7 +616,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
 
   taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32);
 
-  if (taosMkDir(tsLogDir) != 0) {
+  if (taosMulMkDir(tsLogDir) != 0) {
     uError("failed to create dir:%s since %s", tsLogDir, terrstr());
     cfgCleanup(pCfg);
     return -1;

source/common/src/ttime.c

@@ -406,7 +406,31 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
     default: {
       return -1;
     }
-  }  
+  }
+}
+
+int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) {
+  int32_t charLen = varDataLen(inputData);
+  char *newColData;
+  if (type == TSDB_DATA_TYPE_BINARY) {
+    newColData = taosMemoryCalloc(1,  charLen + 1);
+    memcpy(newColData, varDataVal(inputData), charLen);
+    taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0);
+    taosMemoryFree(newColData);
+  } else if (type == TSDB_DATA_TYPE_NCHAR) {
+    newColData = taosMemoryCalloc(1,  charLen / TSDB_NCHAR_SIZE + 1);
+    int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inputData), charLen, newColData);
+    if (len < 0){
+      taosMemoryFree(newColData);
+      return TSDB_CODE_FAILED;
+    }
+    newColData[len] = 0;
+    taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0);
+    taosMemoryFree(newColData);
+  } else {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {

source/dnode/mgmt/CMakeLists.txt

@@ -20,9 +20,17 @@ add_executable(taosd ${EXEC_SRC})
 target_include_directories(
     taosd
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc"
 )
 target_link_libraries(taosd dnode)
 
+IF (TD_GRANT)
+  TARGET_LINK_LIBRARIES(taosd grant)
+ENDIF ()
+IF (TD_USB_DONGLE)
+  TARGET_LINK_LIBRARIES(taosd usb_dongle)
+ENDIF ()
+
 if(${BUILD_TEST})
     add_subdirectory(test)
 endif(${BUILD_TEST})

source/dnode/mgmt/exe/dndMain.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
 #include "tconfig.h"
+#include "mndGrant.h"
 
 static struct {
   bool     dumpConfig;
@@ -90,8 +91,7 @@ static int32_t dndParseArgs(int32_t argc, char const *argv[]) {
 }
 
 static void dndGenerateGrant() {
-  // grantParseParameter();
+  grantParseParameter();
-  printf("this feature is not implemented yet\n");
 }
 
 static void dndPrintVersion() {

source/dnode/mnode/impl/CMakeLists.txt

@@ -9,6 +9,13 @@ target_link_libraries(
     mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser
 )
 
+IF (TD_GRANT)
+  TARGET_LINK_LIBRARIES(mnode grant)
+ENDIF ()
+IF (TD_USB_DONGLE)
+  TARGET_LINK_LIBRARIES(mnode usb_dongle)
+ENDIF ()
+
 if(${BUILD_TEST})
     add_subdirectory(test)
 endif(${BUILD_TEST})

source/dnode/mnode/impl/inc/mndGrant.h

@@ -36,8 +36,8 @@ typedef enum {
   TSDB_GRANT_CPU_CORES,
 } EGrantType;
 
-int32_t grantInit();
+int32_t mndInitGrant();
-void    grantCleanUp();
+void    mndCleanupGrant();
 void    grantParseParameter();
 int32_t grantCheck(EGrantType grant);
 void    grantReset(EGrantType grant, uint64_t value);

source/dnode/mnode/impl/src/mndGrant.c

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#ifndef _GRANT
+#include "os.h"
+#include "taoserror.h"
+#include "mndGrant.h"
+#include "mndInt.h"
+
+int32_t mndInitGrant(SMnode *pMnode) { return TSDB_CODE_SUCCESS; }
+void    mndCleanupGrant() {}
+void    grantParseParameter() { mError("can't parsed parameter k"); }
+int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; }
+void    grantReset(EGrantType grant, uint64_t value) {}
+void    grantAdd(EGrantType grant, uint64_t value) {}
+void    grantRestore(EGrantType grant, uint64_t value) {}
+
+#endif

source/dnode/mnode/impl/src/mnode.c

@@ -40,6 +40,7 @@
 #include "mndUser.h"
 #include "mndVgroup.h"
 #include "mndQuery.h"
+#include "mndGrant.h"
 
 #define MQ_TIMER_MS    3000
 #define TRNAS_TIMER_MS 6000
@@ -197,6 +198,7 @@ static int32_t mndInitSteps(SMnode *pMnode, bool deploy) {
   if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1;

source/dnode/vnode/CMakeLists.txt

@@ -30,6 +30,7 @@ target_sources(
 
     # tsdb
     # "src/tsdb/tsdbBDBImpl.c"
+    "src/tsdb/tsdbTDBImpl.c"
     "src/tsdb/tsdbCommit.c"
     "src/tsdb/tsdbCompact.c"
     "src/tsdb/tsdbFile.c"
@@ -40,7 +41,7 @@ target_sources(
     "src/tsdb/tsdbRead.c"
     "src/tsdb/tsdbReadImpl.c"
     "src/tsdb/tsdbScan.c"
-    # "src/tsdb/tsdbSma.c"
+    "src/tsdb/tsdbSma.c"
     "src/tsdb/tsdbWrite.c"
 
     # tq

source/dnode/vnode/inc/vnode.h

@@ -357,7 +357,7 @@ STbCfg         *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
 STbCfg         *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema       *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
-STSma          *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid);
+void           *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode);
 STSmaWrapper   *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid);
 SArray         *metaGetSmaTbUids(SMeta *pMeta, bool isDup);
 int             metaGetTbNum(SMeta *pMeta);
@@ -369,8 +369,8 @@ void            metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
 tb_uid_t        metaCtbCursorNext(SMCtbCursor *pCtbCur);
 
 SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
-void         metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
+void         metaCloseSmaCursor(SMSmaCursor *pSmaCur);
-const char  *metaSmaCursorNext(SMSmaCursor *pSmaCur);
+int64_t      metaSmaCursorNext(SMSmaCursor *pSmaCur);
 
 // Options
 void metaOptionsInit(SMetaCfg *pMetaCfg);

source/dnode/vnode/src/inc/tsdbSma.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_TSDB_SMA_H_
+#define _TD_VNODE_TSDB_SMA_H_
+
+#include "tdbInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SSmaKey SSmaKey;
+
+struct SSmaKey {
+  TSKEY   skey;
+  int64_t groupId;
+};
+
+
+typedef struct SDBFile SDBFile;
+
+struct SDBFile {
+  int32_t fid;
+  TDB    *pDB;
+  char   *path;
+};
+
+int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path);
+int32_t tsdbCloseDBEnv(TENV *pEnv);
+int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF);
+int32_t tsdbCloseDBF(SDBFile *pDBF);
+int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
+void   *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
+
+void  tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
+void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
+#if 0
+int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
+int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
+#endif
+
+// internal func
+static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
+  int32_t len = 0;
+  len += taosEncodeFixedI64(pData, tsKey);
+  len += taosEncodeFixedI64(pData, groupId);
+  return len;
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_TSDB_SMA_H_*/

source/dnode/vnode/src/inc/vnodeInt.h

@@ -220,6 +220,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
 #include "tq.h"
 
+#include "tsdbSma.h"
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -667,7 +667,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
   return pTbCfg;
 }
 
-STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
+void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
   STSma *  pCfg = NULL;
   SMetaDB *pDB = pMeta->pDB;
   DBT      key = {0};
@@ -920,7 +920,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
   return pCur;
 }
 
-void metaCloseSmaCurosr(SMSmaCursor *pCur) {
+void metaCloseSmaCursor(SMSmaCursor *pCur) {
   if (pCur) {
     if (pCur->pCur) {
       pCur->pCur->close(pCur->pCur);
@@ -930,7 +930,8 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) {
   }
 }
 
-const char *metaSmaCursorNext(SMSmaCursor *pCur) {
+int64_t metaSmaCursorNext(SMSmaCursor *pCur) {
+#if 0
   DBT skey = {0};
   DBT pkey = {0};
   DBT pval = {0};
@@ -946,6 +947,8 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) {
   } else {
     return NULL;
   }
+#endif
+  return 0;
 }
 
 STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
@@ -972,7 +975,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
       ++pSW->number;
       STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
       if (tptr == NULL) {
-        metaCloseSmaCurosr(pCur);
+        metaCloseSmaCursor(pCur);
         tdDestroyTSmaWrapper(pSW);
         taosMemoryFreeClear(pSW);
         return NULL;
@@ -980,7 +983,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
       pSW->tSma = tptr;
       pBuf = pval.data;
       if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
-        metaCloseSmaCurosr(pCur);
+        metaCloseSmaCursor(pCur);
         tdDestroyTSmaWrapper(pSW);
         taosMemoryFreeClear(pSW);
         return NULL;
@@ -990,8 +993,8 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
     break;
   }
 
-  metaCloseSmaCurosr(pCur);
+  metaCloseSmaCursor(pCur);
-  
+
   return pSW;
 }
 
@@ -1004,7 +1007,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
   int      ret;
 
   // TODO: lock?
-  ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0);
+  ret = pDB->pSmaIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0);
   if (ret != 0) {
     return NULL;
   }

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -22,22 +22,28 @@ typedef struct SPoolMem {
   struct SPoolMem *next;
 } SPoolMem;
 
+#define META_TDB_SMA_TEST
+
 static SPoolMem *openPool();
 static void      clearPool(SPoolMem *pPool);
 static void      closePool(SPoolMem *pPool);
-static void     *poolMalloc(void *arg, size_t size);
+static void *    poolMalloc(void *arg, size_t size);
 static void      poolFree(void *arg, void *ptr);
 
 struct SMetaDB {
   TXN       txn;
-  TENV     *pEnv;
+  TENV *    pEnv;
-  TDB      *pTbDB;
+  TDB *     pTbDB;
-  TDB      *pSchemaDB;
+  TDB *     pSchemaDB;
-  TDB      *pNameIdx;
+  TDB *     pNameIdx;
-  TDB      *pStbIdx;
+  TDB *     pStbIdx;
-  TDB      *pNtbIdx;
+  TDB *     pNtbIdx;
-  TDB      *pCtbIdx;
+  TDB *     pCtbIdx;
   SPoolMem *pPool;
+#ifdef META_TDB_SMA_TEST
+  TDB *pSmaDB;
+  TDB *pSmaIdx;
+#endif
 };
 
 typedef struct __attribute__((__packed__)) {
@@ -46,7 +52,7 @@ typedef struct __attribute__((__packed__)) {
 } SSchemaDbKey;
 
 typedef struct {
-  char    *name;
+  char *   name;
   tb_uid_t uid;
 } SNameIdxKey;
 
@@ -55,6 +61,11 @@ typedef struct {
   tb_uid_t uid;
 } SCtbIdxKey;
 
+typedef struct {
+  tb_uid_t uid;
+  int64_t  smaUid;
+} SSmaIdxKey;
+
 static int   metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
 static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
 static int   metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
@@ -115,6 +126,17 @@ static inline int metaCtbIdxCmpr(const void *arg1, int len1, const void *arg2, i
   return metaUidCmpr(&pKey1->uid, sizeof(tb_uid_t), &pKey2->uid, sizeof(tb_uid_t));
 }
 
+static inline int metaSmaIdxCmpr(const void *arg1, int len1, const void *arg2, int len2) {
+  int         c;
+  SSmaIdxKey *pKey1 = (SSmaIdxKey *)arg1;
+  SSmaIdxKey *pKey2 = (SSmaIdxKey *)arg2;
+
+  c = metaUidCmpr(arg1, sizeof(tb_uid_t), arg2, sizeof(tb_uid_t));
+  if (c) return c;
+
+  return metaUidCmpr(&pKey1->smaUid, sizeof(int64_t), &pKey2->smaUid, sizeof(int64_t));
+}
+
 int metaOpenDB(SMeta *pMeta) {
   SMetaDB *pMetaDb;
   int      ret;
@@ -143,6 +165,15 @@ int metaOpenDB(SMeta *pMeta) {
     return -1;
   }
 
+#ifdef META_TDB_SMA_TEST
+  ret = tdbDbOpen("sma.db", sizeof(int64_t), TDB_VARIANT_LEN, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pSmaDB));
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return -1;
+  }
+#endif
+
   // open schema DB
   ret = tdbDbOpen("schema.db", sizeof(SSchemaDbKey), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv,
                   &(pMetaDb->pSchemaDB));
@@ -180,6 +211,15 @@ int metaOpenDB(SMeta *pMeta) {
     return -1;
   }
 
+#ifdef META_TDB_SMA_TEST
+  ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, metaSmaIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pSmaIdx));
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return -1;
+  }
+#endif
+
   pMetaDb->pPool = openPool();
   tdbTxnOpen(&pMetaDb->txn, 0, poolMalloc, poolFree, pMetaDb->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
   tdbBegin(pMetaDb->pEnv, NULL);
@@ -193,10 +233,16 @@ void metaCloseDB(SMeta *pMeta) {
     tdbCommit(pMeta->pDB->pEnv, &pMeta->pDB->txn);
     tdbTxnClose(&pMeta->pDB->txn);
     clearPool(pMeta->pDB->pPool);
+#ifdef META_TDB_SMA_TEST
+    tdbDbClose(pMeta->pDB->pSmaIdx);
+#endif
     tdbDbClose(pMeta->pDB->pCtbIdx);
     tdbDbClose(pMeta->pDB->pNtbIdx);
     tdbDbClose(pMeta->pDB->pStbIdx);
     tdbDbClose(pMeta->pDB->pNameIdx);
+#ifdef META_TDB_SMA_TEST
+    tdbDbClose(pMeta->pDB->pSmaDB);
+#endif
     tdbDbClose(pMeta->pDB->pSchemaDB);
     tdbDbClose(pMeta->pDB->pTbDB);
     taosMemoryFree(pMeta->pDB);
@@ -205,14 +251,14 @@ void metaCloseDB(SMeta *pMeta) {
 
 int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
   tb_uid_t       uid;
-  SMetaDB       *pMetaDb;
+  SMetaDB *      pMetaDb;
-  void          *pKey;
+  void *         pKey;
-  void          *pVal;
+  void *         pVal;
   int            kLen;
   int            vLen;
   int            ret;
   char           buf[512];
-  void          *pBuf;
+  void *         pBuf;
   SCtbIdxKey     ctbIdxKey;
   SSchemaDbKey   schemaDbKey;
   SSchemaWrapper schemaWrapper;
@@ -329,11 +375,11 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
 STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
   int      ret;
   SMetaDB *pMetaDb = pMeta->pDB;
-  void    *pKey;
+  void *   pKey;
-  void    *pVal;
+  void *   pVal;
   int      kLen;
   int      vLen;
-  STbCfg  *pTbCfg;
+  STbCfg * pTbCfg;
 
   // Fetch
   pKey = &uid;
@@ -385,14 +431,14 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
 }
 
 static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
-  void           *pKey;
+  void *          pKey;
-  void           *pVal;
+  void *          pVal;
   int             kLen;
   int             vLen;
   int             ret;
   SSchemaDbKey    schemaDbKey;
   SSchemaWrapper *pSchemaWrapper;
-  void           *pBuf;
+  void *          pBuf;
 
   // fetch
   schemaDbKey.uid = uid;
@@ -419,9 +465,9 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   tb_uid_t        quid;
   SSchemaWrapper *pSW;
   STSchemaBuilder sb;
-  SSchemaEx      *pSchema;
+  SSchemaEx *     pSchema;
-  STSchema       *pTSchema;
+  STSchema *      pTSchema;
-  STbCfg         *pTbCfg;
+  STbCfg *        pTbCfg;
 
   pTbCfg = metaGetTbInfoByUid(pMeta, uid);
   if (pTbCfg->type == META_CHILD_TABLE) {
@@ -452,7 +498,7 @@ struct SMTbCursor {
 
 SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
   SMTbCursor *pTbCur = NULL;
-  SMetaDB    *pDB = pMeta->pDB;
+  SMetaDB *   pDB = pMeta->pDB;
 
   pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
   if (pTbCur == NULL) {
@@ -474,12 +520,12 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
 }
 
 char *metaTbCursorNext(SMTbCursor *pTbCur) {
-  void  *pKey = NULL;
+  void * pKey = NULL;
-  void  *pVal = NULL;
+  void * pVal = NULL;
   int    kLen;
   int    vLen;
   int    ret;
-  void  *pBuf;
+  void * pBuf;
   STbCfg tbCfg;
 
   for (;;) {
@@ -491,7 +537,6 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
       taosMemoryFree(tbCfg.name);
       taosMemoryFree(tbCfg.stbCfg.pTagSchema);
       continue;
-      ;
     } else if (tbCfg.type == META_CHILD_TABLE) {
       kvRowFree(tbCfg.ctbCfg.pTag);
     }
@@ -503,17 +548,17 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
 }
 
 struct SMCtbCursor {
-  TDBC    *pCur;
+  TDBC *   pCur;
   tb_uid_t suid;
-  void    *pKey;
+  void *   pKey;
-  void    *pVal;
+  void *   pVal;
   int      kLen;
   int      vLen;
 };
 
 SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
   SMCtbCursor *pCtbCur = NULL;
-  SMetaDB     *pDB = pMeta->pDB;
+  SMetaDB *    pDB = pMeta->pDB;
   int          ret;
 
   pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
@@ -566,51 +611,326 @@ int metaGetTbNum(SMeta *pMeta) {
   return 0;
 }
 
+struct SMSmaCursor {
+  TDBC    *pCur;
+  tb_uid_t uid;
+  void    *pKey;
+  void    *pVal;
+  int      kLen;
+  int      vLen;
+};
+
 STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
   // TODO
-  ASSERT(0);
+  // ASSERT(0);
-  return NULL;
+  // return NULL;
+#ifdef META_TDB_SMA_TEST
+  STSmaWrapper *pSW = NULL;
+
+  pSW = taosMemoryCalloc(1, sizeof(*pSW));
+  if (pSW == NULL) {
+    return NULL;
+  }
+
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
+  if (pCur == NULL) {
+    taosMemoryFree(pSW);
+    return NULL;
+  }
+
+  void       *pBuf = NULL;
+  SSmaIdxKey *pSmaIdxKey = NULL;
+
+  while (true) {
+    // TODO: lock during iterate?
+    if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
+      pSmaIdxKey = pCur->pKey;
+      ASSERT(pSmaIdxKey != NULL);
+
+      void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false);
+
+      if (pSmaVal == NULL) {
+        tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid);
+        continue;
+      }
+
+
+      ++pSW->number;
+      STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
+      if (tptr == NULL) {
+        TDB_FREE(pSmaVal);
+        metaCloseSmaCursor(pCur);
+        tdDestroyTSmaWrapper(pSW);
+        taosMemoryFreeClear(pSW);
+        return NULL;
+      }
+      pSW->tSma = tptr;
+      pBuf = pSmaVal;
+      if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
+        TDB_FREE(pSmaVal);
+        metaCloseSmaCursor(pCur);
+        tdDestroyTSmaWrapper(pSW);
+        taosMemoryFreeClear(pSW);
+        return NULL;
+      }
+      TDB_FREE(pSmaVal);
+      continue;
+    }
+    break;
+  }
+
+  metaCloseSmaCursor(pCur);
+
+  return pSW;
+
+#endif
 }
 
 int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) {
   // TODO
   ASSERT(0);
+#ifndef META_TDB_SMA_TEST
+  DBT key = {0};
+
+  key.data = (void *)indexName;
+  key.size = strlen(indexName);
+
+  metaDBWLock(pMeta->pDB);
+  // TODO: No guarantee of consistence.
+  // Use transaction or DB->sync() for some guarantee.
+  pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0);
+  metaDBULock(pMeta->pDB);
+#endif
   return 0;
 }
 
 int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   // TODO
-  ASSERT(0);
+  // ASSERT(0);
+
+#ifdef META_TDB_SMA_TEST
+  int32_t ret = 0;
+  SMetaDB *pMetaDb = pMeta->pDB;
+  void   *pBuf = NULL, *qBuf = NULL;
+  void   *key = {0}, *val = {0};
+
+  // save sma info
+  int32_t len = tEncodeTSma(NULL, pSmaCfg);
+  pBuf = taosMemoryCalloc(1, len);
+  if (pBuf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  key = (void *)&pSmaCfg->indexUid;
+  qBuf = pBuf;
+  tEncodeTSma(&qBuf, pSmaCfg);
+  val = pBuf;
+
+  int32_t kLen = sizeof(pSmaCfg->indexUid);
+  int32_t vLen = POINTER_DISTANCE(qBuf, pBuf);
+
+  ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn);
+  if (ret < 0) {
+    taosMemoryFreeClear(pBuf);
+    return -1;
+  }
+
+  // add sma idx
+  SSmaIdxKey smaIdxKey;
+  smaIdxKey.uid = pSmaCfg->tableUid;
+  smaIdxKey.smaUid = pSmaCfg->indexUid;
+  key = &smaIdxKey;
+  kLen = sizeof(smaIdxKey);
+  val = NULL;
+  vLen = 0;
+
+  ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn);
+  if (ret < 0) {
+    taosMemoryFreeClear(pBuf);
+    return -1;
+  }
+
+  // release
+  taosMemoryFreeClear(pBuf);
+
+  if (pMeta->pDB->pPool->size > 0) {
+    metaCommit(pMeta);
+  }
+
+#endif
   return 0;
 }
 
-STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
+void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
   // TODO
-  ASSERT(0);
+  // ASSERT(0);
-  return NULL;
+  // return NULL;
+#ifdef META_TDB_SMA_TEST
+  SMetaDB *pDB = pMeta->pDB;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
+  int      kLen = 0;
+  int      vLen = 0;
+  int      ret = -1;
+
+  // Set key
+  pKey = (void *)&indexUid;
+  kLen = sizeof(indexUid);
+
+  // Query
+  ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
+  if (ret != 0 || !pVal) {
+    return NULL;
+  }
+
+  if (!isDecode) {
+    // return raw value
+    return pVal;
+  }
+
+  // Decode
+  STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
+  if (pCfg == NULL) {
+    taosMemoryFree(pVal);
+    return NULL;
+  }
+
+  void *pBuf = pVal;
+  if (tDecodeTSma(pBuf, pCfg) == NULL) {
+    tdDestroyTSma(pCfg);
+    taosMemoryFree(pCfg);
+    TDB_FREE(pVal);
+    return NULL;
+  }
+
+  TDB_FREE(pVal);
+  return pCfg;
+#endif
 }
 
-const char *metaSmaCursorNext(SMSmaCursor *pCur) {
+/**
+ * @brief
+ *
+ * @param pMeta
+ * @param uid 0 means iterate all uids.
+ * @return SMSmaCursor*
+ */
+SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
   // TODO
-  ASSERT(0);
+  // ASSERT(0);
-  return NULL;
+  // return NULL;
+#ifdef META_TDB_SMA_TEST
+  SMSmaCursor *pCur = NULL;
+  SMetaDB     *pDB = pMeta->pDB;
+  int          ret;
+
+  pCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pCur));
+  if (pCur == NULL) {
+    return NULL;
+  }
+
+  pCur->uid = uid;
+  ret = tdbDbcOpen(pDB->pSmaIdx, &(pCur->pCur));
+  if ((ret != 0) || (pCur->pCur == NULL)) {
+    taosMemoryFree(pCur);
+    return NULL;
+  }
+
+  if (uid != 0) {
+    // TODO: move to the specific uid
+  }
+
+  return pCur;
+#endif
 }
 
-void metaCloseSmaCurosr(SMSmaCursor *pCur) {
+/**
+ * @brief
+ *
+ * @param pCur
+ * @return int64_t smaIndexUid
+ */
+int64_t metaSmaCursorNext(SMSmaCursor *pCur) {
   // TODO
-  ASSERT(0);
+  // ASSERT(0);
+  // return NULL;
+#ifdef META_TDB_SMA_TEST
+  int         ret;
+  void       *pBuf;
+  SSmaIdxKey *smaIdxKey;
+
+  ret = tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, &pCur->pVal, &pCur->vLen);
+  if (ret < 0) {
+    return 0;
+  }
+  smaIdxKey = pCur->pKey;
+  return smaIdxKey->smaUid;
+#endif
+}
+
+void metaCloseSmaCursor(SMSmaCursor *pCur) {
+  // TODO
+  // ASSERT(0);
+#ifdef META_TDB_SMA_TEST
+  if (pCur) {
+    if (pCur->pCur) {
+      tdbDbcClose(pCur->pCur);
+    }
+
+    taosMemoryFree(pCur);
+  }
+#endif
 }
 
 SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
   // TODO
   // ASSERT(0); // comment this line to pass CI
-  return NULL;
+  // return NULL:
-}
+#ifdef META_TDB_SMA_TEST
+  SArray  *pUids = NULL;
+  SMetaDB *pDB = pMeta->pDB;
+  void    *pKey;
 
-SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
+  // TODO: lock?
-  // TODO
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
-  ASSERT(0);
+  if (pCur == NULL) {
-  return NULL;
+    return NULL;
+  }
+  // TODO: lock?
+
+  SSmaIdxKey *pSmaIdxKey = NULL;
+  tb_uid_t    uid = 0;
+  while (true) {
+    // TODO: lock during iterate?
+    if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
+      ASSERT(pSmaIdxKey != NULL);
+      pSmaIdxKey = pCur->pKey;
+
+      if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
+        continue;
+      }
+      uid = pSmaIdxKey->uid;
+
+      if (!pUids) {
+        pUids = taosArrayInit(16, sizeof(tb_uid_t));
+        if (!pUids) {
+          metaCloseSmaCursor(pCur);
+          return NULL;
+        }
+      }
+
+      taosArrayPush(pUids, &uid);
+
+      continue;
+    }
+    break;
+  }
+
+  metaCloseSmaCursor(pCur);
+
+  return pUids;
+#endif
 }
 
 static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
@@ -621,6 +941,7 @@ static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
   for (int i = 0; i < pSW->nCols; i++) {
     pSchema = pSW->pSchema + i;
     tlen += taosEncodeFixedI8(buf, pSchema->type);
+    tlen += taosEncodeFixedI8(buf, pSchema->index);
     tlen += taosEncodeFixedI16(buf, pSchema->colId);
     tlen += taosEncodeFixedI32(buf, pSchema->bytes);
     tlen += taosEncodeString(buf, pSchema->name);
@@ -637,6 +958,7 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
   for (int i = 0; i < pSW->nCols; i++) {
     pSchema = pSW->pSchema + i;
     buf = taosDecodeFixedI8(buf, &pSchema->type);
+    buf = taosSkipFixedLen(buf, sizeof(int8_t));
     buf = taosDecodeFixedI16(buf, &pSchema->colId);
     buf = taosDecodeFixedI32(buf, &pSchema->bytes);
     buf = taosDecodeStringTo(buf, pSchema->name);
@@ -781,7 +1103,7 @@ static void closePool(SPoolMem *pPool) {
 }
 
 static void *poolMalloc(void *arg, size_t size) {
-  void     *ptr = NULL;
+  void *    ptr = NULL;
   SPoolMem *pPool = (SPoolMem *)arg;
   SPoolMem *pMem;
 

source/dnode/vnode/src/tq/tq.c

@@ -82,9 +82,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
   memcpy(data, msg, msgLen);
 
   if (msgType == TDMT_VND_SUBMIT) {
-    // if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) {
+    if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) {
-    //   return -1;
+      return -1;
-    // }
+    }
   }
 
   SRpcMsg req = {
@@ -255,7 +255,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 
   return 0;
 }
-
+#if 0
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
@@ -433,6 +433,205 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   return 0;
 }
+#endif
+
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
+  SMqPollReq* pReq = pMsg->pCont;
+  int64_t     consumerId = pReq->consumerId;
+  int64_t     fetchOffset;
+  int64_t     blockingTime = pReq->blockingTime;
+  int32_t     reqEpoch = pReq->epoch;
+
+  if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
+    fetchOffset = 0;
+  } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
+    fetchOffset = walGetLastVer(pTq->pWal);
+  } else {
+    fetchOffset = pReq->currentOffset + 1;
+  }
+
+  vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
+         pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
+
+  SMqPollRspV2 rspV2 = {0};
+  rspV2.dataLen = 0;
+
+  STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
+  if (pConsumer == NULL) {
+    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, pTq->pVnode->vgId);
+    pMsg->pCont = NULL;
+    pMsg->contLen = 0;
+    pMsg->code = -1;
+    tmsgSendRsp(pMsg);
+    return 0;
+  }
+
+  int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch);
+  while (consumerEpoch < reqEpoch) {
+    consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch);
+  }
+
+  STqTopic* pTopic = NULL;
+  int32_t   topicSz = taosArrayGetSize(pConsumer->topics);
+  for (int32_t i = 0; i < topicSz; i++) {
+    STqTopic* topic = taosArrayGet(pConsumer->topics, i);
+    // TODO race condition
+    ASSERT(pConsumer->consumerId == consumerId);
+    if (strcmp(topic->topicName, pReq->topic) == 0) {
+      pTopic = topic;
+      break;
+    }
+  }
+  if (pTopic == NULL) {
+    vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
+          pTq->pVnode->vgId);
+    pMsg->pCont = NULL;
+    pMsg->contLen = 0;
+    pMsg->code = -1;
+    tmsgSendRsp(pMsg);
+    return 0;
+  }
+
+  vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
+         pTq->pVnode->vgId);
+
+  rspV2.reqOffset = pReq->currentOffset;
+  rspV2.skipLogNum = 0;
+
+  while (1) {
+    /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
+    // TODO
+    consumerEpoch = atomic_load_32(&pConsumer->epoch);
+    if (consumerEpoch > reqEpoch) {
+      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
+             consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset, consumerEpoch, reqEpoch);
+      break;
+    }
+    SWalReadHead* pHead;
+    if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
+      // TODO: no more log, set timer to wait blocking time
+      // if data inserted during waiting, launch query and
+      // response to user
+      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
+             pTq->pVnode->vgId, fetchOffset);
+      break;
+    }
+    vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
+           pTq->pVnode->vgId, fetchOffset, pHead->msgType);
+    /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
+    /*pHead = pTopic->pReadhandle->pHead;*/
+    if (pHead->msgType == TDMT_VND_SUBMIT) {
+      SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
+      qTaskInfo_t task = pTopic->buffer.output[workerId].task;
+      ASSERT(task);
+      qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+      SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
+      while (1) {
+        SSDataBlock* pDataBlock = NULL;
+        uint64_t     ts;
+        if (qExecTask(task, &pDataBlock, &ts) < 0) {
+          ASSERT(false);
+        }
+        if (pDataBlock == NULL) {
+          /*pos = fetchOffset % TQ_BUFFER_SIZE;*/
+          break;
+        }
+
+        taosArrayPush(pRes, pDataBlock);
+      }
+
+      if (taosArrayGetSize(pRes) == 0) {
+        vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
+               pReq->epoch, pTq->pVnode->vgId, fetchOffset);
+        fetchOffset++;
+        rspV2.skipLogNum++;
+        taosArrayDestroy(pRes);
+        continue;
+      }
+      rspV2.rspOffset = fetchOffset;
+
+      int32_t blockSz = taosArrayGetSize(pRes);
+      int32_t tlen = 0;
+      for (int32_t i = 0; i < blockSz; i++) {
+        SSDataBlock* pBlock = taosArrayGet(pRes, i);
+        tlen += sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+      }
+
+      void* data = taosMemoryMalloc(tlen);
+      if (data == NULL) {
+        pMsg->code = -1;
+        taosMemoryFree(pHead);
+      }
+
+      rspV2.blockData = data;
+
+      void*   dataBlockBuf = data;
+      int32_t pos;
+      for (int32_t i = 0; i < blockSz; i++) {
+        pos = 0;
+        SSDataBlock* pBlock = taosArrayGet(pRes, i);
+        blockCompressEncode(pBlock, dataBlockBuf, &pos, pBlock->info.numOfCols, false);
+        taosArrayPush(rspV2.blockPos, &rspV2.dataLen);
+        rspV2.dataLen += pos;
+        dataBlockBuf = POINTER_SHIFT(dataBlockBuf, pos);
+      }
+
+      int32_t msgLen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2);
+      void*   buf = rpcMallocCont(msgLen);
+
+      ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
+      ((SMqRspHead*)buf)->epoch = pReq->epoch;
+      ((SMqRspHead*)buf)->consumerId = consumerId;
+
+      void* msgBodyBuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+      tEncodeSMqPollRspV2(&msgBodyBuf, &rspV2);
+
+      /*rsp.pBlockData = pRes;*/
+
+      /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/
+      pMsg->pCont = buf;
+      pMsg->contLen = tlen;
+      pMsg->code = 0;
+      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", pTq->pVnode->vgId, fetchOffset,
+             pHead->msgType, consumerId, pReq->epoch);
+      tmsgSendRsp(pMsg);
+      taosMemoryFree(pHead);
+      return 0;
+    } else {
+      taosMemoryFree(pHead);
+      fetchOffset++;
+      rspV2.skipLogNum++;
+    }
+  }
+
+  /*if (blockingTime != 0) {*/
+  /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
+  /*} else {*/
+
+  rspV2.rspOffset = fetchOffset - 1;
+
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2);
+  void*   buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    pMsg->code = -1;
+    return -1;
+  }
+  ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
+  ((SMqRspHead*)buf)->epoch = pReq->epoch;
+  ((SMqRspHead*)buf)->consumerId = consumerId;
+
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+  tEncodeSMqPollRspV2(&abuf, &rspV2);
+  pMsg->pCont = buf;
+  pMsg->contLen = tlen;
+  pMsg->code = 0;
+  tmsgSendRsp(pMsg);
+  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", pTq->pVnode->vgId, fetchOffset, consumerId,
+         pReq->epoch);
+  /*}*/
+
+  return 0;
+}
 
 int32_t tqProcessRebReq(STQ* pTq, char* msg) {
   SMqMVRebReq req = {0};

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -38,6 +38,29 @@ typedef enum {
   SMA_STORAGE_LEVEL_DFILESET = 1  // use days of TS data       e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma
 } ESmaStorageLevel;
 
+typedef struct SPoolMem {
+  int64_t          size;
+  struct SPoolMem *prev;
+  struct SPoolMem *next;
+} SPoolMem;
+
+struct SSmaEnv {
+  TdThreadRwlock lock;
+  TXN            txn;
+  SPoolMem      *pPool;
+  SDiskID        did;
+  TENV          *dbEnv;  // TODO: If it's better to put it in smaIndex level?
+  char          *path;   // relative path
+  SSmaStat      *pStat;
+};
+
+#define SMA_ENV_LOCK(env)       ((env)->lock)
+#define SMA_ENV_DID(env)        ((env)->did)
+#define SMA_ENV_ENV(env)        ((env)->dbEnv)
+#define SMA_ENV_PATH(env)       ((env)->path)
+#define SMA_ENV_STAT(env)       ((env)->pStat)
+#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
+
 typedef struct {
   STsdb        *pTsdb;
   SDBFile       dFile;
@@ -104,7 +127,8 @@ static void    tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH);
 static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit);
 static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit);
 static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid);
-static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen);
+static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
+                                    TXN *txn);
 static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
 static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel);
 static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid);
@@ -117,9 +141,121 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg);
 // mgmt interface
 static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid);
 
+// Pool Memory
+static SPoolMem *openPool();
+static void      clearPool(SPoolMem *pPool);
+static void      closePool(SPoolMem *pPool);
+static void     *poolMalloc(void *arg, size_t size);
+static void      poolFree(void *arg, void *ptr);
+
+static int tsdbSmaBeginCommit(SSmaEnv *pEnv);
+static int tsdbSmaEndCommit(SSmaEnv *pEnv);
+
 // implementation
-static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); }
+static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) {
-static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); }
+  return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n);
+}
+static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) {
+  return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n);
+}
+
+static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
+  int code = taosThreadRwlockRdlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
+  int code = taosThreadRwlockWrlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
+  int code = taosThreadRwlockUnlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static SPoolMem *openPool() {
+  SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool));
+
+  pPool->prev = pPool->next = pPool;
+  pPool->size = 0;
+
+  return pPool;
+}
+
+static void clearPool(SPoolMem *pPool) {
+  if (!pPool) return;
+  
+  SPoolMem *pMem;
+
+  do {
+    pMem = pPool->next;
+
+    if (pMem == pPool) break;
+
+    pMem->next->prev = pMem->prev;
+    pMem->prev->next = pMem->next;
+    pPool->size -= pMem->size;
+
+    tdbOsFree(pMem);
+  } while (1);
+
+  assert(pPool->size == 0);
+}
+
+static void closePool(SPoolMem *pPool) {
+  if (pPool) {
+    clearPool(pPool);
+    tdbOsFree(pPool);
+  }
+}
+
+static void *poolMalloc(void *arg, size_t size) {
+  void     *ptr = NULL;
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size);
+  if (pMem == NULL) {
+    assert(0);
+  }
+
+  pMem->size = sizeof(*pMem) + size;
+  pMem->next = pPool->next;
+  pMem->prev = pPool;
+
+  pPool->next->prev = pMem;
+  pPool->next = pMem;
+  pPool->size += pMem->size;
+
+  ptr = (void *)(&pMem[1]);
+  return ptr;
+}
+
+static void poolFree(void *arg, void *ptr) {
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = &(((SPoolMem *)ptr)[-1]);
+
+  pMem->next->prev = pMem->prev;
+  pMem->prev->next = pMem->next;
+  pPool->size -= pMem->size;
+
+  tdbOsFree(pMem);
+}
 
 int32_t tsdbInitSma(STsdb *pTsdb) {
   // tSma
@@ -213,7 +349,12 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did)
 
   char aname[TSDB_FILENAME_LEN] = {0};
   tfsAbsoluteName(pTsdb->pTfs, did, path, aname);
-  if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
+  if (tsdbOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
+    tsdbFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  if ((pEnv->pPool = openPool()) == NULL) {
     tsdbFreeSmaEnv(pEnv);
     return NULL;
   }
@@ -248,7 +389,8 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) {
     taosMemoryFreeClear(pSmaEnv->pStat);
     taosMemoryFreeClear(pSmaEnv->path);
     taosThreadRwlockDestroy(&(pSmaEnv->lock));
-    tsdbCloseBDBEnv(pSmaEnv->dbEnv);
+    tsdbCloseDBEnv(pSmaEnv->dbEnv);
+    closePool(pSmaEnv->pPool);
   }
 }
 
@@ -414,7 +556,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t
     }
 
     // cache smaMeta
-    STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid);
+    STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid, true);
     if (pSma == NULL) {
       terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
       taosHashCleanup(pItem->expiredWindows);
@@ -498,10 +640,6 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) {
     return TSDB_CODE_FAILED;
   }
 
-#ifndef TSDB_SMA_TEST
-  TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE];
-#endif
-
   // Firstly, assume that tSma can only be created on super table/normal table.
   // getActiveTimeWindow
 
@@ -563,6 +701,10 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) {
       TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
 
       tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey);
+
+      // TODO: release only when suid changes.
+      tdDestroyTSmaWrapper(pSW);
+      taosMemoryFreeClear(pSW);
     }
   }
 
@@ -676,10 +818,12 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
  * @param dataLen
  * @return int32_t
  */
-static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
+static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
+                                    TXN *txn) {
   SDBFile *pDBFile = &pSmaH->dFile;
+
   // TODO: insert sma data blocks into B+Tree(TDB)
-  if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
+  if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
     tsdbWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
              REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
     return TSDB_CODE_FAILED;
@@ -826,6 +970,30 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe
   return daysPerFile;
 }
 
+static int tsdbSmaBeginCommit(SSmaEnv *pEnv) {
+  TXN *pTxn = &pEnv->txn;
+  // start a new txn
+  tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  if (tdbBegin(pEnv->dbEnv, pTxn) != 0) {
+    tsdbWarn("tsdbSma tdb restart txn fail");
+    return -1;
+  }
+  return 0;
+}
+
+static int tsdbSmaEndCommit(SSmaEnv *pEnv) {
+  TXN *pTxn = &pEnv->txn;
+
+  // Commit current txn
+  if (tdbCommit(pEnv->dbEnv, pTxn) != 0) {
+    tsdbWarn("tsdbSma tdb commit fail");
+    return -1;
+  }
+  tdbTxnClose(pTxn);
+  clearPool(pEnv->pPool);
+  return 0;
+}
+
 /**
  * @brief Insert/Update Time-range-wise SMA data.
  *  - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g.
@@ -911,14 +1079,10 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
     int64_t      groupId = pDataBlock->info.groupId;
     for (int32_t j = 0; j < rows; ++j) {
       printf("|");
-      TSKEY skey = 1649295200000;  // TSKEY_INITIAL_VAL;  // the start key of TS window by interval
+      TSKEY skey = TSKEY_INITIAL_VAL;  //  the start key of TS window by interval
       void *pSmaKey = &smaKey;
       bool  isStartKey = false;
-      {
+
-        // just for debugging
-        isStartKey = true;
-        tsdbEncodeTSmaKey(groupId, skey, &pSmaKey);
-      }
       int32_t tlen = 0;     // reset the len
       pDataBuf = &dataBuf;  // reset the buf
       for (int32_t k = 0; k < colNum; ++k) {
@@ -929,7 +1093,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
             if (!isStartKey) {
               isStartKey = true;
               skey = *(TSKEY *)var;
-              printf("==> skey = %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
+              printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
               tsdbEncodeTSmaKey(groupId, skey, &pSmaKey);
             } else {
               printf(" %" PRIi64 " |", *(int64_t *)var);
@@ -1010,6 +1174,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
         // TODO: tsdbStartTSmaCommit();
         if (fid != tSmaH.dFile.fid) {
           if (tSmaH.dFile.fid != TSDB_IVLD_FID) {
+            tsdbSmaEndCommit(pEnv);
             tsdbCloseDBF(&tSmaH.dFile);
           }
           tsdbSetTSmaDataFile(&tSmaH, indexUid, fid);
@@ -1020,12 +1185,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
             tsdbUnRefSmaStat(pTsdb, pStat);
             return TSDB_CODE_FAILED;
           }
+          tsdbSmaBeginCommit(pEnv);
         }
 
-        if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen) != 0) {
+        if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
           tsdbWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
                    " since %s",
                    REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno));
+          tsdbSmaEndCommit(pEnv);
           tsdbDestroyTSmaWriteH(&tSmaH);
           tsdbUnRefSmaStat(pTsdb, pStat);
           return TSDB_CODE_FAILED;
@@ -1044,9 +1211,10 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
       printf("\n");
     }
   }
-
+  tsdbSmaEndCommit(pEnv);  // TODO: not commit for every insert
   tsdbDestroyTSmaWriteH(&tSmaH);
   tsdbUnRefSmaStat(pTsdb, pStat);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1370,8 +1538,8 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
   tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb),
             tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN);
 
-  void    *result = NULL;
+  void   *result = NULL;
-  uint32_t valueSize = 0;
+  int32_t valueSize = 0;
   if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) {
     tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s",
              REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno));
@@ -1422,7 +1590,7 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) {
     return -1;
   }
   tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName,
-         vCreateSmaReq.tSma.indexUid);
+            vCreateSmaReq.tSma.indexUid);
 
   // record current timezone of server side
   vCreateSmaReq.tSma.timezoneInt = tsTimezone;
@@ -1464,7 +1632,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
     return -1;
   }
 
-  tsdbTSmaSub(pTsdb, 1);  
+  tsdbTSmaSub(pTsdb, 1);
 
   // TODO: return directly or go on follow steps?
   return TSDB_CODE_SUCCESS;

source/dnode/vnode/src/tsdb/tsdbTDBImpl.c

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define ALLOW_FORBID_FUNC
+
+#include "vnodeInt.h"
+
+int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) {
+  int   ret = 0;
+
+  if (path == NULL) return -1;
+
+  ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param
+
+  if (ret != 0) {
+    tsdbError("Failed to create tsdb db env, ret = %d", ret);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); }
+
+static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
+  const SSmaKey *pKey1 = (const SSmaKey *)arg1;
+  const SSmaKey *pKey2 = (const SSmaKey *)arg2;
+
+  ASSERT(len1 == len2 && len1 == sizeof(SSmaKey));
+
+  if (pKey1->skey < pKey2->skey) {
+    return -1;
+  } else if (pKey1->skey > pKey2->skey) {
+    return 1;
+  }
+  if (pKey1->groupId < pKey2->groupId) {
+    return -1;
+  } else if (pKey1->groupId > pKey2->groupId) {
+    return 1;
+  }
+
+  return 0;
+}
+
+static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) {
+  int            ret;
+  FKeyComparator compFunc;
+
+  // Create a database
+  compFunc = tsdbSmaKeyCmpr;
+  ret = tdbDbOpen(pFName, TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, ppDB);
+
+  return 0;
+}
+
+static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); }
+
+int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) {
+  // TEnv is shared by a group of SDBFile
+  if (!pEnv || !pDBF) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
+
+  // Open DBF
+  if (tsdbOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
+    terrno = TSDB_CODE_TDB_INIT_FAILED;
+    tsdbCloseDBDb(pDBF->pDB);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t tsdbCloseDBF(SDBFile *pDBF) {
+  int32_t ret = 0;
+  if (pDBF->pDB) {
+    ret = tsdbCloseDBDb(pDBF->pDB);
+    pDBF->pDB = NULL;
+  }
+  taosMemoryFreeClear(pDBF->path);
+  return ret;
+}
+
+int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
+  int32_t ret;
+
+  ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
+  if (ret < 0) {
+    tsdbError("Failed to create insert sma data into db, ret = %d", ret);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) {
+  void *result;
+  void *pVal;
+  int   ret;
+
+  ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
+
+  if (ret < 0) {
+    tsdbError("Failed to get sma data from db, ret = %d", ret);
+    return NULL;
+  }
+
+  ASSERT(*valLen >= 0);
+
+  result = taosMemoryMalloc(*valLen);
+
+  if (result == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  // TODO: lock?
+  // TODO: Would the key/value be destoryed during return the data?
+  // TODO: How about the key is updated while value length is changed? The original value buffer would be freed
+  // automatically?
+  memcpy(result, pVal, *valLen);
+
+  return result;
+}

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -19,7 +19,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
   // TODO
 
   // blockDebugShowData(data);
-  // tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data);
+  tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data);
 }
 
 void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
@@ -232,9 +232,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       tdDestroyTSma(&vCreateSmaReq.tSma);
       // TODO: return directly or go on follow steps?
 #endif
-    //   if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
+      if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
-    //     // TODO
+        // TODO
-    //   }
+      }
     // } break;
     // case TDMT_VND_CANCEL_SMA: {  // timeRangeSMA
     // } break;

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -210,7 +210,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
 
   // get value by indexName
   STSma *qSmaCfg = NULL;
-  qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1);
+  qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1, true);
   assert(qSmaCfg != NULL);
   printf("name1 = %s\n", qSmaCfg->indexName);
   printf("timezone1 = %" PRIi8 "\n", qSmaCfg->timezoneInt);
@@ -221,7 +221,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   tdDestroyTSma(qSmaCfg);
   taosMemoryFreeClear(qSmaCfg);
 
-  qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2);
+  qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2, true);
   assert(qSmaCfg != NULL);
   printf("name2 = %s\n", qSmaCfg->indexName);
   printf("timezone2 = %" PRIi8 "\n", qSmaCfg->timezoneInt);
@@ -233,11 +233,12 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   taosMemoryFreeClear(qSmaCfg);
 
   // get index name by table uid
+#if 0
   SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid);
   assert(pSmaCur != NULL);
   uint32_t indexCnt = 0;
   while (1) {
-    const char *indexName = metaSmaCursorNext(pSmaCur);
+    const char *indexName = (const char *)metaSmaCursorNext(pSmaCur);
     if (indexName == NULL) {
       break;
     }
@@ -245,8 +246,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
     ++indexCnt;
   }
   EXPECT_EQ(indexCnt, nCntTSma);
-  metaCloseSmaCurosr(pSmaCur);
+  metaCloseSmaCursor(pSmaCur);
-
+#endif
   // get wrapper by table uid
   STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid);
   assert(pSW != NULL);

source/libs/executor/src/dataDispatcher.c

@@ -92,7 +92,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput,
     return false;
   }
 
-  pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockDataGetSerialMetaSize(pInput->pData) + blockDataGetSize(pInput->pData);
+  pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pInput->pData);
 
   pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
   if (pBuf->pData == NULL) {

source/libs/function/inc/builtins.h

@@ -22,7 +22,7 @@ extern "C" {
 
 #include "functionMgt.h"
 
-#define FUNCTION_NAME_MAX_LENGTH 16
+#define FUNCTION_NAME_MAX_LENGTH 32
 
 #define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n)    (1 << n)
 

source/libs/function/src/builtins.c

@@ -393,6 +393,46 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .sprocessFunc = castFunction,
     .finalizeFunc = NULL
   },
+  {
+    .name = "to_iso8601",
+    .type = FUNCTION_TYPE_TO_ISO8601,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .checkFunc    = checkAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = toISO8601Function,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "to_unixtimestamp",
+    .type = FUNCTION_TYPE_TO_UNIXTIMESTAMP,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .checkFunc    = checkAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = toUnixtimestampFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "timetruncate",
+    .type = FUNCTION_TYPE_TIMETRUNCATE,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .checkFunc    = checkAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = timeTruncateFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "timediff",
+    .type = FUNCTION_TYPE_TIMEDIFF,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .checkFunc    = checkAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = timeDiffFunction,
+    .finalizeFunc = NULL
+  },
   {
     .name = "_rowts",
     .type = FUNCTION_TYPE_ROWTS,
@@ -609,6 +649,22 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) {
       pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType};
       break;
     }
+    case FUNCTION_TYPE_TO_ISO8601: {
+      pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY};
+      break;
+    }
+    case FUNCTION_TYPE_TO_UNIXTIMESTAMP: {
+      pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
+      break;
+    }
+    case FUNCTION_TYPE_TIMETRUNCATE: {
+      pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
+      break;
+    }
+    case FUNCTION_TYPE_TIMEDIFF: {
+      pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
+      break;
+    }
 
     case FUNCTION_TYPE_TBNAME: {
       // todo

source/libs/nodes/src/nodesCloneFuncs.c

@@ -95,7 +95,6 @@ static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) {
 static void exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
   dataTypeCopy(&pSrc->resType, &pDst->resType);
   COPY_CHAR_ARRAY_FIELD(aliasName);
-  // CLONE_NODE_LIST_FIELD(pAssociationList);
 }
 
 static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
@@ -222,15 +221,19 @@ static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) {
 }
 
 static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
+  COPY_ALL_SCALAR_FIELDS;
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   CLONE_NODE_LIST_FIELD(pScanCols);
   CLONE_OBJECT_FIELD(pMeta, tableMetaClone);
   CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
-  COPY_SCALAR_FIELD(scanType);
+  CLONE_NODE_LIST_FIELD(pDynamicScanFuncs);
-  COPY_SCALAR_FIELD(scanFlag);
+  return (SNode*)pDst;
-  COPY_SCALAR_FIELD(scanRange);
+}
-  COPY_SCALAR_FIELD(tableName);
+
-  COPY_SCALAR_FIELD(showRewrite);
+static SNode* logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
+  COPY_ALL_SCALAR_FIELDS;
+  COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
+  CLONE_NODE_FIELD(pOnConditions);
   return (SNode*)pDst;
 }
 
@@ -263,15 +266,8 @@ static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNo
 static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pDst) {
   COPY_ALL_SCALAR_FIELDS;
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
-  // COPY_SCALAR_FIELD(winType);
   CLONE_NODE_LIST_FIELD(pFuncs);
-  // COPY_SCALAR_FIELD(interval);
-  // COPY_SCALAR_FIELD(offset);
-  // COPY_SCALAR_FIELD(sliding);
-  // COPY_SCALAR_FIELD(intervalUnit);
-  // COPY_SCALAR_FIELD(slidingUnit);
   CLONE_NODE_FIELD(pFill);
-  // COPY_SCALAR_FIELD(sessionGap);
   CLONE_NODE_FIELD(pTspk);
   return (SNode*)pDst;
 }
@@ -360,6 +356,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
       return downstreamSourceCopy((const SDownstreamSourceNode*)pNode, (SDownstreamSourceNode*)pDst);
     case QUERY_NODE_LOGIC_PLAN_SCAN:
       return logicScanCopy((const SScanLogicNode*)pNode, (SScanLogicNode*)pDst);
+    case QUERY_NODE_LOGIC_PLAN_JOIN:
+      return logicJoinCopy((const SJoinLogicNode*)pNode, (SJoinLogicNode*)pDst);
     case QUERY_NODE_LOGIC_PLAN_AGG:
       return logicAggCopy((const SAggLogicNode*)pNode, (SAggLogicNode*)pDst);
     case QUERY_NODE_LOGIC_PLAN_PROJECT:

source/libs/nodes/src/nodesUtilFuncs.c

@@ -158,6 +158,16 @@ SNodeptr nodesMakeNode(ENodeType type) {
     case QUERY_NODE_SHOW_FUNCTIONS_STMT:
     case QUERY_NODE_SHOW_INDEXES_STMT:
     case QUERY_NODE_SHOW_STREAMS_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
+    case QUERY_NODE_SHOW_LICENCE_STMT:
+    case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
+    case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
+    case QUERY_NODE_SHOW_CREATE_STABLE_STMT: 
+    case QUERY_NODE_SHOW_QUERIES_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
+    case QUERY_NODE_SHOW_TOPICS_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
     case QUERY_NODE_SHOW_BNODES_STMT:
     case QUERY_NODE_SHOW_SNODES_STMT:
       return makeNode(type, sizeof(SShowStmt));

source/libs/parser/inc/sql.y

@@ -79,13 +79,13 @@ alter_account_option ::= USERS literal.
 alter_account_option ::= CONNS literal.                                           { }
 alter_account_option ::= STATE literal.                                           { }
 
-/************************************************ create/alter/drop/show user *****************************************/
+/************************************************ create/alter/drop user **********************************************/
 cmd ::= CREATE USER user_name(A) PASS NK_STRING(B).                               { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); }
 cmd ::= ALTER USER user_name(A) PASS NK_STRING(B).                                { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); }
 cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B).                           { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); }
 cmd ::= DROP USER user_name(A).                                                   { pCxt->pRootNode = createDropUserStmt(pCxt, &A); }
 
-/************************************************ create/drop/alter/show dnode ****************************************/
+/************************************************ create/drop/alter dnode *********************************************/
 cmd ::= CREATE DNODE dnode_endpoint(A).                                           { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
 cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B).                       { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); }
 cmd ::= DROP DNODE NK_INTEGER(A).                                                 { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); }
@@ -124,7 +124,7 @@ cmd ::= DROP SNODE ON DNODE NK_INTEGER(A).
 cmd ::= CREATE MNODE ON DNODE NK_INTEGER(A).                                      { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &A); }
 cmd ::= DROP MNODE ON DNODE NK_INTEGER(A).                                        { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &A); }
 
-/************************************************ create/drop/show/use database ***************************************/
+/************************************************ create/drop/use database ********************************************/
 cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C).               { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, A, &B, C); }
 cmd ::= DROP DATABASE exists_opt(A) db_name(B).                                   { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); }
 cmd ::= USE db_name(A).                                                           { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); }
@@ -332,6 +332,7 @@ cmd ::= SHOW ACCOUNTS.
 cmd ::= SHOW APPS.                                                                { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); }
 cmd ::= SHOW CONNECTIONS.                                                         { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT, NULL, NULL); }
 cmd ::= SHOW LICENCE.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); }
+cmd ::= SHOW GRANTS.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); }
 cmd ::= SHOW CREATE DATABASE db_name(A).                                          { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
 cmd ::= SHOW CREATE TABLE full_table_name(A).                                     { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
 cmd ::= SHOW CREATE STABLE full_table_name(A).                                    { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, A); }

source/libs/parser/src/parCalcConst.c

@@ -143,7 +143,7 @@ static int32_t rewriteConditionForFromTable(SCalcConstContext* pCxt, SNode* pTab
     if (TSDB_CODE_SUCCESS == pCxt->code) {
       pCxt->code = rewriteConditionForFromTable(pCxt, pJoin->pRight);
     }
-    if (TSDB_CODE_SUCCESS == pCxt->code) {
+    if (TSDB_CODE_SUCCESS == pCxt->code && NULL != pJoin->pOnCond) {
       pCxt->code = rewriteCondition(pCxt, &pJoin->pOnCond);
     }
   }

source/libs/parser/src/parTokenizer.c

@@ -80,6 +80,7 @@ static SKeyword keywordTable[] = {
     {"FSYNC",         TK_FSYNC},
     {"FUNCTION",      TK_FUNCTION},
     {"FUNCTIONS",     TK_FUNCTIONS},
+    {"GRANTS",        TK_GRANTS},
     {"GROUP",         TK_GROUP},
     {"HAVING",        TK_HAVING},
     {"IF",            TK_IF},
@@ -131,10 +132,10 @@ static SKeyword keywordTable[] = {
     {"PRECISION",     TK_PRECISION},
     {"PRIVILEGE",     TK_PRIVILEGE},
     {"PREV",          TK_PREV},
-    {"_QENDTS",        TK_QENDTS},
+    {"_QENDTS",       TK_QENDTS},
     {"QNODE",         TK_QNODE},
     {"QNODES",        TK_QNODES},
-    {"_QSTARTTS",      TK_QSTARTTS},
+    {"_QSTARTTS",     TK_QSTARTTS},
     {"QTIME",         TK_QTIME},
     {"QUERIES",       TK_QUERIES},
     {"QUERY",         TK_QUERY},
@@ -144,7 +145,7 @@ static SKeyword keywordTable[] = {
     {"RESET",         TK_RESET},
     {"RETENTIONS",    TK_RETENTIONS},
     {"ROLLUP",        TK_ROLLUP},
-    {"_ROWTS",         TK_ROWTS},
+    {"_ROWTS",        TK_ROWTS},
     {"SCORES",        TK_SCORES},
     {"SELECT",        TK_SELECT},
     {"SESSION",       TK_SESSION},
@@ -163,7 +164,7 @@ static SKeyword keywordTable[] = {
     {"STATE",         TK_STATE},
     {"STATE_WINDOW",  TK_STATE_WINDOW},
     {"STORAGE",       TK_STORAGE},
-    {"STREAM",       TK_STREAM},
+    {"STREAM",        TK_STREAM},
     {"STREAMS",       TK_STREAMS},
     {"STREAM_MODE",   TK_STREAM_MODE},
     {"SYNCDB",        TK_SYNCDB},
@@ -192,8 +193,8 @@ static SKeyword keywordTable[] = {
     {"VGROUPS",       TK_VGROUPS},
     {"VNODES",        TK_VNODES},
     {"WAL",           TK_WAL},
-    {"_WDURATION",     TK_WDURATION},
+    {"_WDURATION",    TK_WDURATION},
-    {"_WENDTS",        TK_WENDTS},
+    {"_WENDTS",       TK_WENDTS},
     {"WHERE",         TK_WHERE},
     {"_WSTARTTS",     TK_WSTARTTS},
     // {"ID",           TK_ID},
@@ -221,7 +222,6 @@ static SKeyword keywordTable[] = {
     // {"UMINUS",       TK_UMINUS},
     // {"UPLUS",        TK_UPLUS},
     // {"BITNOT",       TK_BITNOT},
-    // {"GRANTS",       TK_GRANTS},
     // {"DOT",          TK_DOT},
     // {"CTIME",        TK_CTIME},
     // {"LP",           TK_LP},

source/libs/parser/src/parTranslater.c

@@ -762,7 +762,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) {
   size_t  nums = taosArrayGetSize(pTables);
   for (size_t i = 0; i < nums; ++i) {
     STableNode* pTable = taosArrayGetP(pTables, i);
-    int32_t     code = createColumnNodeByTable(pCxt, pTable, *pCols);
+    int32_t code = createColumnNodeByTable(pCxt, pTable, *pCols);
     if (TSDB_CODE_SUCCESS != code) {
       return code;
     }
@@ -829,11 +829,39 @@ static int32_t createFirstLastAllCols(STranslateContext* pCxt, SFunctionNode* pS
   return TSDB_CODE_SUCCESS;
 }
 
+static bool isTableStar(SNode* pNode) {
+  return (QUERY_NODE_COLUMN == nodeType(pNode)) && (0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
+}
+
+static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SNodeList** pOutput) {
+  *pOutput = nodesMakeList();
+  if (NULL == *pOutput) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY);
+  }
+  bool foundTable = false;
+  SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
+  size_t  nums = taosArrayGetSize(pTables);
+  for (size_t i = 0; i < nums; ++i) {
+    STableNode* pTable = taosArrayGetP(pTables, i);
+    if (0 == strcmp(pTable->tableAlias, pCol->tableAlias)) {
+      int32_t code = createColumnNodeByTable(pCxt, pTable, *pOutput);
+      if (TSDB_CODE_SUCCESS != code) {
+        return code;
+      }
+      foundTable = true;
+      break;
+    }
+  }
+  if (!foundTable) {
+    return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pCol->tableAlias);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (NULL == pSelect->pProjectionList) {  // select * ...
     return createAllColumns(pCxt, &pSelect->pProjectionList);
   } else {
-    // todo : t.*
     SNode* pNode = NULL;
     WHERE_EACH(pNode, pSelect->pProjectionList) {
       if (isFirstLastStar(pNode)) {
@@ -844,6 +872,14 @@ static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
         INSERT_LIST(pSelect->pProjectionList, pFuncs);
         ERASE_NODE(pSelect->pProjectionList);
         continue;
+      } else if (isTableStar(pNode)) {
+        SNodeList* pCols = NULL;
+        if (TSDB_CODE_SUCCESS != createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols)) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+        INSERT_LIST(pSelect->pProjectionList, pCols);
+        ERASE_NODE(pSelect->pProjectionList);
+        continue;
       }
       WHERE_NEXT;
     }

source/libs/planner/src/planOptimizer.c

@@ -147,8 +147,80 @@ static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
   return code;
 }
 
+static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
+  // todo
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdPartitionCondition(SJoinLogicNode* pJoin, SNodeList** pMultiTableCond, SNodeList** pSingleTableCond) {
+  // todo
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdPushJoinCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNodeList* pMultiTableCond) {
+  // todo
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdPushJoinCondToChildren(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNodeList* pSingleTableCond) {
+  // todo
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
+  if (NULL != pJoin->node.pConditions) {
+    SNodeList* pMultiTableCond = NULL;
+    SNodeList* pSingleTableCond = NULL;
+    int32_t code = cpdPartitionCondition(pJoin, &pMultiTableCond, &pSingleTableCond);
+    if (TSDB_CODE_SUCCESS == code && NULL != pMultiTableCond) {
+      code = cpdPushJoinCondToOnCond(pCxt, pJoin, pMultiTableCond);
+    }
+    if (TSDB_CODE_SUCCESS == code && NULL != pSingleTableCond) {
+      code = cpdPushJoinCondToChildren(pCxt, pJoin, pSingleTableCond);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdPushAggCondition(SOptimizeContext* pCxt, SAggLogicNode* pAgg) {
+  // todo
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  switch (nodeType(pLogicNode)) {
+    case QUERY_NODE_LOGIC_PLAN_SCAN:
+      code = cpdOptimizeScanCondition(pCxt, (SScanLogicNode*)pLogicNode);
+      break;
+    case QUERY_NODE_LOGIC_PLAN_JOIN:
+      code = cpdPushJoinCondition(pCxt, (SJoinLogicNode*)pLogicNode);
+      break;
+    case QUERY_NODE_LOGIC_PLAN_AGG:
+      code = cpdPushAggCondition(pCxt, (SAggLogicNode*)pLogicNode);
+      break;
+    default:
+      break;
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    SNode* pChild = NULL;
+    FOREACH(pChild, pLogicNode->pChildren) {
+      code = cpdPushCondition(pCxt, (SLogicNode*)pChild);
+      if (TSDB_CODE_SUCCESS != code) {
+        break;
+      }
+    }
+  }
+  return code;
+}
+
+static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
+  return cpdPushCondition(pCxt, pLogicNode);
+}
+
 static const SOptimizeRule optimizeRuleSet[] = {
-  { .pName = "OptimizeScanData", .optimizeFunc = osdOptimize }
+  { .pName = "OptimizeScanData", .optimizeFunc = osdOptimize },
+  { .pName = "ConditionPushDown", .optimizeFunc = cpdOptimize }
 };
 
 static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule));

source/libs/planner/src/planPhysiCreater.c

@@ -492,58 +492,25 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan,
   return TSDB_CODE_FAILED;
 }
 
-static int32_t createColFromDataBlockDesc(SDataBlockDescNode* pDesc, SNodeList* pCols) {
-  SNode* pNode;
-  FOREACH(pNode, pDesc->pSlots) {
-    SSlotDescNode* pSlot = (SSlotDescNode*)pNode;
-    SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
-    if (NULL == pCol) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    pCol->node.resType = pSlot->dataType;
-    pCol->dataBlockId = pDesc->dataBlockId;
-    pCol->slotId = pSlot->slotId;
-    pCol->colId = -1;
-    int32_t code = nodesListStrictAppend(pCols, pCol);
-    if (TSDB_CODE_SUCCESS != code) {
-      return code;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* pLeftDesc, SDataBlockDescNode* pRightDesc, SNodeList** pList) {
-  SNodeList* pCols = nodesMakeList();
-  if (NULL == pCols) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  int32_t code = createColFromDataBlockDesc(pLeftDesc, pCols);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = createColFromDataBlockDesc(pRightDesc, pCols);
-  }
-
-  if (TSDB_CODE_SUCCESS == code) {
-    *pList = pCols;
-  } else {
-    nodesDestroyList(pCols);
-  }
-
-  return code;
-}
-
 static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) {
   SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_JOIN);
   if (NULL == pJoin) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  SDataBlockDescNode* pLeftDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
+  int32_t code = TSDB_CODE_SUCCESS;
-  SDataBlockDescNode* pRightDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc;
 
-  int32_t code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions, &pJoin->pOnConditions);
+  pJoin->joinType = pJoinLogicNode->joinType;
+  if (NULL != pJoinLogicNode->pOnConditions) {
+    SDataBlockDescNode* pLeftDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
+    SDataBlockDescNode* pRightDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc;
+    code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions, &pJoin->pOnConditions);
+  }
   if (TSDB_CODE_SUCCESS == code) {
-    code = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc, &pJoin->pTargets);
+    pJoin->pTargets = nodesCloneList(pJoinLogicNode->node.pTargets);
+    if (NULL == pJoin->pTargets) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);

source/libs/planner/src/planSpliter.c

@@ -190,6 +190,7 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan
     pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
   }
   pSubplan->id.queryId = pCxt->queryId;
+  pSubplan->id.groupId = 1;
   setLogicNodeParent(pSubplan->pNode);
 
   int32_t code = applySplitRule(pSubplan);

source/libs/planner/test/plannerTest.cpp

@@ -150,7 +150,7 @@ private:
   SQuery* query_;
 };
 
-TEST_F(PlannerTest, simple) {
+TEST_F(PlannerTest, selectBasic) {
   setDatabase("root", "test");
 
   bind("SELECT * FROM t1");
@@ -164,14 +164,27 @@ TEST_F(PlannerTest, selectConstant) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, stSimple) {
+TEST_F(PlannerTest, selectStableBasic) {
   setDatabase("root", "test");
 
   bind("SELECT * FROM st1");
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, groupBy) {
+TEST_F(PlannerTest, selectJoin) {
+  setDatabase("root", "test");
+
+  bind("SELECT * FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1");
+  ASSERT_TRUE(run());
+
+  bind("SELECT t1.* FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1");
+  ASSERT_TRUE(run());
+}
+
+TEST_F(PlannerTest, selectGroupBy) {
   setDatabase("root", "test");
 
   bind("SELECT count(*) FROM t1");
@@ -187,14 +200,14 @@ TEST_F(PlannerTest, groupBy) {
   // ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, subquery) {
+TEST_F(PlannerTest, selectSubquery) {
   setDatabase("root", "test");
 
   bind("SELECT count(*) FROM (SELECT c1 + c3 a, c1 + count(*) b FROM t1 where c2 = 'abc' GROUP BY c1, c3) where a > 100 group by b");
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, interval) {
+TEST_F(PlannerTest, selectInterval) {
   setDatabase("root", "test");
 
   bind("SELECT count(*) FROM t1 interval(10s)");
@@ -210,14 +223,14 @@ TEST_F(PlannerTest, interval) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, sessionWindow) {
+TEST_F(PlannerTest, selectSessionWindow) {
   setDatabase("root", "test");
 
   bind("SELECT count(*) FROM t1 session(ts, 10s)");
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, stateWindow) {
+TEST_F(PlannerTest, selectStateWindow) {
   setDatabase("root", "test");
 
   bind("SELECT count(*) FROM t1 state_window(c1)");
@@ -227,7 +240,7 @@ TEST_F(PlannerTest, stateWindow) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, partitionBy) {
+TEST_F(PlannerTest, selectPartitionBy) {
   setDatabase("root", "test");
 
   bind("SELECT * FROM t1 partition by c1");
@@ -243,7 +256,7 @@ TEST_F(PlannerTest, partitionBy) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, orderBy) {
+TEST_F(PlannerTest, selectOrderBy) {
   setDatabase("root", "test");
 
   bind("SELECT c1 FROM t1 order by c1");
@@ -259,7 +272,7 @@ TEST_F(PlannerTest, orderBy) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, groupByOrderBy) {
+TEST_F(PlannerTest, selectGroupByOrderBy) {
   setDatabase("root", "test");
 
   bind("select count(*), sum(c1) from t1 order by sum(c1)");
@@ -269,7 +282,7 @@ TEST_F(PlannerTest, groupByOrderBy) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, distinct) {
+TEST_F(PlannerTest, selectDistinct) {
   setDatabase("root", "test");
 
   bind("SELECT distinct c1 FROM t1");
@@ -282,7 +295,7 @@ TEST_F(PlannerTest, distinct) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, limit) {
+TEST_F(PlannerTest, selectLimit) {
   setDatabase("root", "test");
 
   bind("SELECT * FROM t1 limit 2");
@@ -295,7 +308,7 @@ TEST_F(PlannerTest, limit) {
   ASSERT_TRUE(run());
 }
 
-TEST_F(PlannerTest, slimit) {
+TEST_F(PlannerTest, selectSlimit) {
   setDatabase("root", "test");
 
   bind("SELECT * FROM t1 partition by c1 slimit 2");

source/libs/scalar/inc/sclInt.h

@@ -46,8 +46,9 @@ typedef struct SScalarCtx {
 int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
 SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
 
-#define GET_PARAM_TYPE(_c)   ((_c)->columnData->info.type)
+#define GET_PARAM_TYPE(_c)      ((_c)->columnData->info.type)
-#define GET_PARAM_BYTES(_c)  ((_c)->columnData->info.bytes)
+#define GET_PARAM_BYTES(_c)     ((_c)->columnData->info.bytes)
+#define GET_PARAM_PRECISON(_c)  ((_c)->columnData->info.precision)
 
 void sclFreeParam(SScalarParam *param);
 

source/libs/scalar/src/sclfunc.c

@@ -1,6 +1,7 @@
 #include "function.h"
 #include "scalar.h"
 #include "tdatablock.h"
+#include "ttime.h"
 #include "sclInt.h"
 #include "sclvector.h"
 
@@ -801,6 +802,446 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  int32_t type = GET_PARAM_TYPE(pInput);
+  if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (inputNum != 1) {
+    return TSDB_CODE_FAILED;
+  }
+
+  char *input  = pInput[0].columnData->pData;
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    if (colDataIsNull_s(pInput[0].columnData, i)) {
+      colDataAppendNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    char fraction[20] = {0};
+    bool hasFraction = false;
+    NUM_TO_STRING(type, input, sizeof(fraction), fraction);
+    int32_t tsDigits = (int32_t)strlen(fraction);
+
+    char buf[64] = {0};
+    int64_t timeVal;
+    GET_TYPED_DATA(timeVal, int64_t, type, input);
+    if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
+      if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+        timeVal = timeVal / 1000;
+      } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+        timeVal = timeVal / (1000 * 1000);
+      } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+        timeVal = timeVal / (1000 * 1000 * 1000);
+      } else {
+        assert(0);
+      }
+      hasFraction = true;
+      memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS);
+    }
+
+    struct tm *tmInfo = localtime((const time_t *)&timeVal);
+    strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S%z", tmInfo);
+    int32_t len = (int32_t)strlen(buf);
+
+    if (hasFraction) {
+      int32_t fracLen = (int32_t)strlen(fraction) + 1;
+      char *tzInfo = strchr(buf, '+');
+      if (tzInfo) {
+        memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
+      } else {
+        tzInfo = strchr(buf, '-');
+        memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
+      }
+
+      char tmp[32];
+      sprintf(tmp, ".%s", fraction);
+      memcpy(tzInfo, tmp, fracLen);
+      len += fracLen;
+    }
+
+    memmove(buf + VARSTR_HEADER_SIZE, buf, len);
+    varDataSetLen(buf, len);
+
+    colDataAppend(pOutput->columnData, i, buf, false);
+    input   += tDataTypes[type].bytes;
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  int32_t type = GET_PARAM_TYPE(pInput);
+  int32_t timePrec = GET_PARAM_PRECISON(pInput);
+  if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (inputNum != 1) {
+    return TSDB_CODE_FAILED;
+  }
+
+  char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0];
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    if (colDataIsNull_s(pInput[0].columnData, i)) {
+      colDataAppendNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    int64_t timeVal = 0;
+    convertStringToTimestamp(type, input, timePrec, &timeVal);
+
+    colDataAppend(pOutput->columnData, i, (char *)&timeVal, false);
+    input += varDataTLen(input);
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  int32_t type = GET_PARAM_TYPE(&pInput[0]);
+  int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);
+  if (inputNum != 2) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP &&
+      type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (GET_PARAM_TYPE(&pInput[1]) != TSDB_DATA_TYPE_BIGINT) { //time_unit
+    return TSDB_CODE_FAILED;
+  }
+
+  int64_t timeUnit, timeVal = 0;
+  GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
+
+  int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
+                   (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
+
+  char *input = NULL;
+  if (IS_VAR_DATA_TYPE(type)) {
+    input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0];
+  } else {
+    input = pInput[0].columnData->pData;
+  }
+
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    if (colDataIsNull_s(pInput[0].columnData, i)) {
+      colDataAppendNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
+      convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal);
+      //If converted value is less than 10digits in second, use value in second instead
+      int64_t timeValSec = timeVal / 1000000000;
+      if (timeValSec < 1000000000) {
+        timeVal = timeValSec;
+      }
+    } else if (type == TSDB_DATA_TYPE_BIGINT) { /* unix timestamp */
+      GET_TYPED_DATA(timeVal, int64_t, type, input);
+    } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { /* timestamp column*/
+      GET_TYPED_DATA(timeVal, int64_t, type, input);
+      int64_t timeValSec = timeVal / factor;
+      if (timeValSec < 1000000000) {
+        timeVal = timeValSec;
+      }
+    }
+
+    char buf[20] = {0};
+    NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal, sizeof(buf), buf);
+    int32_t tsDigits = (int32_t)strlen(buf);
+    timeUnit = timeUnit * 1000 / factor;
+
+    switch (timeUnit) {
+      case 0: { /* 1u */
+        if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000 * 1000;
+        //} else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+        //  //timeVal = timeVal / 1000;
+        } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
+          timeVal = timeVal * factor;
+        } else {
+          timeVal = timeVal * 1;
+        }
+        break;
+      }
+      case 1: { /* 1a */
+        if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal * 1;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal / 1000 * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000000 * 1000000;
+        } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS){
+          timeVal = timeVal * factor;
+        } else {
+          assert(0);
+        }
+        break;
+      }
+      case 1000: { /* 1s */
+        if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal / 1000 * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal / 1000000 * 1000000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000000000 * 1000000000;
+        } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
+          timeVal = timeVal * factor;
+        } else {
+          assert(0);
+        }
+        break;
+      }
+      case 60000: { /* 1m */
+        if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal / 1000 / 60 * 60 * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal / 1000000 / 60 * 60 * 1000000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000000000 / 60 * 60 * 1000000000;
+        } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
+          timeVal = timeVal * factor / factor / 60 * 60 * factor;
+        } else {
+          assert(0);
+        }
+        break;
+      }
+      case 3600000: { /* 1h */
+        if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal / 1000 / 3600 * 3600 * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal / 1000000 / 3600 * 3600 * 1000000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000000000 / 3600 * 3600 * 1000000000;
+        } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
+          timeVal = timeVal * factor / factor / 3600 * 3600 * factor;
+        } else {
+          assert(0);
+        }
+        break;
+      }
+      case 86400000: { /* 1d */
+        if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal / 1000 / 86400 * 86400 * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000;
+        } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
+          timeVal = timeVal * factor / factor / 86400* 86400 * factor;
+        } else {
+          assert(0);
+        }
+        break;
+      }
+      case 604800000: { /* 1w */
+        if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal / 1000 / 604800 * 604800 * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal / 1000000 / 604800 * 604800 * 1000000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000000000 / 604800 * 604800 * 1000000000;
+        } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
+          timeVal = timeVal * factor / factor / 604800 * 604800* factor;
+        } else {
+          assert(0);
+        }
+        break;
+      }
+      default: {
+        timeVal = timeVal * 1;
+        break;
+      }
+    }
+
+    //truncate the timestamp to db precision
+    switch (timePrec) {
+      case TSDB_TIME_PRECISION_MILLI: {
+        if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal / 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000000;
+        }
+        break;
+      }
+      case TSDB_TIME_PRECISION_MICRO: {
+        if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal = timeVal / 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal * 1000;
+        }
+        break;
+      }
+      case TSDB_TIME_PRECISION_NANO: {
+        if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal = timeVal * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal = timeVal * 1000000;
+        }
+        break;
+      }
+    }
+
+    colDataAppend(pOutput->columnData, i, (char *)&timeVal, false);
+    if (IS_VAR_DATA_TYPE(type)) {
+      input += varDataTLen(input);
+    } else {
+      input += tDataTypes[type].bytes;
+    }
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  if (inputNum != 2 && inputNum != 3) {
+    return TSDB_CODE_FAILED;
+  }
+
+  int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]);
+  int64_t timeUnit = -1, timeVal[2] = {0};
+  if (inputNum == 3) {
+    if (GET_PARAM_TYPE(&pInput[2]) != TSDB_DATA_TYPE_BIGINT) {
+      return TSDB_CODE_FAILED;
+    }
+    GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
+  }
+
+  char *input[2];
+  for (int32_t k = 0; k < 2; ++k) {
+    int32_t type = GET_PARAM_TYPE(&pInput[k]);
+    if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP &&
+        type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
+      return TSDB_CODE_FAILED;
+    }
+
+    if (IS_VAR_DATA_TYPE(type)) {
+      input[k] = pInput[k].columnData->pData + pInput[k].columnData->varmeta.offset[0];
+    } else {
+      input[k] = pInput[k].columnData->pData;
+    }
+  }
+
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    for (int32_t k = 0; k < 2; ++k) {
+      if (colDataIsNull_s(pInput[0].columnData, i)) {
+        colDataAppendNULL(pOutput->columnData, i);
+        continue;
+      }
+
+      int32_t type = GET_PARAM_TYPE(&pInput[k]);
+      if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
+        convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]);
+      } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/
+        GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]);
+        if (type == TSDB_DATA_TYPE_TIMESTAMP) {
+          int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
+                           (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
+          int64_t timeValSec = timeVal[k] / factor;
+          if (timeValSec < 1000000000) {
+            timeVal[k] = timeValSec;
+          }
+        }
+
+        char buf[20] = {0};
+        NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal[k], sizeof(buf), buf);
+        int32_t tsDigits = (int32_t)strlen(buf);
+        if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
+          timeVal[k] = timeVal[k] * 1000000000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+          timeVal[k] = timeVal[k] * 1000000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
+          timeVal[k] = timeVal[k] * 1000;
+        } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
+          timeVal[k] = timeVal[k];
+        }
+      }
+
+      if (pInput[k].numOfRows != 1) {
+        if (IS_VAR_DATA_TYPE(type)) {
+          input[k] += varDataTLen(input[k]);
+        } else {
+          input[k] += tDataTypes[type].bytes;
+        }
+      }
+    }
+
+    int64_t result = (timeVal[0] >= timeVal[1]) ? (timeVal[0] - timeVal[1]) :
+                                                  (timeVal[1] - timeVal[0]);
+
+    if (timeUnit < 0) { // if no time unit given use db precision
+      switch(timePrec) {
+        case TSDB_TIME_PRECISION_MILLI: {
+          result = result / 1000000;
+          break;
+        }
+        case TSDB_TIME_PRECISION_MICRO: {
+          result = result / 1000;
+          break;
+        }
+        case TSDB_TIME_PRECISION_NANO: {
+          result = result / 1;
+          break;
+        }
+      }
+    } else {
+      int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
+                       (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
+      timeUnit = timeUnit * 1000 / factor;
+      switch(timeUnit) {
+        case 0: { /* 1u */
+          result = result / 1000;
+          break;
+        }
+        case 1: { /* 1a */
+          result = result / 1000000;
+          break;
+        }
+        case 1000: { /* 1s */
+          result = result / 1000000000;
+          break;
+        }
+        case 60000: { /* 1m */
+          result = result / 1000000000 / 60;
+          break;
+        }
+        case 3600000: { /* 1h */
+          result = result / 1000000000 / 3600;
+          break;
+        }
+        case 86400000: { /* 1d */
+          result = result / 1000000000 / 86400;
+          break;
+        }
+        case 604800000: { /* 1w */
+          result = result / 1000000000 / 604800;
+          break;
+        }
+        default: {
+          break;
+        }
+      }
+    }
+
+    colDataAppend(pOutput->columnData, i, (char *)&result, false);
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   return doScalarFunctionUnique(pInput, inputNum, pOutput, atan);
 }

source/libs/tdb/src/db/tdbBtree.c

@@ -1196,21 +1196,22 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
     return -1;
   }
 
-  // TODO: vLen may be zero
-  pVal = TDB_REALLOC(*ppVal, cd.vLen);
-  if (pVal == NULL) {
-    TDB_FREE(pKey);
-    return -1;
-  }
-
   *ppKey = pKey;
-  *ppVal = pVal;
-
   *kLen = cd.kLen;
-  *vLen = cd.vLen;
-
   memcpy(pKey, cd.pKey, cd.kLen);
-  memcpy(pVal, cd.pVal, cd.vLen);
+
+  if (ppVal) {
+    // TODO: vLen may be zero
+    pVal = TDB_REALLOC(*ppVal, cd.vLen);
+    if (pVal == NULL) {
+      TDB_FREE(pKey);
+      return -1;
+    }
+
+    *ppVal = pVal;
+    *vLen = cd.vLen;
+    memcpy(pVal, cd.pVal, cd.vLen);
+  }
 
   ret = tdbBtcMoveToNext(pBtc);
   if (ret < 0) {

source/os/src/osDir.c

@@ -76,6 +76,47 @@ int32_t taosMkDir(const char *dirname) {
   return code;
 }
 
+int32_t taosMulMkDir(const char *dirname) {
+  if (dirname == NULL) return -1;
+  char *temp = strdup(dirname);
+  char *pos = temp;
+  int32_t code = 0;
+
+  if (strncmp(temp, "/", 1) == 0) {
+    pos += 1;
+  } else if (strncmp(temp, "./", 2) == 0) {
+    pos += 2;
+  }
+  
+  for ( ; *pos != '\0'; pos++) {
+    if (*pos == '/') {
+      *pos = '\0';
+      code = mkdir(temp, 0755);
+      if (code < 0 && errno != EEXIST) {
+        free(temp);
+        return code;
+      }
+      *pos = '/';
+    }
+  }
+  
+  if (*(pos - 1) != '/') {
+    code = mkdir(temp, 0755);
+    if (code < 0 && errno != EEXIST) {
+      free(temp);
+      return code;
+    }
+  }
+  free(temp);
+
+  // int32_t code = mkdir(dirname, 0755);
+  if (code < 0 && errno == EEXIST) {
+    return 0;
+  }
+
+  return code;
+}
+
 void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
   DIR *dir = opendir(dirname);
   if (dir == NULL) return;

source/os/src/osFile.c

@@ -768,7 +768,7 @@ int32_t taosUmaskFile(int32_t maskVal) {
 }
 
 int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; }
-int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) {
+int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
   if (pFile == NULL) {
     return -1;
   }

source/os/src/osSystem.c

@@ -29,6 +29,8 @@
 struct termios oldtio;
 #endif
 
+typedef struct FILE TdCmd;
+
 void* taosLoadDll(const char* filename) {
 #if defined(WINDOWS)
   return NULL;
@@ -178,3 +180,33 @@ void resetTerminalMode() {
   }
 #endif
 }
+
+TdCmdPtr taosOpenCmd(const char *cmd) {
+  if (cmd == NULL) return NULL;
+  return (TdCmdPtr)popen(cmd, "r");
+}
+
+int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) {
+  if (pCmd == NULL) {
+    return -1;
+  }
+
+  size_t len = 0;
+  return getline(ptrBuf, &len, (FILE*)pCmd);
+}
+
+int32_t taosEOFCmd(TdCmdPtr pCmd) {
+  if (pCmd == NULL) {
+    return 0;
+  }
+  return feof((FILE*)pCmd);
+}
+
+int64_t taosCloseCmd(TdCmdPtr *ppCmd) {
+  if (ppCmd == NULL || *ppCmd == NULL) {
+    return 0;
+  }
+  pclose((FILE*)(*ppCmd));
+  *ppCmd = NULL;
+  return 0;
+}

tests/pytest/insert/binary.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import platform
 import sys
 from util.log import *
 from util.cases import *
@@ -13,6 +14,23 @@ class TDTestCase:
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 
+    def getPath(self, tool="taos"):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        paths = []
+        for root, dirs, files in os.walk(projPath):
+            if ((tool) in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    paths.append(os.path.join(root, tool))
+                    break
+        return paths[0]
+
     def run(self):
         tdSql.prepare()
 
@@ -53,16 +71,27 @@ class TDTestCase:
         tdLog.info("tdSql.checkData(0, 0, '34567')")
         tdSql.checkData(0, 0, '34567')
         tdLog.info("insert into tb values (now+4a, \"'';\")")
-        config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '')
+
-        result = ''.join(os.popen(r"""taos  -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines())
+        if platform.system() == "Linux":
-        if "Query OK" not in result: tdLog.exit("err:insert '';")            
+            config_dir = subprocess.check_output(
-        tdLog.info('drop database db')
+                str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"),
-        tdSql.execute('drop database db')
+                stderr=subprocess.STDOUT,
-        tdLog.info('show databases')
+                shell=True).decode('utf-8').replace(
-        tdSql.query('show databases')
+                    '\n',
-        tdLog.info('tdSql.checkRow(0)')
+                    '')
-        tdSql.checkRows(0)
+
-# convert end
+            binPath = self.getPath("taos")
+            if (binPath == ""):
+                tdLog.exit("taos not found!")
+            else:
+                tdLog.info("taos found: %s" % binPath)
+
+            result = ''.join(
+                os.popen(
+                    r"""%s -s "insert into db.tb values (now+4a, \"'';\")" -c %s""" %
+                    (binPath, (config_dir))).readlines())
+            if "Query OK" not in result:
+                tdLog.exit("err:insert '';")
 
     def stop(self):
         tdSql.close()

tests/script/jenkins/basic.txt

@@ -59,6 +59,15 @@
 ./test.sh -f tsim/tmq/oneTopic.sim
 ./test.sh -f tsim/tmq/multiTopic.sim
 
+./test.sh -f tsim/tmq/mainConsumerInMultiTopic.sim
+./test.sh -f tsim/tmq/mainConsumerInOneTopic.sim
+
+#fail ./test.sh -f tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim
+#fail ./test.sh -f tsim/tmq/main2Con1Cgrp1TopicFrStb.sim
+./test.sh -f tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim
+./test.sh -f tsim/tmq/main2Con1Cgrp2TopicFrStb.sim
+
+
 # --- stable
 ./test.sh -f tsim/stable/disk.sim
 ./test.sh -f tsim/stable/dnode3.sim
@@ -80,4 +89,7 @@
 ./test.sh -f tsim/qnode/basic1.sim -m
 ./test.sh -f tsim/mnode/basic1.sim -m
 
+# --- sma
+./test.sh -f tsim/sma/tsmaCreateInsertData.sim
+
 #======================b1-end===============

tests/script/tmp/back.sim

@@ -0,0 +1,6 @@
+sql connect
+$x = 1
+begin:
+  sql insert into db.tb values(now, $x ) -x begin  
+  $x = $x + 1   
+goto begin

tests/script/tmp/data.sim

@@ -0,0 +1,14 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create database db 
+sql create table db.tb (ts timestamp, i int)
+sql insert into db.tb values(now, 1)
+
+print ======== start back
+run_back tmp/back.sim
+
+sleep 2000
+return -1

tests/script/tsim/query/complex_group.sim

@@ -97,167 +97,167 @@ print ================ query 1 group by  filter
 sql select count(*) from ct3 group by c1
 print ====> sql : select count(*) from ct3 group by c1
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c2
 print ====> sql : select count(*) from ct3 group by c2
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c3
 print ====> sql : select count(*) from ct3 group by c3
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c4
 print ====> sql : select count(*) from ct3 group by c4
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c5
 print ====> sql : select count(*) from ct3 group by c5
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c6
 print ====> sql : select count(*) from ct3 group by c6
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c7
 print ====> sql : select count(*) from ct3 group by c7
 print ====> rows: $rows
-if $rows != 2 then
+if $rows != 3 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c8
 print ====> sql : select count(*) from ct3 group by c8
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c9
 print ====> sql : select count(*) from ct3 group by c9
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c10
 print ====> sql : select count(*) from ct3 group by c10
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 print ================ query 2 complex with group by
 sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
-print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
@@ -307,170 +307,170 @@ print ================ query 1 group by  filter
 sql select count(*) from ct3 group by c1
 print ====> sql : select count(*) from ct3 group by c1
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c2
 print ====> sql : select count(*) from ct3 group by c2
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c3
 print ====> sql : select count(*) from ct3 group by c3
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c4
 print ====> sql : select count(*) from ct3 group by c4
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c5
 print ====> sql : select count(*) from ct3 group by c5
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c6
 print ====> sql : select count(*) from ct3 group by c6
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c7
 print ====> sql : select count(*) from ct3 group by c7
 print ====> rows: $rows
-if $rows != 2 then
+if $rows != 3 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c8
 print ====> sql : select count(*) from ct3 group by c8
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c9
 print ====> sql : select count(*) from ct3 group by c9
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 sql select count(*) from ct3 group by c10
 print ====> sql : select count(*) from ct3 group by c10
 print ====> rows: $rows
-if $rows != 8 then
+if $rows != 9 then
     return -1
 endi
 
 print ================ query 2 complex with group by
 sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
-print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
-print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
+print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-#system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/query/complex_having.sim

@@ -77,9 +77,9 @@ sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL,
 print ================ start query ======================
 
 print ================ query 1 having condition
-sql_error select c1 from ct1 group by c1 having count(c1)
+sql select c1 from ct1 group by c1 having count(c1)
-sql_error select c1 from ct4 group by c1 having count(c1)
+sql select c1 from ct4 group by c1 having count(c1)
-sql_error select count(c1) from ct1 group by c1 having count(c1)
+sql select count(c1) from ct1 group by c1 having count(c1)
 
 sql  select sum(c1) ,count(c7) from ct4 group by c7 having count(c7) > 1  ;
 print ====> sql : select sum(c1) ,count(c7) from ct4 group by c7 having count(c7) > 1  ;
@@ -98,22 +98,22 @@ endi
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 2  and  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 2  and  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 2 then
+if $rows != 7 then
     return -1
 endi
 
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 2 then
+if $rows != 7 then
     return -1
 endi
 
 
 print ================ query 1 complex with  having condition
 
-sql select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) < 1 limit 1 offset 1
+sql select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) > 1 limit 1 offset 0
-print ====> sql : select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) < 1 limit 1 offset 1
+print ====> sql : select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) > 1 limit 1 offset 0
 print ====> rows: $rows
 if $rows != 1 then
     return -1
@@ -250,9 +250,9 @@ if $data00 != 20 then
 endi
 
 print ================ query 1 having condition
-sql_error select c1 from ct1 group by c1 having count(c1)
+sql select c1 from ct1 group by c1 having count(c1)
-sql_error select c1 from ct4 group by c1 having count(c1)
+sql select c1 from ct4 group by c1 having count(c1)
-sql_error select count(c1) from ct1 group by c1 having count(c1)
+sql select count(c1) from ct1 group by c1 having count(c1)
 
 sql  select sum(c1) ,count(c7) from ct4 group by c7 having count(c7) > 1  ;
 print ====> sql : select sum(c1) ,count(c7) from ct4 group by c7 having count(c7) > 1  ;
@@ -271,22 +271,22 @@ endi
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 2  and  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 2  and  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 2 then
+if $rows != 7 then
     return -1
 endi
 
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 2 then
+if $rows != 7 then
     return -1
 endi
 
 
 print ================ query 1 complex with  having condition
 
-sql select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) < 1 limit 1 offset 1
+sql select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) > 1 limit 1 offset 0
-print ====> sql : select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) < 1 limit 1 offset 1
+print ====> sql : select count(c1) from ct4 where c1 > 2 group by c7 having count(c1) > 1 limit 1 offset 0
 print ====> rows: $rows
 if $rows != 1 then
     return -1

tests/script/tsim/query/complex_where.sim

@@ -192,14 +192,14 @@ if $data01 != 1 then
     return -1
 endi
 
-sql select c1 from stb1 where stb1 > 5 and c1 <= 6
+sql select c1 from stb1 where c1 > 5 and c1 <= 6
 print ====> sql : select c1 from stb1 where c1 > 5 and c1 <= 6
 print ====> rows: $rows
 print ====> rows0: $data00
-if $rows != 4 then
+if $rows != 3 then
     return -1
 endi
-if $data01 != 6 then
+if $data00 != 6 then
     return -1
 endi
 
@@ -210,7 +210,7 @@ print ====> rows0: $data00
 if $rows != 32 then
     return -1
 endi
-if $data01 != 1 then
+if $data00 != 1 then
     return -1
 endi
 
@@ -221,7 +221,7 @@ print ====> rows0: $data00
 if $rows != 17 then
     return -1
 endi
-if $data01 != 5 then
+if $data00 != 5 then
     return -1
 endi
 
@@ -240,7 +240,7 @@ if $rows != 12 then
     return -1
 endi
 
-sql_error select c1 from stb1 where c7 between false and true
+sql select c1 from stb1 where c7 between false and true
 
 sql select c1 from stb1 where c1 in (1,2,3)
 print ====> sql : select c1 from stb1 where c1 in (1,2,3)
@@ -272,98 +272,98 @@ endi
 
 
 print ================ query 2 complex with where
-sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select count(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
@@ -510,14 +510,14 @@ if $data01 != 1 then
     return -1
 endi
 
-sql select c1 from stb1 where stb1 > 5 and c1 <= 6
+sql select c1 from stb1 where c1 > 5 and c1 <= 6
 print ====> sql : select c1 from stb1 where c1 > 5 and c1 <= 6
 print ====> rows: $rows
 print ====> rows0: $data00
-if $rows != 4 then
+if $rows != 3 then
     return -1
 endi
-if $data01 != 6 then
+if $data00 != 6 then
     return -1
 endi
 
@@ -528,7 +528,7 @@ print ====> rows0: $data00
 if $rows != 32 then
     return -1
 endi
-if $data01 != 1 then
+if $data00 != 1 then
     return -1
 endi
 
@@ -539,7 +539,7 @@ print ====> rows0: $data00
 if $rows != 17 then
     return -1
 endi
-if $data01 != 5 then
+if $data00 != 5 then
     return -1
 endi
 
@@ -558,7 +558,7 @@ if $rows != 12 then
     return -1
 endi
 
-sql_error select c1 from stb1 where c7 between false and true
+sql select c1 from stb1 where c7 between false and true
 
 sql select c1 from stb1 where c1 in (1,2,3)
 print ====> sql : select c1 from stb1 where c1 in (1,2,3)
@@ -590,98 +590,98 @@ endi
 
 
 print ================ query 2 complex with where
-sql select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select count(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select count(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select abs(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select abs(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select acos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select acos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select asin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select asin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select atan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select atan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select ceil(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select ceil(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select cos(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select cos(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select floor(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select floor(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select log(c1,10) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select log(c1,10) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select pow(c1,3) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select pow(c1,3) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select round(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select round(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sqrt(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select sqrt(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select sin(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select sin(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then
     return -1
 endi
 
-sql select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 1
+sql select tan(c1) from ct3 where c1 > 2 group by c1 limit 1 offset 1
 print ====> sql : select tan(c1) from ct3 where c1 > 2 group by c7 limit 1 offset 2
 print ====> rows: $rows
 if $rows != 1 then

tests/script/tsim/show/basic.sim

@@ -28,8 +28,8 @@ sql connect
 
 # select */column from information_schema.xxxx; xxxx include:
 # dnodes, mnodes, modules, qnodes, 
-# user_databases, user_functions, user_indexes, user_stables, user_streams, 
+# user_databases, user_functions, user_indexes, user_stables, user_streams,
-# user_tables, user_table_distributed, user_users, vgroups, 
+# user_tables, user_table_distributed, user_users, vgroups,
 
 print =============== add dnode2 into cluster
 sql create dnode $hostname port 7200

tests/script/tsim/sma/tsmaCreateInsertData.sim

@@ -0,0 +1,48 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+print =============== create database
+sql create database d1
+sql show databases
+if $rows != 2 then 
+  return -1
+endi
+
+print $data00 $data01 $data02
+
+sql use d1
+
+print =============== create super table, include column type for count/sum/min/max/first
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct1 using stb tags(1000)
+
+sql show tables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== insert data, mode1: one row one table in sql
+sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
+sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
+
+
+print =============== create sma index from super table
+sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(5m,10s) sliding(2m)
+print $data00 $data01 $data02  $data03
+
+print =============== trigger stream to execute sma aggr task and insert sma data into sma store
+sql insert into ct1 values(now+5s, 20, 20.0, 30.0)
+#===================================================================
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/consumerMain.sim

@@ -1,267 +0,0 @@
-#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
-# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
-# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
-# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
-#
-# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
-#
-######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
-######## This test case include scene1 and scene3
-######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
-
-system sh/stop_dnodes.sh
-system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
-system sh/exec.sh -n dnode1 -s start
-
-$loop_cnt = 0
-check_dnode_ready:
-	$loop_cnt = $loop_cnt + 1
-	sleep 200
-	if $loop_cnt == 10 then
-	  print ====> dnode not ready!
-		return -1
-	endi
-sql show dnodes
-print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
-if $data00 != 1 then
-  return -1
-endi
-if $data04 != ready then
-  goto check_dnode_ready
-endi
-
-sql connect
-
-$loop_cnt = 0
-$vgroups  = 1
-$dbNamme  = d0
-loop_vgroups:
-print =============== create database $dbNamme vgroups $vgroups
-sql create database $dbNamme vgroups $vgroups
-sql show databases
-print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
-print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
-print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
-
-if $loop_cnt == 0 then 
-  if $rows != 2 then 
-    return -1
-  endi
-  if $data02 != 1 then  # vgroups
-    print vgroups: $data02
-    return -1
-  endi
-else
-  if $rows != 3 then 
-    return -1
-  endi
-  if $data00 == d1 then 
-    if $data02 != 4 then  # vgroups
-      print vgroups: $data02
-      return -1
-    endi
-  else  
-    if $data12 != 4 then  # vgroups
-      print vgroups: $data12
-      return -1
-    endi
-  endi
-endi
-
-sql use $dbNamme
-
-print =============== create super table
-sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
-
-sql show stables
-if $rows != 1 then 
-  return -1
-endi
-
-print =============== create child table
-$tbPrefix = ct
-$tbNum = 2
-
-$i = 0
-while $i < $tbNum
-  $tb = $tbPrefix . $i
-  sql create table $tb using stb tags( $i )
-  $i = $i + 1
-endw
-
-print =============== create normal table
-sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
-
-print =============== create multi topics. notes: now only support:
-print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
-print =============== will support: * from stb
-
-sql create topic topic_stb_column as select ts, c1, c3 from stb
-#sql create topic topic_stb_all as select * from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
-
-sql create topic topic_ctb_column as select ts, c1, c3 from ct0
-sql create topic topic_ctb_all as select * from ct0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
-
-sql create topic topic_ntb_column as select ts, c1, c3 from ntb
-sql create topic topic_ntb_all as select * from ntb
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
-
-sql show tables
-if $rows != 3 then 
-  return -1
-endi
-
-print =============== run_back insert data
-
-if $loop_cnt == 0 then 
-  run_back tsim/tmq/insertDataV1.sim
-else
-  run_back tsim/tmq/insertDataV4.sim
-endi
-
-sleep 1000
-
-#$rowNum = 1000
-#$tstart = 1640966400000  # 2022-01-01 00:00:00.000
-#
-#$i = 0
-#while $i < $tbNum
-#  $tb = $tbPrefix . $i
-#
-#  $x = 0
-#  while $x < $rowNum
-#    $c = $x / 10
-#    $c = $c * 10
-#    $c = $x - $c
-#
-#    $binary = ' . binary
-#    $binary = $binary . $c
-#    $binary = $binary . '
-#
-#    sql insert into $tb values ($tstart , $c , $x , $binary )
-#    sql insert into ntb values ($tstart , $c , $x , $binary )
-#    $tstart = $tstart + 1
-#    $x = $x + 1
-#  endw
-#
-#  $i = $i + 1
-#  $tstart = 1640966400000
-#endw
-
-#root@trd02 /home $ tmq_sim --help                                                                          
-# -c      Configuration directory, default is 
-# -d      The name of the database for cosumer, no default 
-# -t      The topic string for cosumer, no default 
-# -k      The key-value string for cosumer, no default 
-# -g      showMsgFlag, default is 0
-#        
-
-$consumeDelay = 5000
-$expectConsumeMsgCnt = 1000
-print expectConsumeMsgCnt: $expectConsumeMsgCnt, consumeDelay: $consumeDelay
-
-# supported key:
-# group.id:<xxx>
-# enable.auto.commit:<true | false>
-# auto.offset.reset:<earliest | latest | none>
-# td.connect.ip:<fqdn | ipaddress>
-# td.connect.user:root
-# td.connect.pass:taosdata
-# td.connect.port:6030
-# td.connect.db:db
-
-$expect_result = @{consume success: @
-$expect_result = $expect_result . $expectConsumeMsgCnt
-$expect_result = $expect_result . @, @ 
-$expect_result = $expect_result . 0}
-print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-#print cmd result----> $system_content
-##if $system_content != @{consume success: 10000, 0}@ then
-#if $system_content != $expect_result then
-#  return -1
-#endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-#if $system_content != @{consume success: 10000, 0}@ then
-if $system_content >= $expect_result then
-  return -1
-endi
-
-$expect_result = @{consume success: @
-$expect_result = $expect_result . $rowNum
-$expect_result = $expect_result . @, @ 
-$expect_result = $expect_result . 0}
-print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-$expect_result = @{consume success: @
-$expect_result = $expect_result . $totalMsgCnt
-$expect_result = $expect_result . @, @ 
-$expect_result = $expect_result . 0}
-print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
-print cmd result----> $system_content
-if $system_content >= $expect_result then
-  return -1
-endi
-
-if $loop_cnt == 0 then
-  $loop_cnt = 1
-  $vgroups  = 4
-  $dbNamme  = d1
-  goto loop_vgroups
-endi
-
-
-#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/insertDataV1.sim

@@ -22,23 +22,19 @@ while $i < $tbNum
 
   $x = 0
   while $x < $rowNum
-    $c = $x / 10
-    $c = $c * 10
-    $c = $x - $c
-
     $binary = ' . binary
-    $binary = $binary . $c
+    $binary = $binary . $i
     $binary = $binary . '
 
-    #print ====> insert into $tb values ($tstart , $c , $x , $binary )
+    #print ====> insert into $tb values ($tstart , $i , $x , $binary )
-    #print ====> insert into ntb values ($tstart , $c , $x , $binary )
+    #print ====> insert into ntb values ($tstart , $i , $x , $binary )
-    sql insert into $tb values ($tstart , $c , $x , $binary )
+    sql insert into $tb values ($tstart , $i , $x , $binary )
-    sql insert into ntb values ($tstart , $c , $x , $binary )
+    sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' )
     $tstart = $tstart + 1
     $x = $x + 1
   endw
   
-  #print ====> insert rows: $rowNum into $tb and ntb
+  print ====> insert rows: $rowNum into $tb and ntb
 
   $i = $i + 1
 #  $tstart = 1640966400000

tests/script/tsim/tmq/insertDataV4.sim

@@ -22,18 +22,14 @@ while $i < $tbNum
 
   $x = 0
   while $x < $rowNum
-    $c = $x / 10
-    $c = $c * 10
-    $c = $x - $c
-
     $binary = ' . binary
-    $binary = $binary . $c
+    $binary = $binary . $i
     $binary = $binary . '
 
-    #print ====> insert into $tb values ($tstart , $c , $x , $binary )
+    #print ====> insert into $tb values ($tstart , $i , $x , $binary )
-    #print ====> insert into ntb values ($tstart , $c , $x , $binary )
+    #print ====> insert into ntb values ($tstart , $i , $x , $binary )
-    sql insert into $tb values ($tstart , $c , $x , $binary )
+    sql insert into $tb values ($tstart , $i , $x , $binary )
-    sql insert into ntb values ($tstart , $c , $x , $binary )
+    sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' )
     $tstart = $tstart + 1
     $x = $x + 1
   endw

tests/script/tsim/tmq/insertFixedDataV2.sim

@@ -0,0 +1,51 @@
+
+sql connect
+
+print ================ insert data
+$dbNamme  = d0
+$tbPrefix = ct
+$tbNum    = 10
+$rowNum   = 100
+$tstart   = 1640966400000  # 2022-01-01 00:00:00.000
+
+$loopInsertNum = 10
+
+sql use $dbNamme
+
+$loopIndex = 0
+
+loop_insert:
+print ====> loop $loopIndex insert
+$loopIndex = $loopIndex + 1
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $binary = ' . binary
+    $binary = $binary . $i
+    $binary = $binary . '
+
+    #print ====> insert into $tb values ($tstart , $i , $x , $binary )
+    #print ====> insert into ntb values ($tstart , $i , $x , $binary )
+    sql insert into $tb values ($tstart , $i , $x , $binary )
+    sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+  
+  #print ====> insert rows: $rowNum into $tb and ntb
+
+  $i = $i + 1
+#  $tstart = 1640966400000
+endw
+
+
+if $loopIndex < $loopInsertNum then
+  goto loop_insert
+endi
+
+print ====> insert data end ===========
+

tests/script/tsim/tmq/insertFixedDataV4.sim

@@ -0,0 +1,51 @@
+
+sql connect
+
+print ================ insert data
+$dbNamme  = d1
+$tbPrefix = ct
+$tbNum    = 10
+$rowNum   = 100
+$tstart   = 1640966400000  # 2022-01-01 00:00:00.000
+
+$loopInsertNum = 10
+
+sql use $dbNamme
+
+$loopIndex = 0
+
+loop_insert:
+print ====> loop $loopIndex insert
+$loopIndex = $loopIndex + 1
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $binary = ' . binary
+    $binary = $binary . $i
+    $binary = $binary . '
+
+    #print ====> insert into $tb values ($tstart , $i , $x , $binary )
+    #print ====> insert into ntb values ($tstart , $i , $x , $binary )
+    sql insert into $tb values ($tstart , $i , $x , $binary )
+    sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+  
+  #print ====> insert rows: $rowNum into $tb and ntb
+
+  $i = $i + 1
+#  $tstart = 1640966400000
+endw
+
+
+if $loopIndex < $loopInsertNum then
+  goto loop_insert
+endi
+
+print ====> insert data end ===========
+

tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim

@@ -0,0 +1,265 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+######## This test case include scene1 and scene3
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$loop_cnt = 0
+$vgroups  = 2
+$dbNamme  = d0
+loop_vgroups:
+print =============== create database $dbNamme vgroups $vgroups
+sql create database $dbNamme vgroups $vgroups
+sql show databases
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+
+if $loop_cnt == 0 then 
+  if $rows != 2 then 
+    return -1
+  endi
+  if $data02 != 2 then  # vgroups
+    print vgroups: $data02
+    return -1
+  endi
+else
+  if $rows != 3 then 
+    return -1
+  endi
+  if $data00 == d1 then 
+    if $data02 != 4 then  # vgroups
+      print vgroups: $data02
+      return -1
+    endi
+  else  
+    if $data12 != 4 then  # vgroups
+      print vgroups: $data12
+      return -1
+    endi
+  endi
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+$tbPrefix = ct
+$tbNum = 100
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using stb tags( $i )
+  $i = $i + 1
+endw
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create topics from child table
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
+
+#sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+#sql create topic topic_ntb_all as select * from ntb
+#sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
+
+sql show tables
+if $rows != 101 then 
+  return -1
+endi
+
+print =============== run_back insert data
+
+if $loop_cnt == 0 then 
+  run_back tsim/tmq/insertFixedDataV2.sim
+else
+  run_back tsim/tmq/insertFixedDataV4.sim
+endi
+
+#sleep 1000
+
+#$rowNum = 1000
+#$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+#
+#$i = 0
+#while $i < $tbNum
+#  $tb = $tbPrefix . $i
+#
+#  $x = 0
+#  while $x < $rowNum
+#    $c = $x / 10
+#    $c = $c * 10
+#    $c = $x - $c
+#
+#    $binary = ' . binary
+#    $binary = $binary . $c
+#    $binary = $binary . '
+#
+#    sql insert into $tb values ($tstart , $c , $x , $binary )
+#    sql insert into ntb values ($tstart , $c , $x , $binary )
+#    $tstart = $tstart + 1
+#    $x = $x + 1
+#  endw
+#
+#  $i = $i + 1
+#  $tstart = 1640966400000
+#endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$tbNum = 10
+$consumeDelay = 10
+$expectMsgCntFromCtb = 300
+$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum
+print consumeDelay: $consumeDelay
+print insert data child num: $tbNum
+print expectMsgCntFromCtb: $expectMsgCntFromCtb 
+print expectMsgCntFromStb: $expectMsgCntFromStb 
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $rowNum
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $totalMsgCnt
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $expectConsumeMsgCnt
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+##print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+##system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+##print cmd result----> $system_content
+###if $system_content != @{consume success: 10000, 0}@ then
+##if $system_content != success then
+##  return -1
+##endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#print cmd result----> $system_content
+##if $system_content != @{consume success: 10000, 0}@ then
+#if $system_content != success then
+#  return -1
+#endi
+
+if $loop_cnt == 0 then
+  $loop_cnt = 1
+  $vgroups  = 4
+  $dbNamme  = d1
+  goto loop_vgroups
+endi
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim

@@ -0,0 +1,270 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+######## This test case include scene1 and scene3
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$loop_cnt = 0
+$vgroups  = 2
+$dbNamme  = d0
+loop_vgroups:
+print =============== create database $dbNamme vgroups $vgroups
+sql create database $dbNamme vgroups $vgroups
+sql show databases
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+
+if $loop_cnt == 0 then 
+  if $rows != 2 then 
+    return -1
+  endi
+  if $data02 != 2 then  # vgroups
+    print vgroups: $data02
+    return -1
+  endi
+else
+  if $rows != 3 then 
+    return -1
+  endi
+  if $data00 == d1 then 
+    if $data02 != 4 then  # vgroups
+      print vgroups: $data02
+      return -1
+    endi
+  else  
+    if $data12 != 4 then  # vgroups
+      print vgroups: $data12
+      return -1
+    endi
+  endi
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+$tbPrefix = ct
+$tbNum = 100
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using stb tags( $i )
+  $i = $i + 1
+endw
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create multi topics. notes: now only support:
+print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
+print =============== will support: * from stb
+
+sql create topic topic_stb_column as select ts, c1, c3 from stb
+#sql create topic topic_stb_all as select * from stb
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
+
+sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+sql create topic topic_ntb_all as select * from ntb
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
+
+sql show tables
+if $rows != 101 then 
+  return -1
+endi
+
+print =============== run_back insert data
+
+if $loop_cnt == 0 then 
+  run_back tsim/tmq/insertFixedDataV2.sim
+else
+  run_back tsim/tmq/insertFixedDataV4.sim
+endi
+
+#sleep 1000
+
+#$rowNum = 1000
+#$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+#
+#$i = 0
+#while $i < $tbNum
+#  $tb = $tbPrefix . $i
+#
+#  $x = 0
+#  while $x < $rowNum
+#    $c = $x / 10
+#    $c = $c * 10
+#    $c = $x - $c
+#
+#    $binary = ' . binary
+#    $binary = $binary . $c
+#    $binary = $binary . '
+#
+#    sql insert into $tb values ($tstart , $c , $x , $binary )
+#    sql insert into ntb values ($tstart , $c , $x , $binary )
+#    $tstart = $tstart + 1
+#    $x = $x + 1
+#  endw
+#
+#  $i = $i + 1
+#  $tstart = 1640966400000
+#endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$tbNum = 10
+$consumeDelay = 10
+$expectMsgCntFromCtb = 300
+$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum
+print consumeDelay: $consumeDelay
+print insert data child num: $tbNum
+print expectMsgCntFromCtb: $expectMsgCntFromCtb 
+print expectMsgCntFromStb: $expectMsgCntFromStb 
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $rowNum
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $totalMsgCnt
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $expectConsumeMsgCnt
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#print cmd result----> $system_content
+##if $system_content != @{consume success: 10000, 0}@ then
+#if $system_content != success then
+#  return -1
+#endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+print cmd result----> $system_content
+#if $system_content != @{consume success: 10000, 0}@ then
+if $system_content != success then
+  return -1
+endi
+if $loop_cnt == 0 then
+  $loop_cnt = 1
+  $vgroups  = 4
+  $dbNamme  = d1
+  goto loop_vgroups
+endi
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim

@@ -0,0 +1,265 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+######## This test case include scene1 and scene3
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$loop_cnt = 0
+$vgroups  = 2
+$dbNamme  = d0
+loop_vgroups:
+print =============== create database $dbNamme vgroups $vgroups
+sql create database $dbNamme vgroups $vgroups
+sql show databases
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+
+if $loop_cnt == 0 then 
+  if $rows != 2 then 
+    return -1
+  endi
+  if $data02 != 2 then  # vgroups
+    print vgroups: $data02
+    return -1
+  endi
+else
+  if $rows != 3 then 
+    return -1
+  endi
+  if $data00 == d1 then 
+    if $data02 != 4 then  # vgroups
+      print vgroups: $data02
+      return -1
+    endi
+  else  
+    if $data12 != 4 then  # vgroups
+      print vgroups: $data12
+      return -1
+    endi
+  endi
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+$tbPrefix = ct
+$tbNum = 100
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using stb tags( $i )
+  $i = $i + 1
+endw
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create topics from child table
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
+
+#sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+#sql create topic topic_ntb_all as select * from ntb
+#sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
+
+sql show tables
+if $rows != 101 then 
+  return -1
+endi
+
+print =============== run_back insert data
+
+if $loop_cnt == 0 then 
+  run_back tsim/tmq/insertFixedDataV2.sim
+else
+  run_back tsim/tmq/insertFixedDataV4.sim
+endi
+
+#sleep 1000
+
+#$rowNum = 1000
+#$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+#
+#$i = 0
+#while $i < $tbNum
+#  $tb = $tbPrefix . $i
+#
+#  $x = 0
+#  while $x < $rowNum
+#    $c = $x / 10
+#    $c = $c * 10
+#    $c = $x - $c
+#
+#    $binary = ' . binary
+#    $binary = $binary . $c
+#    $binary = $binary . '
+#
+#    sql insert into $tb values ($tstart , $c , $x , $binary )
+#    sql insert into ntb values ($tstart , $c , $x , $binary )
+#    $tstart = $tstart + 1
+#    $x = $x + 1
+#  endw
+#
+#  $i = $i + 1
+#  $tstart = 1640966400000
+#endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$tbNum = 10
+$consumeDelay = 10
+$expectMsgCntFromCtb = 300
+$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum
+print consumeDelay: $consumeDelay
+print insert data child num: $tbNum
+print expectMsgCntFromCtb: $expectMsgCntFromCtb 
+print expectMsgCntFromStb: $expectMsgCntFromStb 
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $rowNum
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $totalMsgCnt
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $expectConsumeMsgCnt
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+##print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+##system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+##print cmd result----> $system_content
+###if $system_content != @{consume success: 10000, 0}@ then
+##if $system_content != success then
+##  return -1
+##endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
+#print cmd result----> $system_content
+##if $system_content != @{consume success: 10000, 0}@ then
+#if $system_content != success then
+#  return -1
+#endi
+
+if $loop_cnt == 0 then
+  $loop_cnt = 1
+  $vgroups  = 4
+  $dbNamme  = d1
+  goto loop_vgroups
+endi
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim

@@ -0,0 +1,270 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+######## This test case include scene1 and scene3
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$loop_cnt = 0
+$vgroups  = 2
+$dbNamme  = d0
+loop_vgroups:
+print =============== create database $dbNamme vgroups $vgroups
+sql create database $dbNamme vgroups $vgroups
+sql show databases
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+
+if $loop_cnt == 0 then 
+  if $rows != 2 then 
+    return -1
+  endi
+  if $data02 != 2 then  # vgroups
+    print vgroups: $data02
+    return -1
+  endi
+else
+  if $rows != 3 then 
+    return -1
+  endi
+  if $data00 == d1 then 
+    if $data02 != 4 then  # vgroups
+      print vgroups: $data02
+      return -1
+    endi
+  else  
+    if $data12 != 4 then  # vgroups
+      print vgroups: $data12
+      return -1
+    endi
+  endi
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+$tbPrefix = ct
+$tbNum = 100
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using stb tags( $i )
+  $i = $i + 1
+endw
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create multi topics. notes: now only support:
+print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
+print =============== will support: * from stb
+
+sql create topic topic_stb_column as select ts, c1, c3 from stb
+#sql create topic topic_stb_all as select * from stb
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
+
+sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+sql create topic topic_ntb_all as select * from ntb
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
+
+sql show tables
+if $rows != 101 then 
+  return -1
+endi
+
+print =============== run_back insert data
+
+if $loop_cnt == 0 then 
+  run_back tsim/tmq/insertFixedDataV2.sim
+else
+  run_back tsim/tmq/insertFixedDataV4.sim
+endi
+
+#sleep 1000
+
+#$rowNum = 1000
+#$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+#
+#$i = 0
+#while $i < $tbNum
+#  $tb = $tbPrefix . $i
+#
+#  $x = 0
+#  while $x < $rowNum
+#    $c = $x / 10
+#    $c = $c * 10
+#    $c = $x - $c
+#
+#    $binary = ' . binary
+#    $binary = $binary . $c
+#    $binary = $binary . '
+#
+#    sql insert into $tb values ($tstart , $c , $x , $binary )
+#    sql insert into ntb values ($tstart , $c , $x , $binary )
+#    $tstart = $tstart + 1
+#    $x = $x + 1
+#  endw
+#
+#  $i = $i + 1
+#  $tstart = 1640966400000
+#endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$tbNum = 10
+$consumeDelay = 10
+$expectMsgCntFromCtb = 300
+$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum
+print consumeDelay: $consumeDelay
+print insert data child num: $tbNum
+print expectMsgCntFromCtb: $expectMsgCntFromCtb 
+print expectMsgCntFromStb: $expectMsgCntFromStb 
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $rowNum
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#$expect_result = @{consume success: @
+#$expect_result = $expect_result . $totalMsgCnt
+#$expect_result = $expect_result . @, @ 
+#$expect_result = $expect_result . 0}
+#print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+#
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
+#print cmd result----> $system_content
+#if $system_content != success then
+#  return -1
+#endi
+
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $expectMsgCntFromStb
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1
+print cmd result----> $system_content
+if $system_content != success then
+  return -1
+endi
+
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1
+#print cmd result----> $system_content
+##if $system_content != @{consume success: 10000, 0}@ then
+#if $system_content != success then
+#  return -1
+#endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1
+print cmd result----> $system_content
+#if $system_content != @{consume success: 10000, 0}@ then
+if $system_content != success then
+  return -1
+endi
+if $loop_cnt == 0 then
+  $loop_cnt = 1
+  $vgroups  = 4
+  $dbNamme  = d1
+  goto loop_vgroups
+endi
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/mainConsumerInMultiTopic.sim

@@ -120,9 +120,9 @@ endi
 print =============== run_back insert data
 
 if $loop_cnt == 0 then 
-  run_back tsim/tmq/insertDataV1.sim
+  run_back tsim/tmq/insertFixedDataV2.sim
 else
-  run_back tsim/tmq/insertDataV4.sim
+  run_back tsim/tmq/insertFixedDataV4.sim
 endi
 
 #sleep 1000
@@ -162,9 +162,16 @@ endi
 # -g      showMsgFlag, default is 0
 #        
 
-$consumeDelay = 50
+$tbNum = 10
-$consumeMsgCntFromTopic = 1000
+$consumeDelay = 5
-print consumeMsgCntFromTopic: $consumeMsgCntFromTopic , consumeDelay: $consumeDelay
+$expectMsgCntFromCtb = 1000
+$expectMsgCntFromNtb = 1000
+$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum
+print consumeDelay: $consumeDelay
+print insert data child num: $tbNum
+print expectMsgCntFromCtb: $expectMsgCntFromCtb 
+print expectMsgCntFromStb: $expectMsgCntFromStb 
+
 
 # supported key:
 # group.id:<xxx>
@@ -177,49 +184,50 @@ print consumeMsgCntFromTopic: $consumeMsgCntFromTopic , consumeDelay: $consumeDe
 # td.connect.db:db
 
 $numOfTopics = 2
-$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics
+$expectMsgCntFromStb = $expectMsgCntFromStb * $numOfTopics
 $expect_result = @{consume success: @
-$expect_result = $expect_result . $expectConsumeMsgCnt
+$expect_result = $expect_result . $expectMsgCntFromStb
 $expect_result = $expect_result . @, @ 
 $expect_result = $expect_result . 0}
 print expect_result----> $expect_result
-#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
-#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
 print cmd result----> $system_content
 #if $system_content != @{consume success: 20000, 0}@ then
-if $system_content < $expect_result then
+if $system_content != $expect_result then
   return -1
 endi
 
 $numOfTopics = 3
-$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics
+$expectMsgCntFromCtb = $expectMsgCntFromCtb * $numOfTopics
 $expect_result = @{consume success: @
-$expect_result = $expect_result . $expectConsumeMsgCnt
+$expect_result = $expect_result . $expectMsgCntFromCtb
 $expect_result = $expect_result . @, @ 
 $expect_result = $expect_result . 0}
 print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
 print cmd result----> $system_content
 #if $system_content != @{consume success: 300, 0}@ then
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
 $numOfTopics = 3
-$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics
+$expectMsgCntFromNtb = $expectMsgCntFromNtb * $tbNum
+$expectMsgCntFromNtb = $expectMsgCntFromNtb * $numOfTopics
 $expect_result = @{consume success: @
-$expect_result = $expect_result . $expectConsumeMsgCnt
+$expect_result = $expect_result . $expectMsgCntFromNtb
 $expect_result = $expect_result . @, @ 
 $expect_result = $expect_result . 0}
 print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb
 print cmd result----> $system_content
 #if $system_content != @{consume success: 30000, 0}@ then
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 

tests/script/tsim/tmq/mainConsumerInOneTopic.sim

@@ -120,9 +120,9 @@ endi
 print =============== run_back insert data
 
 if $loop_cnt == 0 then 
-  run_back tsim/tmq/insertDataV1.sim
+  run_back tsim/tmq/insertFixedDataV2.sim
 else
-  run_back tsim/tmq/insertDataV4.sim
+  run_back tsim/tmq/insertFixedDataV4.sim
 endi
 
 #sleep 1000
@@ -162,9 +162,15 @@ endi
 # -g      showMsgFlag, default is 0
 #        
 
-$consumeDelay = 50
+$tbNum = 10
-$expectConsumeMsgCnt = 1000
+$consumeDelay = 5
-print expectConsumeMsgCnt: $expectConsumeMsgCnt , consumeDelay: $consumeDelay
+$expectMsgCntFromCtb = 1000
+$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum
+print consumeDelay: $consumeDelay
+print insert data child num: $tbNum
+print expectMsgCntFromCtb: $expectMsgCntFromCtb 
+print expectMsgCntFromStb: $expectMsgCntFromStb 
+
 
 # supported key:
 # group.id:<xxx>
@@ -177,82 +183,82 @@ print expectConsumeMsgCnt: $expectConsumeMsgCnt , consumeDelay: $consumeDelay
 # td.connect.db:db
 
 $expect_result = @{consume success: @
-$expect_result = $expect_result . $expectConsumeMsgCnt
+$expect_result = $expect_result . $expectMsgCntFromStb
 $expect_result = $expect_result . @, @ 
 $expect_result = $expect_result . 0}
 print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
 print cmd result----> $system_content
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
-#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
-#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
 #print cmd result----> $system_content
 ##if $system_content != @{consume success: 10000, 0}@ then
-#if $system_content < $expectConsumeMsgCnt then
+#if $system_content != $expect_result then
 #  return -1
 #endi
 
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb
 print cmd result----> $system_content
 #if $system_content != @{consume success: 10000, 0}@ then
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
 $expect_result = @{consume success: @
-$expect_result = $expect_result . $rowNum
+$expect_result = $expect_result . $expectMsgCntFromCtb
 $expect_result = $expect_result . @, @ 
 $expect_result = $expect_result . 0}
 print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
 print cmd result----> $system_content
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
 print cmd result----> $system_content
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
 print cmd result----> $system_content
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
 $expect_result = @{consume success: @
-$expect_result = $expect_result . $totalMsgCnt
+$expect_result = $expect_result . $expectMsgCntFromStb
 $expect_result = $expect_result . @, @ 
 $expect_result = $expect_result . 0}
 print expect_result----> $expect_result
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
 print cmd result----> $system_content
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
 print cmd result----> $system_content
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 
-print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
-system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb
 print cmd result----> $system_content
-if $system_content < $expectConsumeMsgCnt then
+if $system_content != $expect_result then
   return -1
 endi
 

tests/test/c/tmqDemo.c

@@ -195,7 +195,7 @@ void parseArgument(int32_t argc, char *argv[]) {
 }
 
 static int  running = 1;
-static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }
+/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/
 
 // calc dir size (not include itself 4096Byte)
 int64_t getDirectorySize(char *dir)
@@ -363,9 +363,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   }
 
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1);
     if (tmqmessage) {
-      msg_process(tmqmessage);
+      /*msg_process(tmqmessage);*/
       tmq_message_destroy(tmqmessage);
 
       if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
@@ -392,12 +392,12 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog
   int32_t skipLogNum = 0;
   int64_t startTime = taosGetTimestampUs();
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 3000);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000);
     if (tmqmessage) {
       batchCnt++;
-      skipLogNum += tmqGetSkipLogNum(tmqmessage);
+      /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/
 	  if (0 != g_stConfInfo.showMsgFlag) {
-        msg_process(tmqmessage);
+        /*msg_process(tmqmessage);*/
 	  }
       tmq_message_destroy(tmqmessage);
     } else {

tests/test/c/tmqSim.c

@@ -13,51 +13,65 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-// clang-format off
-
 #include <assert.h>
+#include <dirent.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <time.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <time.h>
 #include <unistd.h>
-#include <stdlib.h>
-#include <dirent.h>
 
 #include "taos.h"
 #include "taoserror.h"
 #include "tlog.h"
 
-#define GREEN "\033[1;32m"
+#define GREEN     "\033[1;32m"
-#define NC "\033[0m"
+#define NC        "\033[0m"
 #define min(a, b) (((a) < (b)) ? (a) : (b))
 
-#define  MAX_SQL_STR_LEN    (1024 * 1024)
+#define MAX_SQL_STR_LEN (1024 * 1024)
-#define  MAX_ROW_STR_LEN    (16 * 1024)
+#define MAX_ROW_STR_LEN (16 * 1024)
 
 typedef struct {
-    // input from argvs
+  int32_t   expectMsgCnt;  
-	char	dbName[32];
+  int32_t   consumeMsgCnt;
-    char    topicString[256];
+  TdThread thread;
-	char    keyString[1024];
+} SThreadInfo;
-	int32_t showMsgFlag;
-	int32_t consumeDelay;    // unit s
-	int32_t consumeMsgCnt;
 
-    // save result after parse agrvs	
+typedef struct {
-	int32_t numOfTopic;
+  // input from argvs
-	char    topics[32][64];
+  char    dbName[32];
+  char    topicString[256];
+  char    keyString[1024];
+    char    topicString1[256];
+	char    keyString1[1024];
+  int32_t showMsgFlag;
+  int32_t consumeDelay;  // unit s
+  int32_t consumeMsgCnt;
+	int32_t checkMode;
+
+  // save result after parse agrvs
+  int32_t numOfTopic;
+  char    topics[32][64];
+
+  int32_t numOfKey;
+  char    key[32][64];
+  char    value[32][64];
 	
-	int32_t numOfKey;
+	int32_t numOfTopic1;
-	char    key[32][64];
+	char    topics1[32][64];
-	char    value[32][64];
+	
+	int32_t numOfKey1;
+	char    key1[32][64];
+	char    value1[32][64];
 } SConfInfo;
 
 static SConfInfo g_stConfInfo;
 
-//char* g_pRowValue = NULL;
+// char* g_pRowValue = NULL;
-//TdFilePtr g_fp = NULL;
+// TdFilePtr g_fp = NULL;
 
 static void printHelp() {
   char indent[10] = "        ";
@@ -71,22 +85,27 @@ static void printHelp() {
   printf("%s%s%s\n", indent, indent, "The topic string for cosumer, no default ");
   printf("%s%s\n", indent, "-k");
   printf("%s%s%s\n", indent, indent, "The key-value string for cosumer, no default ");
+  printf("%s%s\n", indent, "-t1");
+  printf("%s%s%s\n", indent, indent, "The topic1 string for cosumer, no default ");
+  printf("%s%s\n", indent, "-k1");
+  printf("%s%s%s\n", indent, indent, "The key1-value1 string for cosumer, no default ");
   printf("%s%s\n", indent, "-g");
   printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag);
   printf("%s%s\n", indent, "-y");
   printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay);
   printf("%s%s\n", indent, "-m");
   printf("%s%s%s%d\n", indent, indent, "consume msg count, default is s", g_stConfInfo.consumeMsgCnt);
+  printf("%s%s\n", indent, "-j");
+  printf("%s%s%s%d\n", indent, indent, "check mode, default is s", g_stConfInfo.checkMode);
   exit(EXIT_SUCCESS);
 }
 
-void parseArgument(int32_t argc, char *argv[]) {
+void parseArgument(int32_t argc, char* argv[]) {
-
   memset(&g_stConfInfo, 0, sizeof(SConfInfo));
-  g_stConfInfo.showMsgFlag   = 0;
+  g_stConfInfo.showMsgFlag = 0;
-  g_stConfInfo.consumeDelay  = 8000;
+  g_stConfInfo.consumeDelay = 8000;
   g_stConfInfo.consumeMsgCnt = 0;
-  
+
   for (int32_t i = 1; i < argc; i++) {
     if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
       printHelp();
@@ -99,15 +118,21 @@ void parseArgument(int32_t argc, char *argv[]) {
       strcpy(g_stConfInfo.topicString, argv[++i]);
     } else if (strcmp(argv[i], "-k") == 0) {
       strcpy(g_stConfInfo.keyString, argv[++i]);
+    } else if (strcmp(argv[i], "-t1") == 0) {
+      strcpy(g_stConfInfo.topicString1, argv[++i]);
+    } else if (strcmp(argv[i], "-k1") == 0) {
+      strcpy(g_stConfInfo.keyString1, argv[++i]);
     } else if (strcmp(argv[i], "-g") == 0) {
       g_stConfInfo.showMsgFlag = atol(argv[++i]);
     } else if (strcmp(argv[i], "-y") == 0) {
       g_stConfInfo.consumeDelay = atol(argv[++i]);
     } else if (strcmp(argv[i], "-m") == 0) {
       g_stConfInfo.consumeMsgCnt = atol(argv[++i]);
+    } else if (strcmp(argv[i], "-j") == 0) {
+      g_stConfInfo.checkMode = atol(argv[++i]);
     } else {
       printf("%s unknow para: %s %s", GREEN, argv[++i], NC);
-	  exit(-1);
+      exit(-1);
     }
   }
 
@@ -117,64 +142,89 @@ void parseArgument(int32_t argc, char *argv[]) {
   pPrint("%s topicString:%s %s", GREEN, g_stConfInfo.topicString, NC);
   pPrint("%s keyString:%s %s", GREEN, g_stConfInfo.keyString, NC);  
   pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC);
-#endif  
+#endif
 }
 
-void splitStr(char **arr, char *str, const char *del) {
+void splitStr(char** arr, char* str, const char* del) {
-  char *s = strtok(str, del);
+  char* s = strtok(str, del);
-  while(s != NULL) {
+  while (s != NULL) {
     *arr++ = s;
     s = strtok(NULL, del);
   }
 }
 
-void ltrim(char *str) 
+void ltrim(char* str) {
-{ 
+  if (str == NULL || *str == '\0') {
-    if (str == NULL || *str == '\0') 
+    return;
-    { 
+  }
-        return; 
+  int   len = 0;
-    } 
+  char* p = str;
-    int len = 0; 
+  while (*p != '\0' && isspace(*p)) {
-    char *p = str;
+    ++p;
-    while (*p != '\0' && isspace(*p)) 
+    ++len;
-    { 
+  }
-        ++p; ++len; 
+  memmove(str, p, strlen(str) - len + 1);
-    } 
+  // return str;
-    memmove(str, p, strlen(str) - len + 1); 
-    //return str; 
 }
 
-
 void parseInputString() {
-  //printf("topicString: %s\n", g_stConfInfo.topicString);
+  // printf("topicString: %s\n", g_stConfInfo.topicString);
-  //printf("keyString: %s\n\n", g_stConfInfo.keyString);  
+  // printf("keyString: %s\n\n", g_stConfInfo.keyString);
 
-  char *token;
+  char*      token;
   const char delim[2] = ",";
   const char ch = ':';
 
   token = strtok(g_stConfInfo.topicString, delim);
+  while (token != NULL) {
+    // printf("%s\n", token );
+    strcpy(g_stConfInfo.topics[g_stConfInfo.numOfTopic], token);
+    ltrim(g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
+    // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
+    g_stConfInfo.numOfTopic++;
+	
+    token = strtok(NULL, delim);
+  }  
+
+  token = strtok(g_stConfInfo.topicString1, delim);
   while(token != NULL) {
     //printf("%s\n", token );
-	strcpy(g_stConfInfo.topics[g_stConfInfo.numOfTopic], token);  
+	strcpy(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1], token);  
-    ltrim(g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
+    ltrim(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1]);
 	//printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
-	g_stConfInfo.numOfTopic++;
+	g_stConfInfo.numOfTopic1++;
 	
     token = strtok(NULL, delim);
   }
 
   token = strtok(g_stConfInfo.keyString, delim);
+  while (token != NULL) {
+    // printf("%s\n", token );
+    {
+      char* pstr = token;
+      ltrim(pstr);
+      char* ret = strchr(pstr, ch);
+      memcpy(g_stConfInfo.key[g_stConfInfo.numOfKey], pstr, ret - pstr);
+      strcpy(g_stConfInfo.value[g_stConfInfo.numOfKey], ret + 1);
+      // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey],
+      // g_stConfInfo.value[g_stConfInfo.numOfKey]);
+      g_stConfInfo.numOfKey++;
+    }
+	
+    token = strtok(NULL, delim);
+  }
+
+  token = strtok(g_stConfInfo.keyString1, delim);
   while(token != NULL) {
     //printf("%s\n", token );
   	{
 	char* pstr = token;
 	ltrim(pstr);
 	char *ret = strchr(pstr, ch);
-	memcpy(g_stConfInfo.key[g_stConfInfo.numOfKey], pstr, ret-pstr);	
+	memcpy(g_stConfInfo.key1[g_stConfInfo.numOfKey1], pstr, ret-pstr);	
-	strcpy(g_stConfInfo.value[g_stConfInfo.numOfKey], ret+1);
+	strcpy(g_stConfInfo.value1[g_stConfInfo.numOfKey1], ret+1);
 	//printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], g_stConfInfo.value[g_stConfInfo.numOfKey]);
-	g_stConfInfo.numOfKey++;
+	g_stConfInfo.numOfKey1++;
     }
 	
     token = strtok(NULL, delim);
@@ -182,25 +232,58 @@ void parseInputString() {
 }
 
 
-static int  running = 1;
+static int running = 1;
-static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }
+/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/
 
-
+int queryDB(TAOS* taos, char* command) {
-int queryDB(TAOS *taos, char *command) {
+  TAOS_RES* pRes = taos_query(taos, command);
-	TAOS_RES *pRes = taos_query(taos, command);
+  int       code = taos_errno(pRes);
-	int code = taos_errno(pRes);
+  // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
-	//if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
+  if (code != 0) {
-	if (code != 0) {
+    pError("failed to reason:%s, sql: %s", tstrerror(code), command);
-	  pError("failed to reason:%s, sql: %s", tstrerror(code), command);
+    taos_free_result(pRes);
-	  taos_free_result(pRes);
+    return -1;
-	  return -1;
+  }
-	}
+  taos_free_result(pRes);
-	taos_free_result(pRes);
+  return 0;
-	return 0 ;
 }
 
 tmq_t* build_consumer() {
   char sqlStr[1024] = {0};
+
+  TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  assert(pConn != NULL);
+
+  sprintf(sqlStr, "use %s", g_stConfInfo.dbName);
+  TAOS_RES* pRes = taos_query(pConn, sqlStr);
+  if (taos_errno(pRes) != 0) {
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+    taos_free_result(pRes);
+    exit(-1);
+  }
+  taos_free_result(pRes);
+
+  tmq_conf_t* conf = tmq_conf_new();
+  // tmq_conf_set(conf, "group.id", "tg2");
+  for (int32_t i = 0; i < g_stConfInfo.numOfKey; i++) {
+    tmq_conf_set(conf, g_stConfInfo.key[i], g_stConfInfo.value[i]);
+  }
+  tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0);
+  return tmq;
+}
+
+tmq_list_t* build_topic_list() {
+  tmq_list_t* topic_list = tmq_list_new();
+  // tmq_list_append(topic_list, "test_stb_topic_1");
+  for (int32_t i = 0; i < g_stConfInfo.numOfTopic; i++) {
+    tmq_list_append(topic_list, g_stConfInfo.topics[i]);
+  }
+  return topic_list;
+}
+
+
+tmq_t* build_consumer_x() {
+  char sqlStr[1024] = {0};
   
   TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
@@ -216,18 +299,18 @@ tmq_t* build_consumer() {
 
   tmq_conf_t* conf = tmq_conf_new();
   //tmq_conf_set(conf, "group.id", "tg2");
-  for (int32_t i = 0; i < g_stConfInfo.numOfKey; i++) {
+  for (int32_t i = 0; i < g_stConfInfo.numOfKey1; i++) {
-    tmq_conf_set(conf, g_stConfInfo.key[i], g_stConfInfo.value[i]);
+    tmq_conf_set(conf, g_stConfInfo.key1[i], g_stConfInfo.value1[i]);
   }
   tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0);
   return tmq;
 }
 
-tmq_list_t* build_topic_list() {
+tmq_list_t* build_topic_list_x() {
   tmq_list_t* topic_list = tmq_list_new();
   //tmq_list_append(topic_list, "test_stb_topic_1");
-  for (int32_t i = 0; i < g_stConfInfo.numOfTopic; i++) {
+  for (int32_t i = 0; i < g_stConfInfo.numOfTopic1; i++) {
-    tmq_list_append(topic_list, g_stConfInfo.topics[i]);
+    tmq_list_append(topic_list, g_stConfInfo.topics1[i]);
   }
   return topic_list;
 }
@@ -239,21 +322,21 @@ void loop_consume(tmq_t* tmq) {
   int32_t totalRows = 0;
   int32_t skipLogNum = 0;
   while (running) {
-    tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 8000);
+    TAOS_RES* tmqMsg = tmq_consumer_poll(tmq, 8000);
     if (tmqMsg) {
-	  totalMsgs++;
+      totalMsgs++;
 
-	  #if 0
+#if 0
 	  TAOS_ROW row;
 	  while (NULL != (row = tmq_get_row(tmqMsg))) {
         totalRows++;
 	  }
-	  #endif
+#endif
-	  
+
-      skipLogNum += tmqGetSkipLogNum(tmqMsg);
+      /*skipLogNum += tmqGetSkipLogNum(tmqMsg);*/
-	  if (0 != g_stConfInfo.showMsgFlag) {
+      if (0 != g_stConfInfo.showMsgFlag) {
-        msg_process(tmqMsg);
+        /*msg_process(tmqMsg);*/
-	  }
+      }
       tmq_message_destroy(tmqMsg);
     } else {
       break;
@@ -263,40 +346,41 @@ void loop_consume(tmq_t* tmq) {
   err = tmq_consumer_close(tmq);
   if (err) {
     printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err));
-	exit(-1);
+    exit(-1);
   }
-  
+
   printf("{consume success: %d, %d}", totalMsgs, totalRows);
 }
 
-
+int32_t parallel_consume(tmq_t* tmq, int threadLable) {
-void parallel_consume(tmq_t* tmq) {
   tmq_resp_err_t err;
 
   int32_t totalMsgs = 0;
   int32_t totalRows = 0;
   int32_t skipLogNum = 0;
   while (running) {
-    tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, g_stConfInfo.consumeDelay * 1000);
+    TAOS_RES* tmqMsg = tmq_consumer_poll(tmq, g_stConfInfo.consumeDelay * 1000);
     if (tmqMsg) {
-	  totalMsgs++;
+      totalMsgs++;
+
+	  //printf("threadFlag: %d, totalMsgs: %d\n", threadLable, totalMsgs);
 
 	  #if 0
 	  TAOS_ROW row;
 	  while (NULL != (row = tmq_get_row(tmqMsg))) {
         totalRows++;
 	  }
-	  #endif
+#endif
-	  
+
-      skipLogNum += tmqGetSkipLogNum(tmqMsg);
+      /*skipLogNum += tmqGetSkipLogNum(tmqMsg);*/
-	  if (0 != g_stConfInfo.showMsgFlag) {
+      if (0 != g_stConfInfo.showMsgFlag) {
-        msg_process(tmqMsg);
+        /*msg_process(tmqMsg);*/
-	  }
+      }
       tmq_message_destroy(tmqMsg);
 
-	  if (totalMsgs >= g_stConfInfo.consumeMsgCnt) {
+      if (totalMsgs >= g_stConfInfo.consumeMsgCnt) {
         break;
-	  }
+      }
     } else {
       break;
     }
@@ -305,20 +389,23 @@ void parallel_consume(tmq_t* tmq) {
   err = tmq_consumer_close(tmq);
   if (err) {
     printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err));
-	exit(-1);
+    exit(-1);
   }
-  
+
-  printf("%d", totalMsgs); // output to sim for check result
+  //printf("%d", totalMsgs); // output to sim for check result
+  return totalMsgs;
 }
 
-int main(int32_t argc, char *argv[]) {
+
-  parseArgument(argc, argv);
+void *threadFunc(void *param) {
-  parseInputString();
+  int32_t totalMsgs = 0;
-    
+
-  tmq_t*      tmq        = build_consumer();
+  SThreadInfo *pInfo = (SThreadInfo *)param;
-  tmq_list_t* topic_list = build_topic_list();
+
+  tmq_t*      tmq        = build_consumer_x();
+  tmq_list_t* topic_list = build_topic_list_x();
   if ((NULL == tmq) || (NULL == topic_list)){
-    return -1;
+    return NULL;
   }
   
   tmq_resp_err_t err = tmq_subscribe(tmq, topic_list);
@@ -327,10 +414,59 @@ int main(int32_t argc, char *argv[]) {
     exit(-1);
   }
 
-  if (0 == g_stConfInfo.consumeMsgCnt) {
+  //if (0 == g_stConfInfo.consumeMsgCnt) {
+  //  loop_consume(tmq);
+  //} else {
+    pInfo->consumeMsgCnt = parallel_consume(tmq, 1);
+  //}
+
+  err = tmq_unsubscribe(tmq);
+  if (err) {
+    printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
+	pInfo->consumeMsgCnt = -1;
+    return NULL;
+  }
+
+  return NULL;
+}
+
+
+int main(int32_t argc, char* argv[]) {
+  parseArgument(argc, argv);
+  parseInputString();
+
+  int32_t numOfThreads = 1;
+  TdThreadAttr thattr;
+  taosThreadAttrInit(&thattr);
+  taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
+  SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo));
+
+  if (g_stConfInfo.numOfTopic1) {
+    // pthread_create one thread to consume
+    for (int32_t i = 0; i < numOfThreads; ++i) {
+      pInfo[i].expectMsgCnt = 0;
+      pInfo[i].consumeMsgCnt = 0;
+      taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i));
+    }
+  }
+
+  int32_t totalMsgs = 0;
+  tmq_t*      tmq = build_consumer();
+  tmq_list_t* topic_list = build_topic_list();
+  if ((NULL == tmq) || (NULL == topic_list)) {
+    return -1;
+  }
+
+  tmq_resp_err_t err = tmq_subscribe(tmq, topic_list);
+  if (err) {
+    printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
+    exit(-1);
+  }
+
+  if (0 == g_stConfInfo.numOfTopic1) {
     loop_consume(tmq);
   } else {
-    parallel_consume(tmq);
+    totalMsgs = parallel_consume(tmq, 0);
   }
 
   err = tmq_unsubscribe(tmq);
@@ -339,6 +475,27 @@ int main(int32_t argc, char *argv[]) {
     exit(-1);
   }
 
+  if (g_stConfInfo.numOfTopic1) {  
+    for (int32_t i = 0; i < numOfThreads; i++) {
+      taosThreadJoin(pInfo[i].thread, NULL);
+    }
+
+	//printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt);
+	if (0 == g_stConfInfo.checkMode) {
+      if ((totalMsgs +  pInfo->consumeMsgCnt) == g_stConfInfo.consumeMsgCnt) {
+    	 printf("success");
+      } else {
+    	 printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt);
+      }	
+	} else if (1 == g_stConfInfo.checkMode) {
+      if ((totalMsgs == g_stConfInfo.consumeMsgCnt) && (pInfo->consumeMsgCnt == g_stConfInfo.consumeMsgCnt)) {
+    	 printf("success");
+      } else {
+    	 printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt);
+      }	
+	}
+  }
+
   return 0;
 }
 

tests/tsim/src/simSystem.c

@@ -80,24 +80,24 @@ SScript *simProcessCallOver(SScript *script) {
       simExecSuccess = false;
       simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s",
               script->fileName, script->error);
-      return NULL;
     } else {
       simExecSuccess = true;
       simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX,
               script->fileName);
-      simCloseTaosdConnect(script);
-      simScriptSucced++;
-      simScriptPos--;
-
-      simFreeScript(script);
-      if (simScriptPos == -1) {
-        simInfo("----------------------------------------------------------------------");
-        simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced);
-        return NULL;
-      }
-
-      return simScriptList[simScriptPos];
     }
+
+    simCloseTaosdConnect(script);
+    simScriptSucced++;
+    simScriptPos--;
+    simFreeScript(script);
+
+    if (simScriptPos == -1 && simExecSuccess) {
+      simInfo("----------------------------------------------------------------------");
+      simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced);
+      return NULL;
+    }
+
+    return simScriptList[simScriptPos];
   } else {
     simDebug("script:%s,  is stopped", script->fileName);
     simFreeScript(script);