diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 849d31f93e..5c6ffd4b10 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -76,15 +76,19 @@ IF ("${CPUTYPE}" STREQUAL "") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") MESSAGE(STATUS "The current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") + SET(TD_INTEL_64 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "The current platform is x86") SET(PLATFORM_ARCH_STR "i386") + SET(TD_INTEL_32 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "The current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") + SET(TD_ARM_32 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") MESSAGE(STATUS "The current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") + SET(TD_ARM_64 TRUE) ENDIF () ELSE () # if generate ARM version: @@ -92,18 +96,23 @@ ELSE () IF (${CPUTYPE} MATCHES "aarch32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") + SET(TD_ARM_32 TRUE) ELSEIF (${CPUTYPE} MATCHES "aarch64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") + SET(TD_ARM_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") MESSAGE(STATUS "input cpuType: mips64") + SET(TD_MIPS_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "x64") SET(PLATFORM_ARCH_STR "amd64") MESSAGE(STATUS "input cpuType: x64") + SET(TD_INTEL_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "x86") SET(PLATFORM_ARCH_STR "i386") MESSAGE(STATUS "input cpuType: x86") + SET(TD_INTEL_32 TRUE) ELSE () MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () diff --git a/docs/zh/12-taos-sql/12-interval.md b/docs/zh/12-taos-sql/12-interval.md index b0619ea5ce..8bf0c578e7 100644 --- a/docs/zh/12-taos-sql/12-interval.md +++ b/docs/zh/12-taos-sql/12-interval.md @@ -1,13 +1,85 @@ --- -sidebar_label: 按窗口切分聚合 -title: 按窗口切分聚合 +sidebar_label: 时序数据特色查询 +title: 时序数据特色查询 --- +TDengine 是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准 SQL 的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。 -TDengine 支持按时间段窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。 -窗口子句用于针对查询的数据集合进行按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）三种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 +TDengine 提供的特色查询包括标签切分查询和窗口切分查询。 -## 时间窗口 +## 标签切分查询 + +超级表查询中，当需要针对标签进行数据切分然后在切分出的数据空间内再进行一系列的计算时使用标签切分子句，标签切分的语句如下： + +```sql +PARTITION BY tag_list +``` + +其中 `tag_list` 是标签列的列表，还可以包括 tbname 伪列。 + +TDengine 按如下方式处理标签切分子句： + +标签切分子句位于 `WHERE` 子句之后，且不能和 `JOIN` 子句一起使用。 +标签切分子句将超级表数据按指定的标签组合进行切分，然后对每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、`GROUP BY` 子句或`SELECT` 子句）。 +标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按 10 分钟进行降采样，取其最大值。 + +```sql +select max(current) from meters partition by location interval(10m) +``` + +## 窗口切分查询 + +TDengine 支持按时间段窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）三种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。窗口切分查询语法如下： + +```sql +SELECT function_list FROM tb_name + [WHERE where_condition] + [SESSION(ts_col, tol_val)] + [STATE_WINDOW(col)] + [INTERVAL(interval [, offset]) [SLIDING sliding]] + [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] +``` + +在上述语法中的具体限制如下 + +### 窗口切分查询中使用函数的限制 + +- 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 +- 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 +- 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 + +### 窗口子句的规则 + +- 窗口子句位于标签切分子句之后，GROUP BY 子句之前，且不可以和 GROUP BY 子句一起使用。 +- 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含： + - 常量。 + - 聚集函数。 + - 包含上面表达式的表达式。 +- 窗口子句不可以和 GROUP BY 子句一起使用。 +- WHERE 语句可以指定查询的起止时间和其他过滤条件。 + +### FILL 子句 + +FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： + +1. 不进行填充：NONE（默认填充模式）。 +2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1。 +3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 +4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 +5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 +6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 + +:::info + +1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 +2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 +3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 GROUP BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 GROUP BY 语句分组，则返回结果中每个 GROUP 内不按照时间序列严格单调递增。 + +::: + +### 时间窗口 + +时间窗口又可分为滑动时间窗口和翻转时间窗口。 INTERVAL 子句用于产生相等时间周期的窗口，SLIDING 用以指定窗口向前滑动的时间。每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口（time window ）大小和每次前向增量时间（forward sliding times）。如图，[t0s, t0e] ，[t1s , t1e]， [t2s, t2e] 是分别是执行三次连续查询的时间窗口范围，窗口的前向滑动的时间范围 sliding time 标识 。查询过滤、聚合等操作按照每个时间窗口为独立的单位执行。当 SLIDING 与 INTERVAL 相等的时候，滑动窗口即为翻转窗口。 @@ -25,11 +97,13 @@ SLIDING 的向前滑动的时间不能超过一个窗口的时间范围。以下 SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m); ``` -当 SLIDING 与 INTERVAL 取值相等的时候，滑动窗口即为翻转窗口。 -_ 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 -_ 从 2.1.5.0 版本开始，INTERVAL 语句允许的最短时间间隔调整为 1 微秒（1u），当然如果所查询的 DATABASE 的时间精度设置为毫秒级，那么允许的最短时间间隔为 1 毫秒（1a）。 \* **注意**：用到 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 +使用时间窗口需要注意： -## 状态窗口 +- 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 +- 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 +- 返回的结果中时间序列严格单调递增。 + +### 状态窗口 使用整数（布尔值）或字符串来标识产生记录时候设备的状态量。产生的记录如果具有相同的状态量数值则归属于同一个状态窗口，数值改变后该窗口关闭。如下图所示，根据状态量确定的状态窗口分别是[2019-04-28 14:22:07，2019-04-28 14:22:10]和[2019-04-28 14:22:11，2019-04-28 14:22:12]两个。（状态窗口暂不支持对超级表使用） @@ -41,7 +115,7 @@ _ 从 2.1.5.0 版本开始，INTERVAL 语句允许的最短时间间隔调整为 SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status); ``` -## 会话窗口 +### 会话窗口 会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30]和[2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。 @@ -54,48 +128,7 @@ SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status); SELECT COUNT(*), FIRST(ts) FROM temp_tb_1 SESSION(ts, tol_val); ``` -这种类型的查询语法如下： - -``` -SELECT function_list FROM tb_name - [WHERE where_condition] - [SESSION(ts_col, tol_val)] - [STATE_WINDOW(col)] - [INTERVAL(interval [, offset]) [SLIDING sliding]] - [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] - -SELECT function_list FROM stb_name - [WHERE where_condition] - [INTERVAL(interval [, offset]) [SLIDING sliding]] - [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] - [GROUP BY tags] -``` - -- 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 -- 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 -- 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 -- - -- WHERE 语句可以指定查询的起止时间和其他过滤条件。 -- FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： - 1. 不进行填充：NONE（默认填充模式）。 - 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。 - 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 - 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 - 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 - 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 - -:::info - -1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 -2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 -3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 GROUP BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 GROUP BY 语句分组，则返回结果中每个 GROUP 内不按照时间序列严格单调递增。 - -::: - -时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](/develop/continuous-query)。 - -## 示例 +### 示例 智能电表的建表语句如下： diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/zh/12-taos-sql/13-operators.md b/docs/zh/12-taos-sql/16-operators.md similarity index 100% rename from docs/zh/12-taos-sql/13-operators.md rename to docs/zh/12-taos-sql/16-operators.md diff --git a/docs/zh/12-taos-sql/16-json.md b/docs/zh/12-taos-sql/17-json.md similarity index 100% rename from docs/zh/12-taos-sql/16-json.md rename to docs/zh/12-taos-sql/17-json.md diff --git a/docs/zh/12-taos-sql/14-limit.md b/docs/zh/12-taos-sql/19-limit.md similarity index 100% rename from docs/zh/12-taos-sql/14-limit.md rename to docs/zh/12-taos-sql/19-limit.md diff --git a/docs/zh/12-taos-sql/22-information.md b/docs/zh/12-taos-sql/22-information.md new file mode 100644 index 0000000000..0695aa5172 --- /dev/null +++ b/docs/zh/12-taos-sql/22-information.md @@ -0,0 +1,5 @@ +--- +sidebar_label: Information内置数据库 +title: Information内置数据库 +--- + diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 94e0b86821..38c3eba647 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -15,10 +15,10 @@ #include #include +#include #include #include #include "taos.h" -#include static int running = 1; static void msg_process(TAOS_RES* msg) { @@ -28,8 +28,8 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - tmq_raw_data *raw = tmq_get_raw_meta(msg); - if(raw){ + tmq_raw_data* raw = tmq_get_raw_meta(msg); + if (raw) { TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); if (pConn == NULL) { return; @@ -55,7 +55,7 @@ static void msg_process(TAOS_RES* msg) { } tmq_free_raw_meta(raw); char* result = tmq_get_json_meta(msg); - if(result){ + if (result) { printf("meta result: %s\n", result); } tmq_free_json_meta(result); @@ -96,7 +96,9 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)"); + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -152,6 +154,7 @@ int32_t init_env() { } taos_free_result(pRes); +#if 0 pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); if (taos_errno(pRes) != 0) { printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); @@ -264,7 +267,9 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)"); + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -277,6 +282,7 @@ int32_t init_env() { return -1; } taos_free_result(pRes); +#endif return 0; } @@ -296,8 +302,15 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic topic2 as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -353,6 +366,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5aee218ddc..930b7be3ef 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -80,8 +80,6 @@ typedef struct { SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid bool needSortTableByGroupId; - void* pTagCond; - void* pTagIndexCond; uint64_t suid; } STableListInfo; @@ -268,26 +266,6 @@ typedef struct SSortExecInfo { int32_t readBytes; // read io bytes } SSortExecInfo; -//====================================================================================================================== -// for grant -typedef enum { - TSDB_GRANT_ALL, - TSDB_GRANT_TIME, - TSDB_GRANT_USER, - TSDB_GRANT_DB, - TSDB_GRANT_TIMESERIES, - TSDB_GRANT_DNODE, - TSDB_GRANT_ACCT, - TSDB_GRANT_STORAGE, - TSDB_GRANT_SPEED, - TSDB_GRANT_QUERY_TIME, - TSDB_GRANT_CONNS, - TSDB_GRANT_STREAMS, - TSDB_GRANT_CPU_CORES, -} EGrantType; - -int32_t grantCheck(EGrantType grant); - #ifdef __cplusplus } #endif diff --git a/include/common/tgrant.h b/include/common/tgrant.h new file mode 100644 index 0000000000..ad23c661b1 --- /dev/null +++ b/include/common/tgrant.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_GRANT_H_ +#define _TD_COMMON_GRANT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +typedef enum { + TSDB_GRANT_ALL, + TSDB_GRANT_TIME, + TSDB_GRANT_USER, + TSDB_GRANT_DB, + TSDB_GRANT_TIMESERIES, + TSDB_GRANT_DNODE, + TSDB_GRANT_ACCT, + TSDB_GRANT_STORAGE, + TSDB_GRANT_SPEED, + TSDB_GRANT_QUERY_TIME, + TSDB_GRANT_CONNS, + TSDB_GRANT_STREAMS, + TSDB_GRANT_CPU_CORES, +} EGrantType; + +int32_t grantCheck(EGrantType grant); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_GRANT_H_*/ \ No newline at end of file diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 836712d8db..2f4c80f025 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2071,9 +2071,10 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); // TDMT_VND_ALTER_TABLE ===================== typedef struct { - char* tbName; - int8_t action; - char* colName; + char* tbName; + int8_t action; + char* colName; + int32_t colId; // TSDB_ALTER_TABLE_ADD_COLUMN int8_t type; int8_t flags; @@ -2924,7 +2925,6 @@ typedef struct { SMqRspHead head; STqOffsetVal reqOffset; STqOffsetVal rspOffset; - int32_t skipLogNum; int32_t blockNum; int8_t withTbName; int8_t withSchema; diff --git a/include/common/ttime.h b/include/common/ttime.h index de55b016cd..2f4129f979 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -63,12 +63,13 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; time_t t = taosTime(NULL); - struct tm* tm = taosLocalTime(&t, NULL); - tm->tm_hour = 0; - tm->tm_min = 0; - tm->tm_sec = 0; + struct tm tm; + taosLocalTime(&t, &tm); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; - return (int64_t)taosMktime(tm) * factor; + return (int64_t)taosMktime(&tm) * factor; } int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3bd02b1b48..1c3e5903f6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -26,6 +26,12 @@ extern "C" { #define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN +typedef enum EDataOrderLevel { + DATA_ORDER_LEVEL_NONE = 1, + DATA_ORDER_LEVEL_IN_BLOCK, + DATA_ORDER_LEVEL_IN_GROUP +} EDataOrderLevel; + typedef struct SLogicNode { ENodeType type; SNodeList* pTargets; // SColumnNode @@ -36,6 +42,8 @@ typedef struct SLogicNode { uint8_t precision; SNode* pLimit; SNode* pSlimit; + EDataOrderLevel requireDataOrder; // requirements for input data + EDataOrderLevel resultDataOrder; // properties of the output data } SLogicNode; typedef enum EScanType { @@ -78,7 +86,7 @@ typedef struct SScanLogicNode { SNodeList* pGroupTags; bool groupSort; int8_t cacheLastMode; - bool hasNormalCols; // neither tag column nor primary key tag column + bool hasNormalCols; // neither tag column nor primary key tag column } SScanLogicNode; typedef struct SJoinLogicNode { @@ -317,6 +325,7 @@ typedef STableScanPhysiNode SStreamScanPhysiNode; typedef struct SProjectPhysiNode { SPhysiNode node; SNodeList* pProjections; + bool mergeDataBlock; } SProjectPhysiNode; typedef struct SIndefRowsFuncPhysiNode { diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index ab53c37279..770b0442ea 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -116,6 +116,13 @@ int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5c4d8ce250..8c69c0f2de 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -83,6 +83,7 @@ typedef struct { int32_t srcVgId; int32_t childId; int64_t sourceVer; + int64_t reqId; SArray* blocks; // SArray } SStreamDataBlock; @@ -324,6 +325,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); if (pSubmitClone == NULL) { + qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); + terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } @@ -412,6 +415,7 @@ typedef struct { typedef struct { int64_t streamId; + int64_t reqId; int32_t srcTaskId; int32_t srcNodeId; int32_t dstTaskId; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e8eef0d287..73374c7f77 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1609,11 +1609,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; + int32_t totalLen = 0; int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); memcpy(p1, p, len); p += len; p1 += len; + totalLen += len; len = sizeof(int32_t) * numOfCols; int32_t* colLength = (int32_t*)p; @@ -1621,6 +1623,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int memcpy(p1, p, len); p += len; p1 += len; + totalLen += len; char* pStart = p; char* pStart1 = p1; @@ -1636,6 +1639,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int memcpy(pStart1, pStart, len); pStart += len; pStart1 += len; + totalLen += len; len = 0; for (int32_t j = 0; j < numOfRows; ++j) { @@ -1680,24 +1684,30 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int len += varDataTLen(dst); } colLen1 = len; + totalLen += colLen1; colLength1[i] = htonl(len); } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { len = numOfRows * sizeof(int32_t); memcpy(pStart1, pStart, len); pStart += len; pStart1 += len; + totalLen += len; + totalLen += colLen; memcpy(pStart1, pStart, colLen); } else { len = BitmapLen(pResultInfo->numOfRows); memcpy(pStart1, pStart, len); pStart += len; pStart1 += len; + totalLen += len; + totalLen += colLen; memcpy(pStart1, pStart, colLen); } pStart += colLen; pStart1 += colLen1; } + *(int32_t*)(pResultInfo->convertJson) = totalLen; pResultInfo->pData = pResultInfo->convertJson; return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5f26db67fc..83f2cf9307 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -609,7 +609,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { } kvVal->type = TSDB_DATA_TYPE_BIGINT; kvVal->i = (int64_t)result; - } else if ((left == 3 && strncasecmp(endptr, "u64", left) == 0)) { + } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { if (result >= (double)UINT64_MAX || result < 0) { errno = 0; uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); @@ -1047,6 +1047,10 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab continue; } + if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if (!kv) return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ed0ec516b2..5b845fd455 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1052,6 +1052,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t code = -1; req.consumerId = tmq->consumerId; + tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; @@ -1146,14 +1147,6 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } -#if 0 -int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return 0; - SMqPollRsp* pRsp = &tmq_message->msg; - return pRsp->skipLogNum; -} -#endif - int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; @@ -1296,9 +1289,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { offsetNew = *pOffset; } - /*tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64 ", vgKey is %s", - * tmq->consumerId, epoch,*/ - /*pVgEp->vgId, offset, vgKey);*/ SMqClientVg clientVg = { .pollCnt = 0, .currentOffsetNew = offsetNew, diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 17025db730..59eb841ab3 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -499,6 +499,32 @@ TEST(testCase, smlGetTimestampLen_Test) { ASSERT_EQ(len, 3); } +TEST(testCase, smlParseNumber_Test) { + SSmlKv kv = {0}; + char buf[64] = {0}; + SSmlMsgBuf msg = {0}; + msg.buf = buf; + msg.len = 64; + kv.value = "3.2e-900"; + kv.length = 8; + bool res = smlParseNumber(&kv, &msg); + printf("res:%d,v:%f, %f\n", res,kv.d, HUGE_VAL); +} + +//#include +//TEST(testCase, number_Test) { +// char *str[] = { +//// "-000 0999", +// "- abc", +// }; +// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ +// errno = 0; +// char *end = NULL; +// long result = strtol(str[i], &end, 10); +// printf("errno:%d,len:%d,result:%ld\n", errno, end - str[i], result); +// } +// +//} /* TEST(testCase, smlProcess_influx_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d9ac84ec06..e1b05cc026 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1661,9 +1661,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } */ -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -1674,9 +1671,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { ms += 1000; } } - - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm = {0}; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a95d58d353..b79c412914 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5648,7 +5648,6 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->skipLogNum) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; if (pRsp->blockNum != 0) { if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1; @@ -5674,7 +5673,6 @@ int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->skipLogNum) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; if (pRsp->blockNum != 0) { pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *)); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d728bbe49e..944ee6a731 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -902,7 +902,7 @@ const char* fmtts(int64_t ts) { void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { char ts[40] = {0}; - struct tm* ptm; + struct tm ptm; int32_t fractionLen; char* format = NULL; @@ -939,10 +939,10 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) assert(false); } - ptm = taosLocalTime(", NULL); - int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); + taosLocalTime(", &ptm); + int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += snprintf(ts + length, fractionLen, format, mod); - length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); + length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); tstrncpy(buf, ts, bufLen); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 06c782e217..93df7f8ab2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -153,9 +153,15 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pQueryQ, pMsg); + if ((pMsg->msgType == TDMT_SCH_QUERY) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_GRANT_EXPIRED; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-query queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pQueryQ, pMsg); + } break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); @@ -166,9 +172,14 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - - dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pWriteQ, pMsg); + if ((pMsg->msgType == TDMT_VND_SUBMIT) && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_VND_NO_WRITE_AUTH; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pWriteQ, pMsg); + } break; case SYNC_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8ad1ac56e4..f72b69a7de 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,7 @@ #include "tcache.h" #include "tdatablock.h" #include "tglobal.h" +#include "tgrant.h" #include "tqueue.h" #include "ttime.h" #include "version.h" diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5509ee88bf..a60db8a8c2 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -92,7 +92,9 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqConsumerLostMsg *pLostMsg = pMsg->pCont; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); - ASSERT(pConsumer); + if (pConsumer == NULL) { + return 0; + } mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId, mndConsumerStatusName(pConsumer->status)); @@ -450,6 +452,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t code = -1; SArray *newSub = subscribe.topicNames; taosArraySortString(newSub, taosArrayCompareString); + taosArrayRemoveDuplicate(newSub, taosArrayCompareString, taosMemoryFree); int32_t newTopicNum = taosArrayGetSize(newSub); // check topic existance @@ -907,8 +910,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); // client id - char clientId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(varDataVal(clientId), pConsumer->clientId, TSDB_CGROUP_LEN); + char clientId[256 + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(clientId), pConsumer->clientId, 256); varDataSetLen(clientId, strlen(varDataVal(clientId))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)clientId, false); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e3b0f3c157..b4b1b383e7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -509,11 +509,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { SUserObj *pUser = NULL; SCreateDbReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DB); - // if (code != 0) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index c26424e049..abac0573da 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -199,6 +199,7 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { int32_t tlen = 0; int32_t sz; tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeString(buf, pConsumer->clientId); tlen += taosEncodeString(buf, pConsumer->cgroup); tlen += taosEncodeFixedI8(buf, pConsumer->updateType); tlen += taosEncodeFixedI32(buf, pConsumer->epoch); @@ -264,6 +265,7 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { int32_t sz; buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeStringTo(buf, pConsumer->clientId); buf = taosDecodeStringTo(buf, pConsumer->cgroup); buf = taosDecodeFixedI8(buf, &pConsumer->updateType); buf = taosDecodeFixedI32(buf, &pConsumer->epoch); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d4c043028d..f26c1a7d32 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -621,11 +621,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DNODE); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index d67e4e8783..05e197150e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -298,7 +298,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + pConsumerEp->consumerId); } imbCnt++; } @@ -312,7 +313,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + pConsumerEp->consumerId); } } } @@ -354,7 +356,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } @@ -371,8 +374,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pConsumerEp->consumerId > 0); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; + if (pRebVg->newConsumerId == pRebVg->oldConsumerId) { + mInfo("mq rebalance: skip vg %d for same consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); + continue; + } taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } else { // if all consumer is removed, put all vg into unassigned diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5c361ed28d..0452659d47 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -363,11 +363,10 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - // code = grantCheck(TSDB_GRANT_USER); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_USER)) != 0) { + code = terrno; + goto _OVER; + } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 57141bbbb5..97eff4804d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -27,6 +27,7 @@ #include "wal.h" #include "tcommon.h" +#include "tgrant.h" #include "tfs.h" #include "tmsg.h" #include "trow.h" diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index c62b7e95bf..757749a9b6 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -52,7 +52,6 @@ typedef struct { int64_t reqOffset; int64_t processedVer; int32_t epoch; - int32_t skipLogNum; // rpc info int64_t reqId; SRpcHandleInfo rpcInfo; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a7fef795ee..776a3c5b7f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -205,7 +205,7 @@ _query: } tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, &schema); + tDecodeSSchemaWrapperEx(&dc, &schema); pSchema = tCloneSSchemaWrapper(&schema); tDecoderClear(&dc); @@ -470,9 +470,9 @@ int64_t metaGetTbNum(SMeta *pMeta) { } // N.B. Called by statusReq per second -int64_t metaGetTimeSeriesNum(SMeta *pMeta) { +int64_t metaGetTimeSeriesNum(SMeta *pMeta) { // TODO - return 400; + return 400; } typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f36e6f874f..fb05aeecd9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -635,6 +635,8 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { pSubmit = streamDataSubmitNew(pReq); if (pSubmit == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("failed to create data submit for stream since out of memory"); failed = true; } @@ -644,12 +646,16 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { SStreamTask* pTask = *(SStreamTask**)pIter; if (!pTask->isDataScan) continue; + qDebug("data submit enqueue stream task: %d", pTask->taskId); + if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { + qError("stream task input failed, task id %d", pTask->taskId); continue; } if (streamLaunchByWrite(pTask, TD_VID(pTq->pVnode)) < 0) { + qError("stream task launch failed, task id %d", pTask->taskId); continue; } } else { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 0bdbe82b77..618b32239a 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -64,9 +64,16 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan(task, pOffset) < 0) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); - pRsp->rspOffset = *pOffset; - return 0; + if (pOffset->type == TMQ_OFFSET__LOG) { + pRsp->rspOffset = *pOffset; + return 0; + } else { + tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + if (qStreamPrepareScan(task, pOffset) < 0) { + pRsp->rspOffset = *pOffset; + return 0; + } + } } int32_t rowCnt = 0; @@ -217,7 +224,6 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR } if (pRsp->blockNum == 0) { - pRsp->skipLogNum++; return -1; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index e9e5f6cd8b..4b2160434f 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -242,6 +242,8 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) void* data = taosMemoryMalloc(msgLen); if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("failed to copy data for stream since out of memory"); return -1; } memcpy(data, msg, msgLen); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b49e12fc08..e5322f1bd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -630,6 +630,9 @@ _exit: tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp); tEncoderClear(&ec); + if (vMetaRsp.pSchemas) { + taosMemoryFree(vMetaRsp.pSchemas); + } return 0; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f9aba30a46..625541e00a 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -15,6 +15,7 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#include "vnode.h" #include "function.h" #include "nodes.h" #include "plannodes.h" @@ -96,9 +97,9 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo } void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); -void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); + +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); @@ -106,7 +107,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getTableList(void* metaHandle, void* vnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); SArray* createSortInfo(SNodeList* pNodeList); SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, @@ -128,4 +129,6 @@ int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); +int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2ffb1be260..a7e033e547 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -311,19 +311,10 @@ typedef struct STableScanInfo { int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; int32_t currentTable; - -#if 0 - struct { - uint64_t uid; - int64_t ts; - } lastStatus; -#endif - - int8_t scanMode; - int8_t noTable; + int8_t scanMode; + int8_t noTable; } STableScanInfo; typedef struct STagScanInfo { @@ -429,8 +420,9 @@ typedef struct SStreamScanInfo { // status for tmq // SSchemaWrapper schema; - STqOffset offset; - + STqOffset offset; + SNode* pTagCond; + SNode* pTagIndexCond; } SStreamScanInfo; typedef struct SSysTableScanInfo { @@ -874,8 +866,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); @@ -966,12 +958,12 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr); SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 17484ec84b..4c86a46b6c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -72,8 +72,12 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); - taosArrayDestroy(pGroupResInfo->pRows); - pGroupResInfo->pRows = NULL; + for(int32_t i = 0; i < taosArrayGetSize(pGroupResInfo->pRows); ++i) { + SResKeyPos* pRes = taosArrayGetP(pGroupResInfo->pRows, i); + taosMemoryFree(pRes); + } + + pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); pGroupResInfo->index = 0; } @@ -261,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { +int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { SMetaReader mr = {0}; metaReaderInit(&mr, metaHandle, 0); metaGetTableEntryByUid(&mr, info->uid); @@ -276,19 +280,22 @@ static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { if (TSDB_CODE_SUCCESS != code) { terrno = code; nodesDestroyNode(pTagCondTmp); - return false; + *pQualified = false; + + return code; } ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); SValueNode* pValue = (SValueNode*)pNew; ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL); - bool result = pValue->datum.b; + *pQualified = pValue->datum.b; + nodesDestroyNode(pNew); - return result; + return TSDB_CODE_SUCCESS; } -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); @@ -300,8 +307,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, pListInfo->suid = pScanNode->suid; - SNode* pTagCond = (SNode*)pListInfo->pTagCond; - SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond; if (pScanNode->tableType == TSDB_SUPER_TABLE) { if (pTagIndexCond) { SIndexMetaArg metaArg = { @@ -341,9 +346,14 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, int32_t i = 0; while (i < taosArrayGetSize(pListInfo->pTableList)) { STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); - bool isOk = isTableOk(info, pTagCond, metaHandle); - if (terrno) return terrno; - if (!isOk) { + + bool qualified = true; + code = isTableOk(info, pTagCond, metaHandle, &qualified); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (!qualified) { taosArrayRemove(pListInfo->pTableList, i); continue; } @@ -358,7 +368,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, // put into list as default group, remove it if grouping sorting is required later taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); - return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f2008ed97a..90952d5786 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -153,7 +153,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { return pTaskInfo; } -static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) { +static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); // let's discard the tables those are not created according to the queried super table. @@ -164,7 +164,7 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons int32_t code = metaGetTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); + qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr); continue; } @@ -172,6 +172,21 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { continue; } + + if (pScanInfo->pTagCond != NULL) { + bool qualified = false; + STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid, .lastKey = 0}; + code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); + continue; + } + + if (!qualified) { + continue; + } + } + /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/ // handle multiple partition @@ -194,7 +209,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo int32_t code = 0; SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); + SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 37d2521e5d..9d3d62cabf 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -332,6 +332,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); uid = pTableInfo->uid; ts = INT64_MIN; + } else { + return -1; } } /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fe90e58527..06bb096e59 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -377,6 +376,10 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } +void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { + colDataDestroy(pColData); +} + void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { @@ -3737,7 +3740,7 @@ void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroyEx(pInfo->groupResInfo.pRows, freeItem); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } @@ -4365,8 +4368,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, - const char* pUser) { + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* pUser) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4374,7 +4376,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; @@ -4394,7 +4396,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; @@ -4407,7 +4409,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } SOperatorInfo* pOperator = - createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; @@ -4424,15 +4426,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo }; if (pHandle) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; } } - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo, &twSup); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4441,7 +4442,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTableListInfo); + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = terrno; return NULL; @@ -4477,8 +4478,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; - int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, - queryId, taskId); + int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -4502,7 +4502,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser); + ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pUser); if (ops[i] == NULL) { return NULL; } @@ -4685,6 +4685,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { return pList; } +#if 0 STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) { int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo); @@ -4718,6 +4719,7 @@ _error: terrno = code; return NULL; } +#endif static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -4761,6 +4763,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } +#if 0 int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) { STableScanInfo* pTableScanInfo = NULL; if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) { @@ -4784,6 +4787,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa // TODO: set uid and ts to data reader return 0; } +#endif int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) { int32_t code = TDB_CODE_SUCCESS; @@ -4935,10 +4939,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead } (*pTaskInfo)->sql = sql; - (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; - (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, - &(*pTaskInfo)->tableqinfoList, pPlan->user); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b80b9af237..e60f6f8a5b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1497,9 +1497,8 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pStreamScan); } -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, - SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, - uint64_t taskId) { +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) { SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1512,6 +1511,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; + pInfo->pTagCond = pTagCond; + int32_t numOfCols = 0; pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); @@ -2550,26 +2551,19 @@ typedef struct STableMergeScanInfo { int32_t tableEndIndex; bool hasGroupId; uint64_t groupId; + SArray* dataReaders; // array of tsdbReaderT* + SReadHandle readHandle; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; - SArray* dataReaders; // array of tsdbReaderT* - SReadHandle readHandle; - - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - - SArray* pSortInfo; - SSortHandle* pSortHandle; - - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - - SArray* sortSourceParams; - uint64_t queryId; - uint64_t taskId; + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; SFileBlockLoadRecorder readRecorder; - int64_t numOfRows; - // int32_t prevGroupId; // previous table group id + int64_t numOfRows; SScanInfo scanInfo; int32_t scanTimes; SNode* pFilterNode; // filter info, which is push down by optimizer @@ -2584,26 +2578,25 @@ typedef struct STableMergeScanInfo { SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SqlFunctionCtx* pPseudoCtx; - // int32_t* rowEntryInfoOffset; SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. - + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + // if the upstream is an interval operator, the interval info is also kept here to get the time + // window to check if current data block needs to be loaded. + SInterval interval; SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTableListInfo); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) { + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { - qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); + qDebug("no table qualified for query, %s" PRIx64, idStr); return TSDB_CODE_SUCCESS; } @@ -3027,8 +3020,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { } SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, - uint64_t taskId) { + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3067,9 +3059,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); - pInfo->queryId = queryId; - pInfo->taskId = taskId; - pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fa29fca74..026222f83a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -63,7 +63,6 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { - assert(win->skey <= win->ekey); SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); @@ -1513,12 +1512,25 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } +static void freeItem(void* param) { + SGroupKeys *pKey = (SGroupKeys*) param; + taosMemoryFree(pKey->pData); +} + void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); + pInfo->pPrevValues = NULL; + pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + + cleanupGroupResInfo(&pInfo->groupResInfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); taosMemoryFreeClear(param); } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index aba1df216e..a1d3fa2968 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -194,8 +194,9 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { void static addTimezoneParam(SNodeList* pList) { char buf[6] = {0}; time_t t = taosTime(NULL); - struct tm* tmInfo = taosLocalTime(&t, NULL); - strftime(buf, sizeof(buf), "%z", tmInfo); + struct tm tmInfo; + taosLocalTime(&t, &tmInfo); + strftime(buf, sizeof(buf), "%z", &tmInfo); int32_t len = (int32_t)strlen(buf); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); @@ -2086,6 +2087,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = topFunction, + .sprocessFunc = topBotScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = topCombine, .pPartialFunc = "top", @@ -2100,6 +2102,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = bottomFunction, + .sprocessFunc = topBotScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = bottomCombine, .pPartialFunc = "bottom", @@ -2229,6 +2232,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastRowFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize }, { @@ -2249,6 +2253,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, .pPartialFunc = "_first_partial", .pMergeFunc = "_first_merge", @@ -2284,6 +2289,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, .pPartialFunc = "_last_partial", .pMergeFunc = "_last_merge", @@ -2331,6 +2337,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, .processFunc = histogramFunction, + .sprocessFunc = histogramScalarFunction, .finalizeFunc = histogramFinalize, .invertFunc = NULL, .combineFunc = histogramCombine, @@ -2463,6 +2470,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, .processFunc = sampleFunction, + .sprocessFunc = sampleScalarFunction, .finalizeFunc = sampleFinalize }, { @@ -2474,6 +2482,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, .processFunc = tailFunction, + .sprocessFunc = tailScalarFunction, .finalizeFunc = NULL }, { @@ -2485,6 +2494,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, .processFunc = uniqueFunction, + .sprocessFunc = uniqueScalarFunction, .finalizeFunc = NULL }, { @@ -2495,6 +2505,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getModeFuncEnv, .initFunc = modeFunctionSetup, .processFunc = modeFunction, + .sprocessFunc = modeScalarFunction, .finalizeFunc = modeFinalize, }, { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f9ebde0657..fda02cd9ca 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -548,37 +548,12 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanStableId = "StableId"; +static const char* jkScanLogicPlanScanType = "ScanType"; static const char* jkScanLogicPlanScanCount = "ScanCount"; static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount"; static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; -// typedef struct SScanLogicNode { -// uint64_t stableId; -// SVgroupsInfo* pVgroupList; -// EScanType scanType; -// uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count -// STimeWindow scanRange; -// SName tableName; -// bool showRewrite; -// double ratio; -// SNodeList* pDynamicScanFuncs; -// int32_t dataRequired; -// int64_t interval; -// int64_t offset; -// int64_t sliding; -// int8_t intervalUnit; -// int8_t slidingUnit; -// SNode* pTagCond; -// SNode* pTagIndexCond; -// int8_t triggerType; -// int64_t watermark; -// int8_t igExpired; -// SArray* pSmaIndexes; -// SNodeList* pGroupTags; -// bool groupSort; -// } SScanLogicNode; - static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -598,6 +573,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanStableId, pNode->stableId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanType, pNode->scanType); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanCount, pNode->scanSeq[0]); } @@ -634,6 +612,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanStableId, &pNode->stableId); } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkScanLogicPlanScanType, pNode->scanType, code); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanScanCount, &pNode->scanSeq[0]); } @@ -1677,6 +1658,7 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) { } static const char* jkProjectPhysiPlanProjections = "Projections"; +static const char* jkProjectPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj; @@ -1685,6 +1667,9 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkProjectPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -1696,6 +1681,9 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkProjectPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 68a60e0b35..a7c08d8f65 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -201,7 +201,8 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) { } static bool isUselessCol(SExprNode* pProj) { - if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId)) { + if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId) && + !fmIsPseudoColumnFunc(((SFunctionNode*)pProj)->funcId)) { return false; } return NULL == ((SExprNode*)pProj)->pAssociation; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 19126b28f1..bac073c631 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1686,7 +1686,7 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { static bool sysTableFromVnode(const char* pTable) { return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || - (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -5968,6 +5968,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS if (NULL == pReq->tagName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema); if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { @@ -6051,6 +6052,7 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; return TSDB_CODE_SUCCESS; } @@ -6071,6 +6073,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4903beddbe..9ced5c1cb6 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -20,6 +20,7 @@ typedef struct SLogicPlanContext { SPlanContext* pPlanCxt; SLogicNode* pCurrRoot; + bool hasScan; } SLogicPlanContext; typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**); @@ -161,6 +162,10 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols return SCAN_TYPE_STREAM; } + if (TSDB_SYSTEM_TABLE == tableType) { + return SCAN_TYPE_SYSTEM_TABLE; + } + if (NULL == pScanCols) { return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE @@ -169,17 +174,6 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols : SCAN_TYPE_TABLE); } - if (TSDB_SYSTEM_TABLE == tableType) { - return SCAN_TYPE_SYSTEM_TABLE; - } - - SNode* pCol = NULL; - FOREACH(pCol, pScanCols) { - if (COLUMN_TYPE_COLUMN == ((SColumnNode*)pCol)->colType) { - return SCAN_TYPE_TABLE; - } - } - return SCAN_TYPE_TABLE; } @@ -300,6 +294,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect nodesDestroyNode((SNode*)pScan); } + pCxt->hasScan = true; + return code; } @@ -1339,9 +1335,9 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } -static void setLogicSubplanType(SLogicSubplan* pSubplan) { +static void setLogicSubplanType(bool hasScan, SLogicSubplan* pSubplan) { if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pSubplan->pNode)) { - pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->subplanType = hasScan ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE; } else { SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pSubplan->pNode; pSubplan->subplanType = (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren) @@ -1351,7 +1347,7 @@ static void setLogicSubplanType(SLogicSubplan* pSubplan) { } int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { - SLogicPlanContext cxt = {.pPlanCxt = pCxt}; + SLogicPlanContext cxt = {.pPlanCxt = pCxt, .pCurrRoot = NULL, .hasScan = false}; SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { @@ -1364,7 +1360,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code) { setLogicNodeParent(pSubplan->pNode); - setLogicSubplanType(pSubplan); + setLogicSubplanType(cxt.hasScan, pSubplan); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 8250fb462f..3f619f506f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -35,7 +35,8 @@ typedef struct SPhysiPlanContext { int32_t errCode; int16_t nextDataBlockId; SArray* pLocationHelper; - SArray* pExecNodeList; // SArray + bool hasScan; + bool hasSysScan; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { @@ -255,7 +256,7 @@ static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlo static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, false); } static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { @@ -495,8 +496,6 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub return TSDB_CODE_OUT_OF_MEMORY; } vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); - SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode); } @@ -577,8 +576,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan->execNode.nodeId = MNODE_HANDLE; pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet; } - SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_DNODE_VARIABLES)) { pScan->mgmtEpSet = pScanLogicNode->pVgroupList->vgroups->epSet; } else { @@ -586,6 +583,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* } tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); + pCxt->hasSysScan = true; return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } @@ -601,6 +599,7 @@ static int32_t createTableMergeScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + pCxt->hasScan = true; switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: case SCAN_TYPE_BLOCK_INFO: @@ -1806,23 +1805,31 @@ static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) { } } +static void setExecNodeList(SPhysiPlanContext* pCxt, SArray* pExecNodeList) { + if (NULL == pExecNodeList) { + return; + } + if (pCxt->hasSysScan || !pCxt->hasScan) { + SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; + taosArrayPush(pExecNodeList, &node); + } +} + int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { SPhysiPlanContext cxt = {.pPlanCxt = pCxt, .errCode = TSDB_CODE_SUCCESS, .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), - .pExecNodeList = pExecNodeList}; + .hasScan = false, + .hasSysScan = false}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - if (QUERY_POLICY_VNODE == tsQueryPolicy) { - taosArrayClear(pExecNodeList); - } - int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); if (TSDB_CODE_SUCCESS == code) { setExplainInfo(pCxt, *pPlan); + setExecNodeList(&cxt, pExecNodeList); } destoryPhysiPlanContext(&cxt); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 7dcba0e1b4..b754c52bbd 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1062,8 +1062,9 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS); } - struct tm *tmInfo = taosLocalTime((const time_t *)&timeVal, NULL); - strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", tmInfo); + struct tm tmInfo; + taosLocalTime((const time_t *)&timeVal, &tmInfo); + strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); int32_t len = (int32_t)strlen(buf); //add timezone string @@ -2601,3 +2602,263 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } + +typedef enum { UNKNOWN_BIN = 0, USER_INPUT_BIN, LINEAR_BIN, LOG_BIN } EHistoBinType; + +static int8_t getHistogramBinType(char* binTypeStr) { + int8_t binType; + if (strcasecmp(binTypeStr, "user_input") == 0) { + binType = USER_INPUT_BIN; + } else if (strcasecmp(binTypeStr, "linear_bin") == 0) { + binType = LINEAR_BIN; + } else if (strcasecmp(binTypeStr, "log_bin") == 0) { + binType = LOG_BIN; + } else { + binType = UNKNOWN_BIN; + } + + return binType; +} + +typedef struct SHistoFuncBin { + double lower; + double upper; + int64_t count; + double percentage; +} SHistoFuncBin; + +static bool getHistogramBinDesc(SHistoFuncBin** bins, int32_t* binNum, char* binDescStr, int8_t binType, bool normalized) { + cJSON* binDesc = cJSON_Parse(binDescStr); + int32_t numOfBins; + double* intervals; + if (cJSON_IsObject(binDesc)) { /* linaer/log bins */ + int32_t numOfParams = cJSON_GetArraySize(binDesc); + int32_t startIndex; + if (numOfParams != 4) { + return false; + } + + cJSON* start = cJSON_GetObjectItem(binDesc, "start"); + cJSON* factor = cJSON_GetObjectItem(binDesc, "factor"); + cJSON* width = cJSON_GetObjectItem(binDesc, "width"); + cJSON* count = cJSON_GetObjectItem(binDesc, "count"); + cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); + + if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { + return false; + } + + if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 + return false; + } + + if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || + (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { + return false; + } + + int32_t counter = (int32_t)count->valueint; + if (infinity->valueint == false) { + startIndex = 0; + numOfBins = counter + 1; + } else { + startIndex = 1; + numOfBins = counter + 3; + } + + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { + // linear bin process + if (width->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble + i * width->valuedouble; + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { + // log bin process + if (start->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else { + taosMemoryFree(intervals); + return false; + } + + if (infinity->valueint == true) { + intervals[0] = -INFINITY; + intervals[numOfBins - 1] = INFINITY; + // in case of desc bin orders, -inf/inf should be swapped + ASSERT(numOfBins >= 4); + if (intervals[1] > intervals[numOfBins - 2]) { + TSWAP(intervals[0], intervals[numOfBins - 1]); + } + } + } else if (cJSON_IsArray(binDesc)) { /* user input bins */ + if (binType != USER_INPUT_BIN) { + return false; + } + numOfBins = cJSON_GetArraySize(binDesc); + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + cJSON* bin = binDesc->child; + if (bin == NULL) { + taosMemoryFree(intervals); + return false; + } + int i = 0; + while (bin) { + intervals[i] = bin->valuedouble; + if (!cJSON_IsNumber(bin)) { + taosMemoryFree(intervals); + return false; + } + if (i != 0 && intervals[i] <= intervals[i - 1]) { + taosMemoryFree(intervals); + return false; + } + bin = bin->next; + i++; + } + } else { + return false; + } + + *binNum = numOfBins - 1; + *bins = taosMemoryCalloc(numOfBins, sizeof(SHistoFuncBin)); + for (int32_t i = 0; i < *binNum; ++i) { + (*bins)[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; + (*bins)[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; + (*bins)[i].count = 0; + } + + taosMemoryFree(intervals); + return true; +} + +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + SHistoFuncBin *bins; + int32_t numOfBins = 0; + int32_t totalCount = 0; + + int8_t binType = getHistogramBinType(varDataVal(pInput[1].columnData->pData)); + char* binDesc = varDataVal(pInput[2].columnData->pData); + int64_t normalized = *(int64_t *)(pInput[3].columnData->pData); + + int32_t type = GET_PARAM_TYPE(pInput); + if (!getHistogramBinDesc(&bins, &numOfBins, binDesc, binType, (bool)normalized)) { + return TSDB_CODE_FAILED; + } + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + char* data = colDataGetData(pInputData, i); + double v; + GET_TYPED_DATA(v, double, type, data); + + for (int32_t k = 0; k < numOfBins; ++k) { + if (v > bins[k].lower && v <= bins[k].upper) { + bins[k].count++; + totalCount++; + break; + } + } + } + + if (normalized) { + for (int32_t k = 0; k < numOfBins; ++k) { + if (totalCount != 0) { + bins[k].percentage = bins[k].count / (double)totalCount; + } else { + bins[k].percentage = 0; + } + } + } + + for (int32_t k = 0; k < numOfBins; ++k) { + int32_t len; + char buf[512] = {0}; + if (!normalized) { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", + bins[k].lower, bins[k].upper, bins[k].count); + } else { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + bins[k].lower, bins[k].upper, bins[k].percentage); + } + varDataSetLen(buf, len); + colDataAppend(pOutputData, k, buf, false); + } + + taosMemoryFree(bins); + pOutput->numOfRows = numOfBins; + return TSDB_CODE_SUCCESS; +} + +int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, 0); + continue; + } + + char* data = colDataGetData(pInputData, i); + colDataAppend(pOutputData, i, data, false); + } + + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} + +int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 2a96db3bfc..99a06575a9 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -143,6 +143,9 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { + qDebug("task %d(child %d) recv retrieve req from task %d, reqId %ld", pTask->taskId, pTask->selfChildId, + pReq->srcTaskId, pReq->reqId); + pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->srcVgId = 0; // decode diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index dbf6350c93..6be15222db 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -57,7 +57,9 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock pDataBlock->info.type = pRetrieve->streamBlockType; + pData->reqId = pReq->reqId; pData->blocks = pArray; + return 0; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e2faf28abe..5dec33d0fb 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -65,6 +65,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->srcNodeId) < 0) return -1; @@ -77,6 +78,7 @@ int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* p int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->srcNodeId) < 0) return -1; @@ -121,6 +123,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t sz = taosArrayGetSize(pTask->childEpInfo); ASSERT(sz > 0); for (int32_t i = 0; i < sz; i++) { + req.reqId = tGenIdPI64(); SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); req.dstNodeId = pEpInfo->nodeId; req.dstTaskId = pEpInfo->taskId; @@ -154,6 +157,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) ASSERT(0); return -1; } + + qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %ld", pTask->taskId, pTask->selfChildId, + pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } return 0; FAIL: diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 9d22da8662..b59a812678 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -54,6 +54,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); + + qDebug("task %d(child %d) processed retrieve, reqId %ld", pTask->taskId, pTask->selfChildId, + pRetrieveBlock->reqId); } break; } @@ -140,76 +143,80 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { } static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { - int32_t cnt = 0; - void* data = NULL; while (1) { - SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); - if (qItem == NULL) { - qDebug("stream exec over, queue empty"); - break; - } - if (data == NULL) { - data = qItem; - if (qItem->type == STREAM_INPUT__DATA_BLOCK) { - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ - } - streamQueueProcessSuccess(pTask->inputQueue); - } else { - if (streamAppendQueueItem(data, qItem) < 0) { - streamQueueProcessFail(pTask->inputQueue); + int32_t cnt = 0; + void* data = NULL; + while (1) { + SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); + if (qItem == NULL) { + qDebug("stream exec over, queue empty"); break; - } else { - cnt++; - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } + if (data == NULL) { + data = qItem; streamQueueProcessSuccess(pTask->inputQueue); - taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); - taosFreeQitem(qItem); + if (qItem->type == STREAM_INPUT__DATA_BLOCK) { + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } else { + break; + } + } else { + if (streamAppendQueueItem(data, qItem) < 0) { + streamQueueProcessFail(pTask->inputQueue); + break; + } else { + cnt++; + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + streamQueueProcessSuccess(pTask->inputQueue); + taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); + taosFreeQitem(qItem); + } } } - } - if (pTask->taskStatus == TASK_STATUS__DROPPING) { - if (data) streamFreeQitem(data); - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return NULL; - } - - if (data == NULL) return pRes; - - if (pTask->execType == TASK_EXEC__NONE) { - ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, data); - return pRes; - } - - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); - streamTaskExecImpl(pTask, data, pRes); - qDebug("stream task %d exec end", pTask->taskId); - - if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); - if (qRes == NULL) { - streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); - return NULL; - } - qRes->type = STREAM_INPUT__DATA_BLOCK; - qRes->blocks = pRes; - if (streamTaskOutput(pTask, qRes) < 0) { - /*streamQueueProcessFail(pTask->inputQueue);*/ + if (pTask->taskStatus == TASK_STATUS__DROPPING) { + if (data) streamFreeQitem(data); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); return NULL; } - if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - qRes->childId = pTask->selfChildId; - qRes->sourceVer = pSubmit->ver; - } - /*streamQueueProcessSuccess(pTask->inputQueue);*/ - pRes = taosArrayInit(0, sizeof(SSDataBlock)); - } - streamFreeQitem(data); + if (data == NULL) break; + + if (pTask->execType == TASK_EXEC__NONE) { + ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); + streamTaskOutput(pTask, data); + return pRes; + } + + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); + streamTaskExecImpl(pTask, data, pRes); + qDebug("stream task %d exec end", pTask->taskId); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + streamQueueProcessFail(pTask->inputQueue); + taosArrayDestroy(pRes); + return NULL; + } + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + if (streamTaskOutput(pTask, qRes) < 0) { + /*streamQueueProcessFail(pTask->inputQueue);*/ + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return NULL; + } + if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pSubmit->ver; + } + /*streamQueueProcessSuccess(pTask->inputQueue);*/ + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + } + + streamFreeQitem(data); + } return pRes; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 908523f2a6..c47964803a 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -483,6 +483,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { pRead->pHead->head.version, ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } @@ -491,6 +492,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { wError("vgId:%d, unexpected wal log index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } pRead->curVersion++; diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 7e6e508817..0cb4228e42 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -357,14 +357,88 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { time_t taosTime(time_t *t) { return time(t); } -time_t taosMktime(struct tm *timep) { return mktime(timep); } +time_t taosMktime(struct tm *timep) { +#ifdef WINDOWS + struct tm tm1 = {0}; + LARGE_INTEGER t; + FILETIME f; + SYSTEMTIME s; + FILETIME ff; + SYSTEMTIME ss; + LARGE_INTEGER offset; + + time_t tt = 0; + localtime_s(&tm1, &tt); + ss.wYear = tm1.tm_year + 1900; + ss.wMonth = tm1.tm_mon + 1; + ss.wDay = tm1.tm_wday; + ss.wHour = tm1.tm_hour; + ss.wMinute = tm1.tm_min; + ss.wSecond = tm1.tm_sec; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + + s.wYear = timep->tm_year + 1900; + s.wMonth = timep->tm_mon + 1; + s.wDay = timep->tm_wday; + s.wHour = timep->tm_hour; + s.wMinute = timep->tm_min; + s.wSecond = timep->tm_sec; + s.wMilliseconds = 0; + SystemTimeToFileTime(&s, &f); + t.QuadPart = f.dwHighDateTime; + t.QuadPart <<= 32; + t.QuadPart |= f.dwLowDateTime; + + t.QuadPart -= offset.QuadPart; + return (time_t)(t.QuadPart / 10000000); +#else + return mktime(timep); +#endif + } struct tm *taosLocalTime(const time_t *timep, struct tm *result) { if (result == NULL) { return localtime(timep); } #ifdef WINDOWS - localtime_s(result, timep); + if (*timep < 0) { + SYSTEMTIME ss,s; + FILETIME ff,f; + LARGE_INTEGER offset; + struct tm tm1; + time_t tt = 0; + localtime_s(&tm1, &tt); + ss.wYear = tm1.tm_year + 1900; + ss.wMonth = tm1.tm_mon + 1; + ss.wDay = tm1.tm_mday; + ss.wHour = tm1.tm_hour; + ss.wMinute = tm1.tm_min; + ss.wSecond = tm1.tm_sec; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + offset.QuadPart += *timep * 10000000; + f.dwLowDateTime = offset.QuadPart & 0xffffffff; + f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; + FileTimeToSystemTime(&f, &s); + result->tm_sec = s.wSecond; + result->tm_min = s.wMinute; + result->tm_hour = s.wHour; + result->tm_mday = s.wDay; + result->tm_mon = s.wMonth-1; + result->tm_year = s.wYear-1900; + result->tm_wday = s.wDayOfWeek; + result->tm_yday = 0; + result->tm_isdst = 0; + } else { + localtime_s(result, timep); + } #else localtime_r(timep, result); #endif diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index fce3cc7043..b856590787 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -22,15 +22,16 @@ class TDTestCase: self.ts = 1434938400000 self.time_step = 1000 - def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): + def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ,cache_value ): + tdSql.execute("drop database if exists test ") tdLog.info(" prepare datas for auto check abs function ") - tdSql.execute(" create database test cachemodel 'last_row' ") + tdSql.execute(f" create database test cachemodel {cache_value} ") tdSql.execute(" use test ") - tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ + tdSql.execute(" create stable test.stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") for tbnum in range(tbnums): - tbname = "sub_tb_%d"%tbnum + tbname = "test.sub_tb_%d"%tbnum tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum)) ts = self.ts @@ -56,17 +57,19 @@ class TDTestCase: coltypes = tdSql.queryResult for coltype in coltypes: colname = coltype[0] - abs_sql = "select abs({}) from {} order by tbname ".format(colname, tbname) - origin_sql = "select {} from {} order by tbname".format(colname, tbname) + abs_sql = "select abs({}) from {} order by tbname ".format(colname, 'test.'+tbname) + origin_sql = "select {} from {} order by tbname".format(colname, 'test.'+tbname) if coltype[1] in support_types: self.check_result_auto(origin_sql , abs_sql) + def prepare_datas(self ,cache_value): + tdSql.execute("drop database if exists db ") + create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachemodel {cache_value}" + tdSql.execute(create_db_sql) - def prepare_datas(self): - tdSql.execute("create database if not exists db keep 3650 duration 1000 cachemodel 'last_row'") tdSql.execute("use db") tdSql.execute( - '''create table stb1 + '''create table db.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int) ''' @@ -74,38 +77,38 @@ class TDTestCase: tdSql.execute( ''' - create table t1 + create table db.t1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) ''' ) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table db.ct{i+1} using stb1 tags ( {i+1} )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into db.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into db.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + "insert into db.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") tdSql.execute( - "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - f'''insert into t1 values + f'''insert into db.t1 values ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) @@ -121,51 +124,53 @@ class TDTestCase: ''' ) - def prepare_tag_datas(self): + def prepare_tag_datas(self,cache_value): + + tdSql.execute("drop database if exists testdb ") # prepare datas - tdSql.execute( - "create database if not exists testdb keep 3650 duration 1000 cachemodel 'last_row'") + tdSql.execute(f"create database if not exists testdb keep 3650 duration 1000 cachemodel {cache_value}") + tdSql.execute(" use testdb ") - tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ + tdSql.execute(f" create stable testdb.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags( t1 int , t2 bigint , t3 smallint , t4 tinyint , t5 float , t6 double , t7 bool , t8 binary(36)\ , t9 nchar(36) , t10 int unsigned , t11 bigint unsigned ,t12 smallint unsigned , t13 tinyint unsigned ,t14 timestamp ) ") tdSql.execute( ''' - create table t1 + create table testdb.t1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) ''' ) for i in range(4): tdSql.execute( - f'create table ct{i+1} using stb1 tags ( {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" ,{111*i}, {1*i},{1*i},{1*i},now())') + f'create table testdb.ct{i+1} using stb1 tags ( {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" ,{111*i}, {1*i},{1*i},{1*i},now())') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i} )" + f"insert into testdb.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i} )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i})" + f"insert into testdb.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i})" ) tdSql.execute( - "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ,0,0,0,0)") + "insert into testdb.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ,0,0,0,0)") tdSql.execute( - "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a , 999 , 9999 , 9 , 9)") + "insert into testdb.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a , 999 , 9999 , 9 , 9)") tdSql.execute( - "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a , 999 , 99999 , 9 , 9)") + "insert into testdb.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a , 999 , 99999 , 9 , 9)") tdSql.execute( - "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ,999 , 99999 , 9 , 9)") + "insert into testdb.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ,999 , 99999 , 9 , 9)") tdSql.execute( - "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + "insert into testdb.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") tdSql.execute( - "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + "insert into testdb.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") tdSql.execute( - "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL ) ") + "insert into testdb.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL ) ") tdSql.execute( - f'''insert into t1 values + f'''insert into testdb.t1 values ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) @@ -216,36 +221,24 @@ class TDTestCase: tdSql.execute("use testdb") # bug need fix - # tdSql.query("select last_row(c1 ,NULL) from t1") + tdSql.error("select last_row(c1 ,NULL) from testdb.t1") error_sql_lists = [ - "select last_row from t1", - "select last_row(-+--+c1) from t1", - # "select +-last_row(c1) from t1", - # "select ++-last_row(c1) from t1", - # "select ++--last_row(c1) from t1", - # "select - -last_row(c1)*0 from t1", - # "select last_row(tbname+1) from t1 ", - "select last_row(123--123)==1 from t1", - "select last_row(c1) as 'd1' from t1", - "select last_row(c1 ,NULL) from t1", - "select last_row(,) from t1;", - "select last_row(abs(c1) ab from t1)", - "select last_row(c1) as int from t1", - "select last_row from stb1", - # "select last_row(-+--+c1) from stb1", - # "select +-last_row(c1) from stb1", - # "select ++-last_row(c1) from stb1", - # "select ++--last_row(c1) from stb1", - # "select - -last_row(c1)*0 from stb1", - # "select last_row(tbname+1) from stb1 ", - "select last_row(123--123)==1 from stb1", - "select last_row(c1) as 'd1' from stb1", - # "select last_row(c1 ,c2 ) from stb1", - "select last_row(c1 ,NULL) from stb1", - "select last_row(,) from stb1;", - "select last_row(abs(c1) ab from stb1)", - "select last_row(c1) as int from stb1" + "select last_row from testdb.t1", + "select last_row(-+--+c1) from testdb.t1", + "select last_row(123--123)==1 from testdb.t1", + "select last_row(c1) as 'd1' from testdb.t1", + "select last_row(c1 ,NULL) from testdb.t1", + "select last_row(,) from testdb.t1;", + "select last_row(abs(c1) ab from testdb.t1)", + "select last_row(c1) as int from testdb.t1", + "select last_row from testdb.stb1", + "select last_row(123--123)==1 from testdb.stb1", + "select last_row(c1) as 'd1' from testdb.stb1", + "select last_row(c1 ,NULL) from testdb.stb1", + "select last_row(,) from testdb.stb1;", + "select last_row(abs(c1) ab from testdb.stb1)", + "select last_row(c1) as int from testdb.stb1" ] for error_sql in error_sql_lists: tdSql.error(error_sql) @@ -261,51 +254,51 @@ class TDTestCase: colname = coltype[0] col_note = coltype[-1] if col_note != "TAG": - abs_sql = "select last_row({}) from {}".format(colname, tbname) + abs_sql = "select last_row({}) from {}".format(colname, "testdb."+tbname) tdSql.query(abs_sql) def basic_abs_function(self): # basic query - tdSql.query("select c1 from ct3") + tdSql.query("select c1 from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select c1 from t1") + tdSql.query("select c1 from testdb.t1") tdSql.checkRows(12) - tdSql.query("select c1 from stb1") + tdSql.query("select c1 from testdb.stb1") tdSql.checkRows(25) # used for empty table , ct3 is empty - tdSql.query("select last_row(c1) from ct3") + tdSql.query("select last_row(c1) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c2) from ct3") + tdSql.query("select last_row(c2) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c3) from ct3") + tdSql.query("select last_row(c3) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c4) from ct3") + tdSql.query("select last_row(c4) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c5) from ct3") + tdSql.query("select last_row(c5) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c6) from ct3") + tdSql.query("select last_row(c6) from testdb.ct3") # used for regular table # bug need fix - tdSql.query("select last_row(c1) from t1") + tdSql.query("select last_row(c1) from testdb.t1") tdSql.checkData(0, 0, None) - tdSql.query("select last_row(c1) from ct4") + tdSql.query("select last_row(c1) from testdb.ct4") tdSql.checkData(0, 0, None) - tdSql.query("select last_row(c1) from stb1") + tdSql.query("select last_row(c1) from testdb.stb1") tdSql.checkData(0, 0, None) # # bug need fix - tdSql.query("select last_row(c1), c2, c3 , c4, c5 from t1") + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.t1") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) tdSql.checkData(0, 2, None) # # bug need fix - tdSql.query("select last_row(c1), c2, c3 , c4, c5 from ct1") + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.ct1") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) tdSql.checkData(0, 2, -999) @@ -313,187 +306,189 @@ class TDTestCase: tdSql.checkData(0, 4,-9.99000) # bug need fix - # tdSql.query("select last_row(c1), c2, c3 , c4, c5 from stb1 where tbname='ct1'") - # tdSql.checkData(0, 0, 9) - # tdSql.checkData(0, 1, -99999) - # tdSql.checkData(0, 2, -999) - # tdSql.checkData(0, 3, None) - # tdSql.checkData(0, 4,-9.99000) + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.stb1 where tbname='ct1'") + tdSql.checkData(0, 0, 9) + tdSql.checkData(0, 1, -99999) + tdSql.checkData(0, 2, -999) + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4,-9.99000) # bug fix - tdSql.query("select last_row(abs(c1)) from ct1") + tdSql.query("select last_row(abs(c1)) from testdb.ct1") tdSql.checkData(0,0,9) # # bug fix - tdSql.query("select last_row(c1+1) from ct1") - tdSql.query("select last_row(c1+1) from stb1") - tdSql.query("select last_row(c1+1) from t1") + tdSql.query("select last_row(c1+1) from testdb.ct1") + tdSql.query("select last_row(c1+1) from testdb.stb1") + tdSql.query("select last_row(c1+1) from testdb.t1") # used for stable table - tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from ct1") + tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from testdb.ct1") tdSql.checkData(0,0,9) tdSql.checkData(0,1,-99999) tdSql.checkData(0,2,-999) tdSql.checkData(0,3,None) # bug need fix - tdSql.query("select last_row(c1 ,c2 ,c3) from stb1 ") + tdSql.query("select last_row(c1 ,c2 ,c3) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,None) - # tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from stb1 where ts 5 ") + "select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,last_row(log(c1,2)-0.5) from db.ct4 where c1>5 ") tdSql.checkData(0, 0, 6) tdSql.checkData(0, 1, 6.000000000) tdSql.checkData(0, 2, 6.000000000) @@ -501,19 +496,19 @@ class TDTestCase: tdSql.checkData(0, 4, 2.084962501) tdSql.query( - "select last_row(c1,c2,c1+5) from ct4 where c1=5 ") + "select last_row(c1,c2,c1+5) from db.ct4 where c1=5 ") tdSql.checkData(0, 0, 5) tdSql.checkData(0, 1, 55555) tdSql.checkData(0, 2, 10.000000000) tdSql.query( - "select last(c1,c2,c1+5) from ct4 where c1=5 ") + "select last(c1,c2,c1+5) from db.ct4 where c1=5 ") tdSql.checkData(0, 0, 5) tdSql.checkData(0, 1, 55555) tdSql.checkData(0, 2, 10.000000000) tdSql.query( - "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from db.ct4 where c1>log(c1,2) limit 1 ") tdSql.checkRows(1) tdSql.checkData(0, 0, 8) tdSql.checkData(0, 1, 88888) @@ -528,39 +523,40 @@ class TDTestCase: def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") - tdSql.execute("create database if not exists bound_test cachemodel 'last_value'") + tdSql.execute("create database if not exists bound_test cachemodel 'LAST_ROW' ") + time.sleep(3) tdSql.execute("use bound_test") tdSql.execute( - "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + "create table bound_test.stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" ) - tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute(f'create table bound_test.sub1_bound using stb_bound tags ( 1 )') tdSql.execute( - f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) # check basic elem for table per row tdSql.query( - "select last(c1) ,last_row(c2), last_row(c3)+1 , last(c4)+1 from sub1_bound ") + "select last(c1) ,last_row(c2), last_row(c3)+1 , last(c4)+1 from bound_test.sub1_bound ") tdSql.checkData(0, 0, -2147483646) tdSql.checkData(0, 1, -9223372036854775806) tdSql.checkData(0, 2, -32765.000000000) tdSql.checkData(0, 3, -125.000000000) # check + - * / in functions tdSql.query( - "select last_row(c1+1) ,last_row(c2) , last(c3*1) , last(c4/2) from sub1_bound ") + "select last_row(c1+1) ,last_row(c2) , last(c3*1) , last(c4/2) from bound_test.sub1_bound ") def test_tag_compute_for_scalar_function(self): @@ -568,200 +564,228 @@ class TDTestCase: # bug need fix - tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") - tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") + tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ") + tdSql.query("select c1 ,t1 from testdb.stb1 where t1 =0 ") tdSql.checkRows(13) - tdSql.query("select last_row(c1,t1) from stb1 ") + tdSql.query("select last_row(c1,t1) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) - # tdSql.query("select last_row(c1),t1 from stb1 ") - # tdSql.checkData(0,0,None) - # tdSql.checkData(0,1,3) - tdSql.query("select last_row(c1,t1),last(t1) from stb1 ") + tdSql.query("select last_row(c1),t1 from testdb.stb1 ") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,3) + tdSql.query("select last_row(c1,t1),last(t1) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) tdSql.checkData(0,2,3) - tdSql.query("select last_row(t1) from stb1 where t1 >0 ") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 >0 ") tdSql.checkRows(1) tdSql.checkData(0,0,3) - tdSql.query("select last_row(t1) from stb1 where t1 =3 ") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 =3 ") tdSql.checkRows(1) tdSql.checkData(0,0,3) - tdSql.query("select last_row(t1) from stb1 where t1 =2") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 =2") tdSql.checkRows(0) # nest query for last_row - # tdSql.query("select last_row(t1) from (select c1 ,t1 from stb1)") - # tdSql.checkData(0,0,61) - # tdSql.query("select distinct(c1) ,t1 from stb1") - # tdSql.checkRows(20) - tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from stb1)") + tdSql.query("select last_row(t1) from (select ts , c1 ,t1 from testdb.stb1)") + tdSql.checkData(0,0,3) + tdSql.query("select distinct(c1) ,t1 from testdb.stb1") + tdSql.checkRows(20) + tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from testdb.stb1)") tdSql.checkData(0,0,None) - tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from testdb.stb1)") tdSql.checkData(0,0,None) - tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") - tdSql.checkData(0,1,None,None) + tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from testdb.stb1)") + tdSql.checkData(0,1,None) - tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts ="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') - # tdSql.checkRows(40) - # tdSql.checkData(0,0,None) - tdSql.query('select max(c1) from stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') + tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)") + tdSql.checkRows(27) + + tdSql.query("select last_row(c1) from testdb.ct1 interval(50s) sliding(30s)") tdSql.checkRows(5) + last_row_result = tdSql.queryResult + tdSql.query("select last(c1) from testdb.ct1 interval(50s) sliding(30s)") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + # bug need fix + tdSql.query('select max(c1) from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(NULL)') + tdSql.checkRows(8) + tdSql.checkData(7,0,None) + tdSql.query('select last_row(c1) from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(value ,2 )') + tdSql.checkRows(8) + tdSql.checkData(7,0,2) + + tdSql.query('select last_row(c1) from testdb.stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') + tdSql.query('select last_row(c1) from (select ts , c1 from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)') + + # join + tdSql.query("use test") + tdSql.query("select last(sub_tb_1.c1), last(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + tdSql.checkCols(2) + last_row_result = tdSql.queryResult + tdSql.query("select last_row(sub_tb_1.c1), last_row(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + tdSql.query("select last(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + + last_row_result = tdSql.queryResult + tdSql.query("select last_row(*), last_row(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + tdSql.query("select last(*), last_row(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + tdSql.query("select last_row(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + def support_super_table_test(self): tdSql.execute(" use testdb ") - self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" ) - self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" ) - self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 order by ts " , "select abs(c1) from testdb.stb1 order by ts" ) + self.check_result_auto( " select c1 from testdb.stb1 order by tbname " , "select abs(c1) from testdb.stb1 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 where c1 > 0 order by tbname " , "select abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 where c1 > 0 order by tbname " , "select abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" ) - self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" ) - self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c1 from testdb.stb1 order by ts " , "select t1, abs(c1) from testdb.stb1 order by ts" ) + self.check_result_auto( " select t2,c1 from testdb.stb1 order by tbname " , "select t2 ,abs(c1) from testdb.stb1 order by tbname" ) + self.check_result_auto( " select t3,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t4,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) pass - - - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - # tdSql.prepare() - - tdLog.printNoPrefix("==========step1:create table ==============") - - self.prepare_datas() - self.prepare_tag_datas() - + + def basic_query(self): + tdLog.printNoPrefix("==========step2:test errors ==============") self.test_errors() @@ -788,12 +812,41 @@ class TDTestCase: tdLog.printNoPrefix("==========step7: check result of query ============") - self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) tdLog.printNoPrefix("==========step8: check abs result of stable query ============") self.support_super_table_test() + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + # tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + # cache_last 0 + self.prepare_datas("'NONE' ") + self.prepare_tag_datas("'NONE'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'NONE'") + self.basic_query() + + # cache_last 1 + self.prepare_datas("'LAST_ROW'") + self.prepare_tag_datas("'LAST_ROW'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_ROW'") + self.basic_query() + + # cache_last 2 + self.prepare_datas("'LAST_VALUE'") + self.prepare_tag_datas("'LAST_VALUE'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_VALUE'") + self.basic_query() + + # cache_last 3 + self.prepare_datas("'BOTH'") + self.prepare_tag_datas("'BOTH'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'") + self.basic_query() + + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py index 65609629bc..82596a32ba 100644 --- a/tests/system-test/7-tmq/stbTagFilter.py +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -5,103 +5,250 @@ import time import socket import os import threading +from enum import Enum from util.log import * from util.sql import * from util.cases import * from util.dnodes import * -from util.common import * sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 1 + self.rowsPerTbl = 10000 + def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) - #tdSql.init(conn.cursor(), logSql) # output sql.txt file + tdSql.init(conn.cursor(), False) - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'db2', + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', - 'vgroups': 1, + 'vgroups': 4, 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 10, + 'ctbNum': 1, + 'rowsPerTbl': 100000, + 'batchNum': 1200, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, + 'pollDelay': 3, 'showMsg': 1, - 'showRow': 1} + 'showRow': 1, + 'snapshot': 0} - topicNameList = ['topic1'] - expectRowsList = [] + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=1,replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) tdLog.info("create stb") - tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("insert data") - tmqCom.asyncInsertData(paraDict) - - tdLog.info("create topics from stb with filter") - # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) - queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 100000, + 'batchNum': 3000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # update to half tables + paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + # queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # start tmq consume processor - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - topicList = topicNameList[0] - ifcheckdata = 0 + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 0 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromStb1 + ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - # tmqCom.getStartCommitNotifyFromTmqsim() - tmqCom.getStartConsumeNotifyFromTmqsim() - tdLog.info("create some new ctb") - paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum'] - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data into new ctb") - pThread = tmqCom.asyncInsertData(paraDict) - - pThread.join() - tdLog.info("wait insert end") - tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) - - tdLog.info("wait the consume result") + tdLog.info("insert process end, and start to check consume result") expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - - if expectRowsList[0] != resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) - tdLog.exit("0 tmq consume rows error!") - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + tdLog.info("run select sql from db") + tdSql.query(queryString) + expectrowcnt = tdSql.getRows() + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("drop topic %s"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 5000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("restart taosd to ensure that the data falls into the disk") + tdSql.query("flush database %s"%(paraDict['dbName'])) + + # update to half tables + paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl / 2) + paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tmqCom.initConsumerTable() + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 1 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromStb1 + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt)) + + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + # tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + def run(self): tdSql.prepare() + self.prepareTestEnv() + tdLog.printNoPrefix("=============================================") + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") self.tmqCase1() + # self.tmqCase2() + + self.prepareTestEnv() + tdLog.printNoPrefix("====================================================================") + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.snapshot = 1 + self.tmqCase1() + # self.tmqCase2() + def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ddbbefe739..d98764edb6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -126,7 +126,7 @@ python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 2-query/max_partition.py - +python3 ./test.py -f 2-query/last_row.py python3 ./test.py -f 6-cluster/5dnode1mnode.py #BUG python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 @@ -274,6 +274,7 @@ python3 ./test.py -f 2-query/irate.py -Q 2 python3 ./test.py -f 2-query/function_null.py -Q 2 python3 ./test.py -f 2-query/count_partition.py -Q 2 python3 ./test.py -f 2-query/max_partition.py -Q 2 +python3 ./test.py -f 2-query/last_row.py -Q 2 #------------querPolicy 3----------- @@ -361,3 +362,4 @@ python3 ./test.py -f 2-query/irate.py -Q 3 python3 ./test.py -f 2-query/function_null.py -Q 3 python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 +python3 ./test.py -f 2-query/last_row.py -Q 3 diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 02fd3c1396..784b45c92b 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -388,13 +388,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog } /*taosSsleep(3);*/ int32_t batchCnt = 0; - int32_t skipLogNum = 0; int64_t startTime = taosGetTimestampUs(); while (running) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000); if (tmqmessage) { batchCnt++; - /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ if (0 != g_stConfInfo.showMsgFlag) { /*msg_process(tmqmessage);*/ } @@ -412,7 +410,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog } if (0 == g_stConfInfo.simCase) { - printf("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + printf("consume result: msgs: %d, time used:%.3f second\n", batchCnt, consumeTime); } else { printf("{consume success: %d}", totalMsgs); } @@ -598,7 +596,8 @@ void printParaIntoFile() { g_fp = pFile; time_t tTime = taosGetTimestampSec(); - struct tm tm = *taosLocalTime(&tTime, NULL); + struct tm tm; + taosLocalTime(&tTime, &tm); taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "# configDir: %s\n", configDir); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 5459e3f159..7030f6fea6 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -171,7 +171,8 @@ static void printHelp() { char* getCurrentTimeString(char* timeString) { time_t tTime = taosGetTimestampSec(); - struct tm tm = *taosLocalTime(&tTime, NULL); + struct tm tm; + taosLocalTime(&tTime, &tm); sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); @@ -420,18 +421,6 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { ms = val % 1000; } - /* - comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -443,8 +432,9 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { } } - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index aaad76bb53..b993a8dbf1 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -635,7 +635,7 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { char timeStr[30] = {0}; time_t tt; - struct tm *tp; + struct tm tp; SCmdLine *line = &script->lines[script->linePos]; int32_t ret = -1; @@ -768,20 +768,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else { tt = (*(int64_t *)row[i]) / 1000000000; } - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - - tp = taosLocalTime(&tt, NULL); - strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp); + taosLocalTime(&tt, &tp); + strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); if (precision == TSDB_TIME_PRECISION_MILLI) { sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); } else if (precision == TSDB_TIME_PRECISION_MICRO) { diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 5b6dda4dae..b9f7610be8 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -175,7 +175,7 @@ SScript *simBuildScriptObj(char *fileName) { SScript *simParseScript(char *fileName) { TdFilePtr pFile; int32_t tokenLen, lineNum = 0; - char *buffer = NULL, name[128], *token, *rest; + char buffer[10*1024], name[128], *token, *rest; SCommand *pCmd; SScript *script; @@ -195,7 +195,7 @@ SScript *simParseScript(char *fileName) { simResetParser(); while (!taosEOFFile(pFile)) { - if (taosGetLineFile(pFile, &buffer) == -1) continue; + if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue; lineNum++; int32_t cmdlen = (int32_t)strlen(buffer); @@ -240,7 +240,6 @@ SScript *simParseScript(char *fileName) { return NULL; } } - if(buffer != NULL) taosMemoryFree(buffer); taosCloseFile(&pFile); script = simBuildScriptObj(fileName); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d6b7f18fb9..0982dbc019 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -231,18 +231,6 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { ms = val % 1000; } - /* - comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -254,8 +242,9 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { } } - struct tm *ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm = {0}; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); @@ -748,6 +737,13 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver void shellReadHistory() { SShellHistory *pHistory = &shell.history; + int64_t file_size; + if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { + return; + } else if (file_size > SHELL_MAX_COMMAND_SIZE) { + taosRemoveFile(pHistory->file); + return; + } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return; diff --git a/tools/taos-tools b/tools/taos-tools index b7b922268c..0b8a3373bb 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit b7b922268c4a06d9db77ffdfde0726f3d9900b72 +Subproject commit 0b8a3373bb7548f8106d13e7d3b0a988d3c4d48a