From d93c4828603f36f460d652cccfa5cab37c5e3d93 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 6 Jul 2021 18:28:45 +0800 Subject: [PATCH 1/5] [TD-5078]: update test case for nested query --- tests/pytest/query/nestedQuery/queryWithOrderLimit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/nestedQuery/queryWithOrderLimit.py b/tests/pytest/query/nestedQuery/queryWithOrderLimit.py index ba66b6ecb6..26eda1120b 100644 --- a/tests/pytest/query/nestedQuery/queryWithOrderLimit.py +++ b/tests/pytest/query/nestedQuery/queryWithOrderLimit.py @@ -49,8 +49,8 @@ class TDTestCase: tdSql.checkRows(10) # bug: https://jira.taosdata.com:18080/browse/TD-5043 - # tdSql.query("select * from (select * from st order by ts desc limit 10 offset 1000)") - # tdSql.checkRows(0) + tdSql.query("select * from (select * from st order by ts desc limit 10 offset 1000)") + tdSql.checkRows(0) tdSql.query("select avg(value), sum(value) from st group by tbname") tdSql.checkRows(self.tables) From 8ab4b3131a3023e46c7fffb1a51764f175c2a6b8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Jul 2021 12:21:46 +0800 Subject: [PATCH 2/5] Feature/sangshuduo/td 5053 taosdump support nanosecond (#6769) * [TD-5053]: taosdump support nanosecond huge code refactoring * fix arm32 compile issue. * [TD-5053]: taosdump supports nanosecond. * fix precision parsing order issue. --- src/kit/taosdump/taosdump.c | 128 +++++++++++++++++++++++++----------- 1 file changed, 89 insertions(+), 39 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index c2b5e11e67..94a89bcba6 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -226,15 +226,16 @@ static struct argp_option options[] = { {"schemaonly", 's', 0, 0, "Only dump schema.", 2}, {"without-property", 'N', 0, 0, "Dump schema without properties.", 2}, {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, - {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, + {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, + {"precision", 'C', "PRECISION", 0, "Epoch precision. Valid value is one of ms, us, and ns. Default is ms.", 6}, {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Max value is 32766. Default is 1.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, - {"debug", 'g', 0, 0, "Print debug info.", 4}, - {"verbose", 'b', 0, 0, "Print verbose debug info.", 5}, - {"performanceprint", 'm', 0, 0, "Print performance debug info.", 5}, + {"debug", 'g', 0, 0, "Print debug info.", 8}, + {"verbose", 'b', 0, 0, "Print verbose debug info.", 9}, + {"performanceprint", 'm', 0, 0, "Print performance debug info.", 10}, {0} }; @@ -262,6 +263,7 @@ typedef struct arguments { bool avro; int64_t start_time; int64_t end_time; + char precision[8]; int32_t data_batch; int32_t max_sql_len; int32_t table_batch; // num of table which will be dump into one output file. @@ -329,8 +331,9 @@ struct arguments g_args = { false, // schemeonly true, // with_property false, // avro format - 0, // start_time + -INT64_MAX, // start_time INT64_MAX, // end_time + "ms", // precision 1, // data_batch TSDB_MAX_SQL_LEN, // max_sql_len 1, // table_batch @@ -508,41 +511,84 @@ static int queryDbImpl(TAOS *taos, char *command) { return 0; } -static void parse_args(int argc, char *argv[], SArguments *arguments) { +static void parse_precision_first( + int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { - if ((strcmp(argv[i], "-S") == 0) - || (strcmp(argv[i], "-E") == 0)) { - if (argv[i+1]) { - char *tmp = strdup(argv[++i]); - - if (tmp) { - int64_t tmpEpoch; - if (strchr(tmp, ':') && strchr(tmp, '-')) { - if (TSDB_CODE_SUCCESS != taosParseTime( - tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { - errorPrint("Input %s, end time error!\n", tmp); - free(tmp); - return; - } - } else { - tmpEpoch = atoll(tmp); - } - - sprintf(argv[i], "%"PRId64"", tmpEpoch); - debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", - __func__, __LINE__, tmp, i, argv[i]); - - free(tmp); - } else { - errorPrint("%s() LN%d, strdup() cannot allocate memory\n", __func__, __LINE__); - exit(-1); - } - } else { + if (strcmp(argv[i], "-C") == 0) { + if (NULL == argv[i+1]) { errorPrint("%s need a valid value following!\n", argv[i]); exit(-1); } - } else if (strcmp(argv[i], "-g") == 0) { - g_args.debug_print = true; + char *tmp = strdup(argv[i+1]); + if (tmp == NULL) { + errorPrint("%s() LN%d, strdup() cannot allocate memory\n", + __func__, __LINE__); + exit(-1); + } + if ((0 != strncasecmp(tmp, "ms", strlen("ms"))) + && (0 != strncasecmp(tmp, "us", strlen("us"))) + && (0 != strncasecmp(tmp, "ns", strlen("ns")))) { + // + errorPrint("input precision: %s is invalid value\n", tmp); + free(tmp); + exit(-1); + } + strncpy(g_args.precision, tmp, strlen(tmp)); + free(tmp); + } + } +} + +static void parse_timestamp( + int argc, char *argv[], SArguments *arguments) { + for (int i = 1; i < argc; i++) { + if ((strcmp(argv[i], "-S") == 0) + || (strcmp(argv[i], "-E") == 0)) { + if (NULL == argv[i+1]) { + errorPrint("%s need a valid value following!\n", argv[i]); + exit(-1); + } + char *tmp = strdup(argv[i+1]); + if (NULL == tmp) { + errorPrint("%s() LN%d, strdup() cannot allocate memory\n", + __func__, __LINE__); + exit(-1); + } + + int64_t tmpEpoch; + if (strchr(tmp, ':') && strchr(tmp, '-')) { + int32_t timePrec; + if (0 == strncasecmp(arguments->precision, + "ms", strlen("ms"))) { + timePrec = TSDB_TIME_PRECISION_MILLI; + } else if (0 == strncasecmp(arguments->precision, + "us", strlen("us"))) { + timePrec = TSDB_TIME_PRECISION_MICRO; + } else if (0 == strncasecmp(arguments->precision, + "ns", strlen("ns"))) { + timePrec = TSDB_TIME_PRECISION_NANO; + } else { + errorPrint("Invalid time precision: %s", + arguments->precision); + free(tmp); + return; + } + + if (TSDB_CODE_SUCCESS != taosParseTime( + tmp, &tmpEpoch, strlen(tmp), + timePrec, 0)) { + errorPrint("Input %s, end time error!\n", tmp); + free(tmp); + return; + } + } else { + tmpEpoch = atoll(tmp); + } + + sprintf(argv[i], "%"PRId64"", tmpEpoch); + debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", + __func__, __LINE__, tmp, i, argv[i]); + free(tmp); } } } @@ -552,8 +598,10 @@ int main(int argc, char *argv[]) { int ret = 0; /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ - if (argc > 2) - parse_args(argc, argv, &g_args); + if (argc > 2) { + parse_precision_first(argc, argv, &g_args); + parse_timestamp(argc, argv, &g_args); + } argp_parse(&argp, argc, argv, 0, 0, &g_args); @@ -584,6 +632,7 @@ int main(int argc, char *argv[]) { printf("avro format: %s\n", g_args.avro?"true":"false"); printf("start_time: %" PRId64 "\n", g_args.start_time); printf("end_time: %" PRId64 "\n", g_args.end_time); + printf("precision: %s\n", g_args.precision); printf("data_batch: %d\n", g_args.data_batch); printf("max_sql_len: %d\n", g_args.max_sql_len); printf("table_batch: %d\n", g_args.table_batch); @@ -634,6 +683,7 @@ int main(int argc, char *argv[]) { fprintf(g_fpOfResult, "avro format: %s\n", g_args.avro?"true":"false"); fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); + fprintf(g_fpOfResult, "precision: %s\n", g_args.precision); fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); From 9f8e0e09ff1aa3df418e9207d06aa2e3fa0d798a Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 7 Jul 2021 15:19:45 +0800 Subject: [PATCH 3/5] [TD-4657] : fix typo. --- documentation20/cn/12.taos-sql/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 34b9aba12b..a55592c08d 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -698,7 +698,7 @@ Query OK, 1 row(s) in set (0.001091s) - 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序( _c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序为非法操作。 - 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。且 `LIMIT 5 OFFSET 2` 可以简写为 `LIMIT 2, 5`。 * 在有 GROUP BY 子句的情况下，LIMIT 参数控制的是每个分组中至多允许输出的条数。 -- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。且 `SLIMIT 5 OFFSET 2` 可以简写为 `SLIMIT 2, 5`。 +- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。且 `SLIMIT 5 SOFFSET 2` 可以简写为 `SLIMIT 2, 5`。 - 通过 “>>” 输出结果可以导出到指定文件。 ### 支持的条件过滤操作 From 3da8055c2d78d87c4b4a92b06174ddab061b6025 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Jul 2021 16:04:08 +0800 Subject: [PATCH 4/5] Feature/sangshuduo/td 3973 use jemalloc (#6784) * [TD-3973]: add jemalloc as submodule. * add macro definitions in cmake. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * link jemalloc works. * make install works. * support jemalloc in release.sh. * release script works. * fix a typo. * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for update check compatiblity. * fix position bug. * add ldconfig for jemalloc library cache refresh. * add /etc/ld.so.conf.d/jemalloc.conf for centos Co-authored-by: Shuduo Sang --- packaging/tools/install.sh | 9 +++++++-- packaging/tools/install_arbi.sh | 8 +++++++- packaging/tools/install_arbi_power.sh | 8 +++++++- packaging/tools/install_client_power.sh | 10 ++++++++-- packaging/tools/install_power.sh | 9 +++++++-- packaging/tools/install_tq.sh | 9 +++++++-- packaging/tools/make_install.sh | 8 +++++++- 7 files changed, 50 insertions(+), 11 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 16a99d7a0c..c8970ed821 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -270,7 +270,13 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } @@ -784,7 +790,6 @@ function update_TDengine() { echo "File taos.tar.gz does not exist" exit 1 fi - install_jemalloc tar -zxf taos.tar.gz install_jemalloc diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index 74a7d8cd3d..3a5e641538 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -165,7 +165,13 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } diff --git a/packaging/tools/install_arbi_power.sh b/packaging/tools/install_arbi_power.sh index 864fbb9844..883db2b716 100755 --- a/packaging/tools/install_arbi_power.sh +++ b/packaging/tools/install_arbi_power.sh @@ -158,7 +158,13 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index bb9be2493b..31da0d6131 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -182,7 +182,13 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } @@ -228,6 +234,7 @@ function update_PowerDB() { exit 1 fi tar -zxf power.tar.gz + install_jemalloc echo -e "${GREEN}Start to update PowerDB client...${NC}" # Stop the client shell if running @@ -241,7 +248,6 @@ function update_PowerDB() { install_log install_header install_lib - install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index bf4b1244ad..b73b0b70c0 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -263,7 +263,13 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } @@ -753,7 +759,6 @@ function update_PowerDB() { echo "File power.tar.gz does not exist" exit 1 fi - install_jemalloc tar -zxf power.tar.gz install_jemalloc diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh index 7d4e374827..4bdb39838a 100755 --- a/packaging/tools/install_tq.sh +++ b/packaging/tools/install_tq.sh @@ -270,7 +270,13 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } @@ -753,7 +759,6 @@ function update_tq() { echo "File tq.tar.gz does not exist" exit 1 fi - install_jemalloc tar -zxf tq.tar.gz install_jemalloc diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index b1d56092c2..2b631caa69 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -218,7 +218,13 @@ function install_jemalloc() { /usr/bin/install -c -d /usr/local/share/man/man3 /usr/bin/install -c -m 644 ${binary_dir}/build/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } From c4787210a9509a5b0949378c333bc76c40d0689f Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 7 Jul 2021 16:05:01 +0800 Subject: [PATCH 5/5] [TD-2569] : fix typo. --- documentation20/cn/12.taos-sql/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index a55592c08d..d5cc8675a4 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -1342,7 +1342,7 @@ SELECT function_list FROM stb_name - 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：TOP、BOTTOM、DIFF 以及四则运算）。 - 查询过滤、聚合等操作按照每个切分窗口为独立的单位执行。聚合查询目前支持三种窗口的划分方式： 1. 时间窗口：聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。当 SLIDING 与 INTERVAL 取值相等的时候，滑动窗口即为翻转窗口。 - 2. 状态窗口：使用整数（布尔值）或字符串来标识产生记录时设备的状态量，产生的记录如果具有相同的状态量取值则归属于同一个状态窗口，数值改变后该窗口关闭。状态量所对应的列作为 STAT_WINDOW 语句的参数来指定。 + 2. 状态窗口：使用整数（布尔值）或字符串来标识产生记录时设备的状态量，产生的记录如果具有相同的状态量取值则归属于同一个状态窗口，数值改变后该窗口关闭。状态量所对应的列作为 STATE_WINDOW 语句的参数来指定。 3. 会话窗口：时间戳所在的列由 SESSION 语句的 ts_col 参数指定，会话窗口根据相邻两条记录的时间戳差值来确定是否属于同一个会话——如果时间戳差异在 tol_val 以内，则认为记录仍属于同一个窗口；如果时间变化超过 tol_val，则自动开启下一个窗口。 - WHERE 语句可以指定查询的起止时间和其他过滤条件。 - FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种：