From 9eae7a6e058f5d315769e79aaf1d5885e4fb8821 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 25 Dec 2024 10:16:15 +0800 Subject: [PATCH 01/21] docs/optimize the description in Update 03-taosadapter.md --- docs/zh/14-reference/01-components/03-taosadapter.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/01-components/03-taosadapter.md b/docs/zh/14-reference/01-components/03-taosadapter.md index b74ee77862..bf5060cec5 100644 --- a/docs/zh/14-reference/01-components/03-taosadapter.md +++ b/docs/zh/14-reference/01-components/03-taosadapter.md @@ -14,7 +14,7 @@ taosAdapter 是一个 TDengine 的配套工具,是 TDengine 集群和应用程 taosAdapter 提供以下功能: -- RESTful 接口 +- Websocket/RESTful 接口 - 兼容 InfluxDB v1 写接口 - 兼容 OpenTSDB JSON 和 telnet 格式写入 - 无缝连接到 Telegraf From 20a0ae79050dd74447d9e489b3020b42cf5a0463 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 25 Dec 2024 11:51:37 +0000 Subject: [PATCH 02/21] fix/remove-resolve-fqdn-retry --- include/common/tglobal.h | 1 - source/common/src/tglobal.c | 6 ------ source/libs/sync/src/syncUtil.c | 12 ++++-------- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 584c4b5775..6ecee822ee 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -291,7 +291,6 @@ extern int tsStreamAggCnt; extern bool tsFilterScalarMode; extern int32_t tsMaxStreamBackendCache; extern int32_t tsPQSortMemThreshold; -extern int32_t tsResolveFQDNRetryTime; extern bool tsStreamCoverage; extern bool tsExperimental; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c58ad32a18..355a34d05b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -327,7 +327,6 @@ char tsUdfdLdLibPath[512] = ""; bool tsDisableStream = false; int64_t tsStreamBufferSize = 128 * 1024 * 1024; bool tsFilterScalarMode = false; -int tsResolveFQDNRetryTime = 100; // seconds int tsStreamAggCnt = 100000; bool tsStreamCoverage = false; @@ -956,7 +955,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); @@ -1822,9 +1820,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "pqSortMemThreshold"); tsPQSortMemThreshold = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "resolveFQDNRetryTime"); - tsResolveFQDNRetryTime = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minDiskFreeSize"); tsMinDiskFreeSize = pItem->i64; @@ -2453,7 +2448,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"randErrorDivisor", &tsRandErrDivisor}, {"randErrorScope", &tsRandErrScope}, {"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed}, - {"resolveFQDNRetryTime", &tsResolveFQDNRetryTime}, {"syncHeartbeatInterval", &tsHeartbeatInterval}, {"syncHeartbeatTimeout", &tsHeartbeatTimeout}, {"syncSnapReplMaxWaitN", &tsSnapReplMaxWaitN}, diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 0fedd96e4c..d9eff2e9b8 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -45,14 +45,10 @@ void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = 0xFFFFFFFF; sDebug("vgId:%d, resolve sync addr from fqdn, ep:%s:%u", vgId, pInfo->nodeFqdn, pInfo->nodePort); - for (int32_t i = 0; i < tsResolveFQDNRetryTime; i++) { - int32_t code = taosGetIpv4FromFqdn(pInfo->nodeFqdn, &ipv4); - if (code) { - sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s, retry", vgId, pInfo->nodeId, pInfo->nodeFqdn); - taosSsleep(1); - } else { - break; - } + + int32_t code = taosGetIpv4FromFqdn(pInfo->nodeFqdn, &ipv4); + if (code) { + sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s, retry", vgId, pInfo->nodeId, pInfo->nodeFqdn); } if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { From 484e4c14380dc10e76e4dd29fb452440c723e10d Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 26 Dec 2024 00:53:39 +0000 Subject: [PATCH 03/21] fix/remove-resolve-fqdn-retry-fix-case --- tests/army/alter/test_alter_config.py | 5 ----- tests/script/tsim/show/basic.sim | 2 +- tests/script/tsim/valgrind/checkError1.sim | 2 +- tests/system-test/0-others/test_alter_config.py | 5 ----- tests/system-test/2-query/db.py | 2 +- 5 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tests/army/alter/test_alter_config.py b/tests/army/alter/test_alter_config.py index 1b59293187..229f3150b3 100644 --- a/tests/army/alter/test_alter_config.py +++ b/tests/army/alter/test_alter_config.py @@ -310,11 +310,6 @@ class TDTestCase: "value": 1024 * 1024 * 20 * 10, "category": "global" }, - { - "name": "resolveFQDNRetryTime", - "value": 500, - "category": "global" - }, { "name": "syncHeartbeatInterval", "value": 3000, diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 4c4f9da912..7569cd832c 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -230,7 +230,7 @@ endi sql_error show create stable t0; sql show variables; -if $rows != 94 then +if $rows != 93 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index b7b04a030f..64f9cdec1f 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -120,7 +120,7 @@ if $rows != 3 then endi sql show variables; -if $rows != 94 then +if $rows != 93 then return -1 endi diff --git a/tests/system-test/0-others/test_alter_config.py b/tests/system-test/0-others/test_alter_config.py index 14e77bb489..bdc3a0e835 100644 --- a/tests/system-test/0-others/test_alter_config.py +++ b/tests/system-test/0-others/test_alter_config.py @@ -207,11 +207,6 @@ class TDTestCase: "value": 1024 * 1024 * 20 * 10, "category": "global" }, - { - "name": "resolveFQDNRetryTime", - "value": 500, - "category": "global" - }, { "name": "syncElectInterval", "value": 50000, diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index 0c5c9773c8..66776e0a23 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -47,7 +47,7 @@ class TDTestCase: def case2(self): tdSql.query("show variables") - tdSql.checkRows(94) + tdSql.checkRows(93) for i in range(self.replicaVar): tdSql.query("show dnode %d variables like 'debugFlag'" % (i + 1)) From 05f836c555e0a60ebf6114728b2a4a6a96db169f Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 26 Dec 2024 16:10:17 +0800 Subject: [PATCH 04/21] fix sma filtering returned wrong rows --- source/client/test/clientTests.cpp | 4 ++++ source/libs/scalar/src/filter.c | 4 ++-- tests/pytest/util/sql.py | 2 +- tests/system-test/2-query/smaTest.py | 35 ++++++++++++++++++++++++---- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 3e4667fbe7..60f0a72e39 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -126,6 +126,10 @@ void fetchCallback(void* param, void* res, int32_t numOfRow) { void queryCallback(void* param, void* res, int32_t code) { if (code != TSDB_CODE_SUCCESS) { (void)printf("failed to execute, reason:%s\n", taos_errstr(res)); + taos_free_result(res); + tsem_t *sem = (tsem_t *)param; + tsem_post(sem); + return; } (void)printf("start to fetch data\n"); taos_fetch_raw_block_a(res, fetchCallback, param); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b329bbbd44..6689e12edd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -5128,8 +5128,8 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SOperatorNode *pOper = (SOperatorNode *)pNode; - SValueNode *valNode = (SValueNode *)pOper->pRight; - if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SExprNode* pLeft = (SExprNode*)pOper->pLeft; + if (IS_NUMERIC_TYPE(pLeft->resType.type) || pLeft->resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SNode* pLeft = NULL, *pRight = NULL; int32_t code = nodesCloneNode(pOper->pLeft, &pLeft); if (TSDB_CODE_SUCCESS != code) { diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index bb7b8411f9..cdfe3ce8a0 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -100,7 +100,7 @@ class TDSql: if drop: s = f'drop database if exists {dbname}' self.cursor.execute(s) - s = f'create database {dbname}' + s = f'create database {dbname} stt_trigger 1' for k, v in kwargs.items(): s += f" {k} {v}" if "duration" not in kwargs: diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 04fb893e75..355ac04707 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -39,6 +39,27 @@ class TDTestCase: self.create_tables(); self.ts = 1500000000000 + def test_TD_33336(self): + sql = "flush database db" + tdSql.execute(sql) + time.sleep(5) + sql = f'select last(ts) + 1d, last(ts) - 1d from db.t1' + tdSql.query(sql, queryTimes=1) + lastTs_add1d = tdSql.queryResult[0][0] + lastTs_sub1d = tdSql.queryResult[0][1] + + sql = f'select count(*) from db.t1 where ts < "{lastTs_add1d}" and vc1 = 1' + tdSql.query(sql, queryTimes=1) + all_row_count = tdSql.queryResult[0][0] + tdLog.debug(f"all rows: {all_row_count}") + + sql = f'select count(*) from db.t1 where ts < "{lastTs_sub1d}" and vc1 = 1' + tdSql.query(sql, queryTimes=1) + row_count_sub1d = tdSql.queryResult[0][0] + tdLog.debug(f"row_count_sub1d: {row_count_sub1d}") + + if row_count_sub1d > all_row_count: + tdLog.exit(f' err rows returned for sql: {sql} row_count_sub1d: {row_count_sub1d} > all_row_count: {all_row_count}') # run case def run(self): @@ -53,6 +74,8 @@ class TDTestCase: # self.test_case2() tdLog.debug(" LIMIT test_case2 ............ [OK]") + self.test_TD_33336() + # stop def stop(self): tdSql.close() @@ -65,11 +88,11 @@ class TDTestCase: # create table def create_tables(self, dbname="db"): # super table - tdSql.execute(f"create table {dbname}.st(ts timestamp, i1 int,i2 int) tags(area int)") + tdSql.execute(f"create table {dbname}.st(ts timestamp, i1 int,i2 int, vc1 varchar(255)) tags(area int)") # child table tdSql.execute(f"create table {dbname}.t1 using {dbname}.st tags(1)") - tdSql.execute(f"create table {dbname}.st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) ") + tdSql.execute(f"create table {dbname}.st1(ts timestamp, i1 int ,i2 int, vc1 varchar(255)) tags(area int) sma(i2) ") tdSql.execute(f"create table {dbname}.t4 using {dbname}.st1 tags(1)") return @@ -98,7 +121,11 @@ class TDTestCase: sql = pre_insert tdLog.debug("insert table %s rows=%d ..." % (tbname, count)) for i in range(count): - sql += " (%d,%d,%d)" % (ts_start + i*1000, i, i+1) + if random.randint(0, 4) > 2: + tail = '' + else: + tail = 'asd' + sql += " (%d,%d,%d,%s)" % (ts_start + i*1000, i, i+1, '"' + str(random.randint(0,5)) + f'{tail}"') if i > 0 and i % 20000 == 0: tdLog.info("%d rows inserted" % i) tdSql.execute(sql) @@ -125,4 +152,4 @@ class TDTestCase: # add case with filename # tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From 91abb86693fc9a9857fff73f447b93eab5b2d77e Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 27 Dec 2024 02:22:54 +0000 Subject: [PATCH 05/21] fix/resolve_fqdn_retry_times --- source/libs/sync/src/syncUtil.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index d9eff2e9b8..38fd660cfd 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -24,6 +24,8 @@ #include "tglobal.h" #include "ttime.h" +#define FQDNRETRYTIMES 100 + static void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { int32_t len = tsnprintf(buf, bufLen, "{num:%d, as:%d, [", pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->replicaNum; ++i) { @@ -46,9 +48,14 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* uint32_t ipv4 = 0xFFFFFFFF; sDebug("vgId:%d, resolve sync addr from fqdn, ep:%s:%u", vgId, pInfo->nodeFqdn, pInfo->nodePort); - int32_t code = taosGetIpv4FromFqdn(pInfo->nodeFqdn, &ipv4); - if (code) { - sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s, retry", vgId, pInfo->nodeId, pInfo->nodeFqdn); + for (int32_t i = 0; i < FQDNRETRYTIMES; i++) { + int32_t code = taosGetIpv4FromFqdn(pInfo->nodeFqdn, &ipv4); + if (code) { + sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s, retry", vgId, pInfo->nodeId, pInfo->nodeFqdn); + taosSsleep(1); + } else { + break; + } } if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { From 8bf82a9ce85e791a22858838c2b9e3fb8264f00e Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Fri, 27 Dec 2024 11:13:52 +0800 Subject: [PATCH 06/21] Update 01-taosd.md --- docs/zh/14-reference/01-components/01-taosd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 7a0a306a41..0a4160a00b 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -250,7 +250,7 @@ charset 的有效值是 UTF-8。 |minimalLogDirGB | |不支持动态修改 |日志文件夹所在磁盘可用空间大小小于该值时,停止写日志,单位 GB,默认值 1| |numOfLogLines | |支持动态修改 立即生效 |单个日志文件允许的最大行数,默认值 10,000,000| |asyncLog | |支持动态修改 立即生效 |日志写入模式,0:同步,1:异步,默认值 1| -|logKeepDays | |支持动态修改 立即生效 |日志文件的最长保存时间,单位:天,默认值 0,意味着无限保存,日志文件不会被重命名,也不会有新的日志文件滚动产生,但日志文件的内容有可能会不断滚动,取决于日志文件大小的设置;当设置为大于 0 的值时,当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy,其中 yyy 为日志文件最后修改的时间戳,并滚动产生新的日志文件| +|logKeepDays | |支持动态修改 立即生效 |日志文件的最长保存时间,单位:天,默认值 0,小于等于0意味着只有两个日志文件相互切换保存日志,超过两个文件保存数量的日志会被删除;当设置为大于 0 的值时,当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy,其中 yyy 为日志文件最后修改的时间戳,并滚动产生新的日志文件| |slowLogThreshold|3.3.3.0 后|支持动态修改 立即生效 |慢查询门限值,大于等于门限值认为是慢查询,单位秒,默认值 3 | |slowLogMaxLen |3.3.3.0 后|支持动态修改 立即生效 |慢查询日志最大长度,取值范围 1-16384,默认值 4096| |slowLogScope |3.3.3.0 后|支持动态修改 立即生效 |慢查询记录类型,取值范围 ALL/QUERY/INSERT/OTHERS/NONE,默认值 QUERY| From fc6af95ba212d67d87f12596dcc545ce8717ebbb Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Fri, 27 Dec 2024 11:51:50 +0800 Subject: [PATCH 07/21] fix(query)[TS-5058]. Add note about scenarios where interval auto offset may not work --- docs/en/14-reference/03-taos-sql/12-distinguished.md | 1 + docs/zh/14-reference/03-taos-sql/12-distinguished.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/14-reference/03-taos-sql/12-distinguished.md b/docs/en/14-reference/03-taos-sql/12-distinguished.md index 4c09f140a1..e98b654be3 100644 --- a/docs/en/14-reference/03-taos-sql/12-distinguished.md +++ b/docs/en/14-reference/03-taos-sql/12-distinguished.md @@ -148,6 +148,7 @@ When using time windows, note: - The window width of the aggregation period is specified by the keyword INTERVAL, with the shortest interval being 10 milliseconds (10a); it also supports an offset (the offset must be less than the interval), which is the offset of the time window division compared to "UTC moment 0". The SLIDING statement is used to specify the forward increment of the aggregation period, i.e., the duration of each window slide forward. - When using the INTERVAL statement, unless in very special cases, it is required to configure the timezone parameter in the taos.cfg configuration files of both the client and server to the same value to avoid frequent cross-time zone conversions by time processing functions, which can cause severe performance impacts. - The returned results have a strictly monotonically increasing time-series. +- When using AUTO as the window offset, if the WHERE time condition is complex, such as multiple AND/OR/IN combinations, AUTO may not take effect. In such cases, you can manually specify the window offset to resolve the issue. - When using AUTO as the window offset, if the window width unit is d (day), n (month), w (week), y (year), such as: INTERVAL(1d, AUTO), INTERVAL(3w, AUTO), the TSMA optimization cannot take effect. If TSMA is manually created on the target table, the statement will report an error and exit; in this case, you can explicitly specify the Hint SKIP_TSMA or not use AUTO as the window offset. ### State Window diff --git a/docs/zh/14-reference/03-taos-sql/12-distinguished.md b/docs/zh/14-reference/03-taos-sql/12-distinguished.md index ef8faff85a..bd6705b33b 100644 --- a/docs/zh/14-reference/03-taos-sql/12-distinguished.md +++ b/docs/zh/14-reference/03-taos-sql/12-distinguished.md @@ -138,6 +138,7 @@ SELECT COUNT(*) FROM meters WHERE _rowts - voltage > 1000000; - 聚合时间段的窗口宽度由关键词 INTERVAL 指定,最短时间间隔 10 毫秒(10a);并且支持偏移 offset(偏移必须小于间隔),也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量,也即每次窗口向前滑动的时长。 - 使用 INTERVAL 语句时,除非极特殊的情况,都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值,以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 - 返回的结果中时间序列严格单调递增。 +- 使用 AUTO 作为窗口偏移量时,如果 WHERE 时间条件比较复杂,比如多个 AND/OR/IN 互相组合,那么 AUTO 可能不生效,这种情况可以通过手动指定窗口偏移量进行解决。 - 使用 AUTO 作为窗口偏移量时,如果窗口宽度的单位是 d (天), n (月), w (周), y (年),比如: INTERVAL(1d, AUTO), INTERVAL(3w, AUTO),此时 TSMA 优化无法生效。如果目标表上手动创建了TSMA,语句会报错退出;这种情况下,可以显式指定 Hint SKIP_TSMA 或者不使用 AUTO 作为窗口偏移量。 ### 状态窗口 From dc1ab696d4a320f1c26017b87c224fe8005db599 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 19:40:34 +0800 Subject: [PATCH 08/21] Create taosd-ci-build.yml --- .github/workflows/taosd-ci-build.yml | 65 ++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/taosd-ci-build.yml diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml new file mode 100644 index 0000000000..3ac3caaa7f --- /dev/null +++ b/.github/workflows/taosd-ci-build.yml @@ -0,0 +1,65 @@ +name: TDengine CI + +on: + pull_request: + branches: + - 'main' + - '3.0' + - '3.1' + push: + branches: + - 'enh/main/TD-33380' + - '3.0' + - '3.1' + - 'main' + +jobs: + build: + runs-on: ubuntu-latest + name: Run unit tests + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.18 + + - name: Install system dependencies + run: | + sudo apt update -y + sudo apt install -y build-essential cmake libgeos-dev + + - name: Install TDengine + run: | + mkdir debug + cd debug + cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false + make -j 4 + sudo make install + which taosd + which taosadapter + which taoskeeper + + - name: Start taosd + run: | + cp /etc/taos/taos.cfg ./ + sudo echo "supportVnodes 256" >> taos.cfg + nohup sudo taosd -c taos.cfg & + + - name: Start taosadapter + run: nohup sudo taosadapter & + + - name: Run tests with coverage + working-directory: tools/keeper + run: | + taosBenchmark -t 10 -n 10 -y + taos -s "select count(*) from test.meters" + + - name: Clean up + if: always() + run: | + if pgrep taosd; then sudo pkill taosd; fi + if pgrep taosadapter; then sudo pkill taosadapter; fi From 0928590519fff3d01e6f1c2c99932680088813fa Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 19:43:09 +0800 Subject: [PATCH 09/21] Update taosd-ci-build.yml --- .github/workflows/taosd-ci-build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 3ac3caaa7f..4495fd737e 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -36,7 +36,7 @@ jobs: run: | mkdir debug cd debug - cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false + cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=true -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false make -j 4 sudo make install which taosd @@ -52,8 +52,7 @@ jobs: - name: Start taosadapter run: nohup sudo taosadapter & - - name: Run tests with coverage - working-directory: tools/keeper + - name: Run tests with taosBenchmark run: | taosBenchmark -t 10 -n 10 -y taos -s "select count(*) from test.meters" From a675882d14c898b080796b418ab36d851128ea41 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 19:53:37 +0800 Subject: [PATCH 10/21] fix: add dependencies for taos-tools compile --- .github/workflows/taosd-ci-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 4495fd737e..8a0294a0a6 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -30,7 +30,8 @@ jobs: - name: Install system dependencies run: | sudo apt update -y - sudo apt install -y build-essential cmake libgeos-dev + sudo apt install -y build-essential cmake libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libssl-dev gawk + - name: Install TDengine run: | From a005a132d9e75d3c98e0e491314279d79f60be4b Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 20:15:52 +0800 Subject: [PATCH 11/21] Update taosd-ci-build.yml --- .github/workflows/taosd-ci-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 8a0294a0a6..3adfde3460 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -8,7 +8,6 @@ on: - '3.1' push: branches: - - 'enh/main/TD-33380' - '3.0' - '3.1' - 'main' From 746cbc09c3338839866c4cc98fff70063031b745 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 26 Dec 2024 17:24:17 +0800 Subject: [PATCH 12/21] test(analytics): add tests for tdanalytics. --- source/util/src/tanalytics.c | 6 +-- tests/script/sh/stop_dnodes.sh | 8 +-- tests/script/tsim/analytics/basic0.sim | 67 +++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/source/util/src/tanalytics.c b/source/util/src/tanalytics.c index e68edd4b76..bf2cb4fd07 100644 --- a/source/util/src/tanalytics.c +++ b/source/util/src/tanalytics.c @@ -20,7 +20,7 @@ #ifdef USE_ANALYTICS #include -#define ANAL_ALGO_SPLIT "," +#define ANALYTICS_ALOG_SPLIT_CHAR "," typedef struct { int64_t ver; @@ -136,7 +136,7 @@ bool taosAnalGetOptStr(const char *option, const char *optName, char *optValue, return false; } - pEnd = strstr(pStart, ANAL_ALGO_SPLIT); + pEnd = strstr(pStart, ANALYTICS_ALOG_SPLIT_CHAR); if (optMaxLen > 0) { if (pEnd > pStart) { int32_t len = (int32_t)(pEnd - pStart); @@ -168,7 +168,7 @@ bool taosAnalGetOptInt(const char *option, const char *optName, int64_t *optValu int32_t bufLen = tsnprintf(buf, sizeof(buf), "%s=", optName); char *pos1 = strstr(option, buf); - char *pos2 = strstr(option, ANAL_ALGO_SPLIT); + char *pos2 = strstr(option, ANALYTICS_ALOG_SPLIT_CHAR); if (pos1 != NULL) { *optValue = taosStr2Int64(pos1 + bufLen, NULL, 10); return true; diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 8923804547..da2083b013 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -13,7 +13,7 @@ if [ -n "$PID" ]; then systemctl stop taosd fi -PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w taosd | grep -v grep | grep -v taosanode | awk '{print $2}'` while [ -n "$PID" ]; do echo kill -9 $PID #pkill -9 taosd @@ -38,10 +38,10 @@ while [ -n "$PID" ]; do else lsof -nti:6030 | xargs kill -9 fi - PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` + PID=`ps -ef|grep -w taos | grep -v grep |grep -v taosanode| awk '{print $2}'` done -PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode|awk '{print $2}'` while [ -n "$PID" ]; do echo kill -9 $PID #pkill -9 tmq_sim @@ -52,5 +52,5 @@ while [ -n "$PID" ]; do else lsof -nti:6030 | xargs kill -9 fi - PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` + PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode| awk '{print $2}'` done \ No newline at end of file diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 77c9184e8f..35774e7682 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -3,7 +3,17 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print =============== create anode +print =============== failed to create anode on '127.0.0.1:1101' +sql_error create anode '127.0.0.1:1101' + +sql show anodes +if $rows != 0 then + return -1 +endi + +sql_error drop anode 1 + +print ================ create anode sql create anode '192.168.1.116:6050' sql show anodes @@ -58,6 +68,61 @@ if $data00 != 1 then return -1 endi +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=-10') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, every=0') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters, conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo=holtwinters , conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo = holtwinters , conf = 50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, , ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, a =') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, = a ,') from ct1 +sql select forecast(c1, 'conf=50 ,algo = arima') from ct1 +sql select forecast(c1, 'conf=50 ,algo = arima, rows=1') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=2') from ct1 +if $rows != 10 then + return -1 +endi + +if $data03 != 28 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + print expect 23-11-15 06:13:20.000 , actual $data00 + return -1 +endi + +if $data10 != @23-11-15 06:13:20.002@ then + print expect 23-11-15 06:13:20.002 , actual $data10 + return -1 +endi + +if $data20 != @23-11-15 06:13:20.004@ then + return -1 +endi + +print test the every option and rows option + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=100,rows=5') from ct1 +if $rows != 5 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + return -1 +endi + +if $data10 != @23-11-15 06:13:20.100@ then + return -1 +endi sql drop anode 1 sql show anodes From 98a3fa1ab5ac633d41d01d41c01fac9a6facf1f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 26 Dec 2024 18:17:02 +0800 Subject: [PATCH 13/21] test: add test case for anamaly detection in tdanalytics. --- .../libs/executor/src/anomalywindowoperator.c | 49 +++++++++---------- tests/script/tsim/analytics/basic0.sim | 44 +++++++++++++++-- 2 files changed, 62 insertions(+), 31 deletions(-) diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index dd1a52022e..eb72edb964 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -61,9 +61,13 @@ static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + size_t keyBufSize = 0; + int32_t num = 0; + SExprInfo* pExprInfo = NULL; + const char* id = GET_TASKID(pTaskInfo); - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; SAnomalyWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAnomalyWindowOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SAnomalyWindowPhysiNode* pAnomalyNode = (SAnomalyWindowPhysiNode*)physiNode; @@ -74,13 +78,13 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p } if (!taosAnalGetOptStr(pAnomalyNode->anomalyOpt, "algo", pInfo->algoName, sizeof(pInfo->algoName))) { - qError("failed to get anomaly_window algorithm name from %s", pAnomalyNode->anomalyOpt); + qError("%s failed to get anomaly_window algorithm name from %s", id, pAnomalyNode->anomalyOpt); code = TSDB_CODE_ANA_ALGO_NOT_FOUND; goto _error; } if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) { - qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName); + qError("%s failed to get anomaly_window algorithm url from %s", id, pInfo->algoName); code = TSDB_CODE_ANA_ALGO_NOT_LOAD; goto _error; } @@ -94,20 +98,18 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p SExprInfo* pScalarExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr); QUERY_CHECK_CODE(code, lino, _error); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); } - size_t keyBufSize = 0; - int32_t num = 0; - SExprInfo* pExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pFuncs, NULL, &pExprInfo, &num); QUERY_CHECK_CODE(code, lino, _error); initResultSizeInfo(&pOperator->resultInfo, 4096); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, id, pTaskInfo->streamInfo.pState, + &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); SSDataBlock* pResBlock = createDataBlockFromDescNode(pAnomalyNode->window.node.pOutputDataBlockDesc); @@ -124,27 +126,19 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p pInfo->anomalyCol = extractColumnFromColumnNode(pColNode); pInfo->anomalyKey.type = pInfo->anomalyCol.type; pInfo->anomalyKey.bytes = pInfo->anomalyCol.bytes; + pInfo->anomalyKey.pData = taosMemoryCalloc(1, pInfo->anomalyCol.bytes); - if (pInfo->anomalyKey.pData == NULL) { - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalyKey.pData, code, lino, _error, terrno) int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes; pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize); - if (pInfo->anomalySup.pResultRow == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.pResultRow, code, lino, _error, terrno) + pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*)); - if (pInfo->anomalySup.blocks == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.blocks, code, lino, _error, terrno) + pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow)); - if (pInfo->anomalySup.windows == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.windows, code, lino, _error, terrno) code = filterInitFromNode((SNode*)pAnomalyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); QUERY_CHECK_CODE(code, lino, _error); @@ -162,18 +156,21 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p *pOptrInfo = pOperator; - qDebug("anomaly_window operator is created, algo:%s url:%s opt:%s", pInfo->algoName, pInfo->algoUrl, + qDebug("%s anomaly_window operator is created, algo:%s url:%s opt:%s", id, pInfo->algoName, pInfo->algoUrl, pInfo->anomalyOpt); return TSDB_CODE_SUCCESS; _error: + qError("%s failed to create anomaly_window operator, line:%d algo:%s code:%s", id, lino, pAnomalyNode->anomalyOpt, + tstrerror(code)); + if (pInfo != NULL) { anomalyDestroyOperatorInfo(pInfo); } destroyOperatorAndDownstreams(pOperator, &downstream, 1); pTaskInfo->code = code; - qError("failed to create anomaly_window operator, algo:%s code:0x%x", pInfo->algoName, code); + return code; } diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 35774e7682..3ac49b1fc3 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -40,7 +40,7 @@ print $data00 $data01 $data02 sql use d0 print =============== create super table, include column type for count/sum/min/max/first -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 tinyint, c5 bigint, c6 varchar(12)) tags (t1 int unsigned) sql show stables if $rows != 1 then @@ -52,10 +52,11 @@ sql create table ct1 using stb tags(1000) print ==================== insert data # input_list = [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40] -sql insert into ct1(ts, c1) values(now-1a, 5)(now+1a, 14)(now+2a, 15)(now+3a, 15)(now+4a, 14) -sql insert into ct1(ts, c1) values(now+5a, 19)(now+6a, 17)(now+7a, 16)(now+8a, 20)(now+9a, 22) -sql insert into ct1(ts, c1) values(now+10a, 8)(now+11a, 21)(now+12a, 28)(now+13a, 11)(now+14a, 9) -sql insert into ct1(ts, c1) values(now+15a, 29)(now+16a, 40) +sql insert into ct1(ts, c1, c2, c3, c4, c5, c6) values(now-1a, 5, 5, 5, 5, 5, 'a')(now+1a, 14, 14, 14, 14, 14, 'a')(now+2a, 15, 15, 15, 15, 15, 'a') +sql insert into ct1 values(now+3a, 15, 15, 15, 15, 15, 'a')(now+4a, 14, 14, 14, 14, 14, 'a')(now+5a, 19, 19, 19, 19, 19, 'a')(now+6a, 17, 17, 17, 17, 17, 'a') +sql insert into ct1 values(now+7a, 16, 16, 16, 16, 16, 'a')(now+8a, 20, 20, 20, 20, 20, 'a')(now+9a, 22, 22, 22, 22, 22, 'a') +sql insert into ct1 values(now+10a, 8, 8, 8, 8, 8, 'a')(now+11a, 21, 21, 21, 21, 21, 'a')(now+12a, 28, 28, 28, 28, 28, 'a')(now+13a, 11, 11, 11, 11, 11, 'a')(now+14a, 9, 9, 9, 9, 9, 'a') +sql insert into ct1 values(now+15a, 29, 29, 29, 29, 29, 'a')(now+16a, 40, 40, 40, 40, 40, 'a') sql select count(*) from ct1 if $data00 != 17 then @@ -68,6 +69,25 @@ if $data00 != 1 then return -1 endi +print ================= try every loaded anomaly detection algorithm +sql select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=lof'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=shesd'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=grubbs'); + +print ================= try every column type of column +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c2, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c3, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c4, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c5, 'algo=ksigma,k=2'); + +print =================== invalid column type +sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2'); +sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 + + sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 @@ -83,8 +103,15 @@ sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ,' sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, , ,') from ct1 sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, a =') from ct1 sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, = a ,') from ct1 + +print =================== valid column type sql select forecast(c1, 'conf=50 ,algo = arima') from ct1 sql select forecast(c1, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c2, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c3, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c4, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=2') from ct1 if $rows != 10 then @@ -131,6 +158,13 @@ if $rows != 0 then return -1 endi +sleep 1000 + +print ===================== query without anodes +sql_error select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql_error select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); + + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check From e8f63cded9e3b946e5daf89a30a09679e27d9e66 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 27 Dec 2024 10:27:05 +0800 Subject: [PATCH 14/21] fix stream state operator issue --- .../executor/src/streamtimewindowoperator.c | 17 ++++++- tests/script/tsim/stream/basic5.sim | 49 ++++++++++++++----- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 91af218a7e..031d2e8bdc 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -4466,6 +4466,19 @@ _end: return code; } +static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pResults) { + SSessionKey checkKey = {0}; + getSessionHashKey(pKey, &checkKey); + if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) { + return true; + } + + if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) { + return true; + } + return false; +} + static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, SSHashObj* pStDeleted) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4518,7 +4531,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); QUERY_CHECK_CODE(code, lino, _end); - releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); + if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) { + releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); + } setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, diff --git a/tests/script/tsim/stream/basic5.sim b/tests/script/tsim/stream/basic5.sim index d7dd603d3c..5d90740b13 100644 --- a/tests/script/tsim/stream/basic5.sim +++ b/tests/script/tsim/stream/basic5.sim @@ -2,13 +2,28 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 135 system sh/cfg.sh -n dnode1 -c streamBufferSize -v 10 +system sh/cfg.sh -n dnode1 -c checkpointinterval -v 60 +system sh/cfg.sh -n dnode1 -c snodeAddress -v 127.0.0.1:873 system sh/exec.sh -n dnode1 -s start sleep 500 sql connect +print step1 ============= -print step1============= +print ================ create snode +sql show snodes +if $rows != 0 then + return -1 +endi + +sql create snode on dnode 1; +sql show snodes; +if $rows != 1 then + return -1 +endi + +print ============== snode created , create db sql create database test3 vgroups 1; sql use test3; @@ -57,7 +72,7 @@ loop8: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -87,7 +102,7 @@ loop9: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -127,7 +142,7 @@ loop10: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -162,7 +177,7 @@ loop11: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -194,7 +209,7 @@ loop11: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -239,7 +254,7 @@ loop12: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -315,7 +330,7 @@ loop13: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -369,7 +384,6 @@ if $data24 != 1 then endi print step4============= - sql create database test6 vgroups 4; sql use test6; sql create stable st(ts timestamp,a int,b int,c int,d int) tags(ta int,tb int,tc int); @@ -396,7 +410,7 @@ loop14: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -449,4 +463,17 @@ if $data25 != 2 then goto loop14 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print sleep for 1min for checkpoint generate +sleep 60000 + +print ================== restart to load checkpoint from snode + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sleep 500 +sql connect + +sleep 30000 + +sql select start_ver, checkpoint_ver from information_schema.ins_stream_tasks; From a334bf19914dbede98207c18441ca7680285dc4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 27 Dec 2024 11:19:24 +0800 Subject: [PATCH 15/21] test: update the sim, set the output as uint, instead of int to avoid overflow. --- source/libs/function/src/builtinsimpl.c | 2 +- tests/script/tsim/stream/basic5.sim | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1c77d12fb2..707018ac65 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6429,7 +6429,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return code; } - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%u] Stt_Rows=[%u] ", pData->numOfInmemRows, pData->numOfSttRows); varDataSetLen(st, len); code = colDataSetVal(pColInfo, row++, st, false); diff --git a/tests/script/tsim/stream/basic5.sim b/tests/script/tsim/stream/basic5.sim index 5d90740b13..866fbd3ebe 100644 --- a/tests/script/tsim/stream/basic5.sim +++ b/tests/script/tsim/stream/basic5.sim @@ -476,4 +476,8 @@ sql connect sleep 30000 -sql select start_ver, checkpoint_ver from information_schema.ins_stream_tasks; +sql select start_ver, checkpoint_ver from information_schema.ins_stream_tasks where level='source'; +sleep 500 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT + From 938f6a5fa05068277db223a7b6b26b881367f5d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Dec 2024 22:28:18 +0800 Subject: [PATCH 16/21] git: set tags --- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taostools_CMakeLists.txt.in | 2 +- cmake/taosws_CMakeLists.txt.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ef6ed4af1d..13826a1a74 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9a6a5329ae..9bbda8309f 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 17446d184d..b013d45911 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 28342a0e4472d2ee644b0a1714d8fb82cdab7603 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 15:44:34 +0800 Subject: [PATCH 17/21] refactor: break long lines with backslash --- .github/workflows/taosd-ci-build.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 3adfde3460..d0915b9547 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -1,4 +1,4 @@ -name: TDengine CI +name: TDengine Build on: pull_request: @@ -29,14 +29,17 @@ jobs: - name: Install system dependencies run: | sudo apt update -y - sudo apt install -y build-essential cmake libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libssl-dev gawk + sudo apt install -y build-essential cmake \ + libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev \ + zlib1g pkg-config libssl-dev gawk - - name: Install TDengine + - name: Build and install TDengine run: | - mkdir debug - cd debug - cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=true -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false + mkdir debug && cd debug + cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false \ + -DBUILD_TOOLS=true -DBUILD_TEST=off \ + -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false make -j 4 sudo make install which taosd From 6987f3692ca1715263435c70e6cdadcc5eaa7115 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 15:57:41 +0800 Subject: [PATCH 18/21] chore: add build badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f827c38975..33a3f948ce 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@

+[![Build Status](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml/badge.svg)] [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
From d855b4f59a7991271161da70771abe7a07bf591a Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 16:04:48 +0800 Subject: [PATCH 19/21] add link to the build workflow --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33a3f948ce..ff72412434 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

-[![Build Status](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml/badge.svg)] +[![Build Status](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml/badge.svg)](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
From 7c2c437a633effe90082ca0c66db85c81ee033ec Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 16:40:57 +0800 Subject: [PATCH 20/21] refactor: build trigger [skip ci] --- .github/workflows/taosd-ci-build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index d0915b9547..0876f5b731 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -6,12 +6,11 @@ on: - 'main' - '3.0' - '3.1' - push: - branches: - - '3.0' - - '3.1' - - 'main' - + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest From 080ebb1b8f8f488a9d26510c696e33e8abd2ab48 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 29 Dec 2024 17:23:22 +0800 Subject: [PATCH 21/21] git: set tags --- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taostools_CMakeLists.txt.in | 2 +- cmake/taosws_CMakeLists.txt.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index b013d45911..17446d184d 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE