From 152d945f3c7a7fcec18660b790de485e2c9f56f0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 27 Nov 2024 11:44:57 +0800 Subject: [PATCH 01/20] enh: add case to describe --- tests/army/cmdline/json/taosCli.json | 57 ++++++++++++++++++++ tests/army/cmdline/taosCli.py | 63 ++++++++++++++++++++++ tests/army/frame/caseBase.py | 79 +++++++++++++++++++++++++++- tests/army/frame/eutil.py | 11 +++- 4 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 tests/army/cmdline/json/taosCli.json create mode 100644 tests/army/cmdline/taosCli.py diff --git a/tests/army/cmdline/json/taosCli.json b/tests/army/cmdline/json/taosCli.json new file mode 100644 index 0000000000..2dcd28afdb --- /dev/null +++ b/tests/army/cmdline/json/taosCli.json @@ -0,0 +1,57 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count":1, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"test", + "drop":"yes" + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 1, + "insert_rows":10, + "timestamp_step":1000, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "int", "count": 1000, "max": 2000, "min": 0 } + ], + "tags":[ + { + "type":"binary", + "name":"location", + "max":64, + "min":1, + "values":[ + "San Francisco", + "Los Angles", + "San Diego", + "San Jose", + "Palo Alto", + "Campbell", + "Mountain View", + "Sunnyvale", + "Santa Clara", + "Cupertino" + ] + } + ] + } + ] + } + ] +} diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py new file mode 100644 index 0000000000..3248ac85ba --- /dev/null +++ b/tests/army/cmdline/taosCli.py @@ -0,0 +1,63 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import frame.eos +import frame.etime +import frame.etool +import frame.etool +import frame.etool +import frame.etool +import taos +import frame.etool +import frame + +from frame.log import * +from frame.sql import * +from frame.cases import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * + + +class TDTestCase(TBase): + updatecfgDict = { + 'slowLogScope':"query" + } + + def checkDescribe(self): + tdLog.info(f"check describe show full.") + + # insert + json = "cmdline/json/taosCli.json" + db, stb, childCount, insertRows = self.insertBenchJson(json) + sql = f"describe {db}.{stb};" + # check result + tdSql.query(sql) + tdSql.checkRows(2 + 1000) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # check show whole + self.checkDescribe() + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 491e432df7..9cd90a4d6f 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -17,9 +17,13 @@ import time import datetime import random import copy +import json +import frame.eutil from frame.log import * from frame.sql import * +from frame import * +import frame # test case base class TBase: @@ -305,4 +309,77 @@ class TBase: if strs != "": strs += sepa strs += f"'{ls}'" - return strs \ No newline at end of file + return strs + +# +# taosBenchmark +# + + # run taosBenchmark and check insert Result + def insertBenchJson(self, jsonFile, options="", checkStep=False): + # exe insert + cmd = f"{options} -f {jsonFile}" + frame.etool.benchMark(command = cmd) + + # + # check insert result + # + with open(jsonFile, "r") as file: + data = json.load(file) + + db = data["databases"][0]["dbinfo"]["name"] + stb = data["databases"][0]["super_tables"][0]["name"] + child_count = data["databases"][0]["super_tables"][0]["childtable_count"] + insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] + timestamp_step = data["databases"][0]["super_tables"][0]["timestamp_step"] + + # drop + try: + drop = data["databases"][0]["dbinfo"]["drop"] + except: + drop = "yes" + + # command is first + if options.find("-Q") != -1: + drop = "no" + + # cachemodel + try: + cachemode = data["databases"][0]["dbinfo"]["cachemodel"] + except: + cachemode = None + + # vgropus + try: + vgroups = data["databases"][0]["dbinfo"]["vgroups"] + except: + vgroups = None + + tdLog.info(f"get json info: db={db} stb={stb} child_count={child_count} insert_rows={insert_rows} \n") + + # all count insert_rows * child_table_count + sql = f"select * from {db}.{stb}" + tdSql.query(sql) + tdSql.checkRows(child_count * insert_rows) + + # timestamp step + if checkStep: + sql = f"select * from (select diff(ts) as dif from {db}.{stb} partition by tbname) where dif != {timestamp_step};" + tdSql.query(sql) + tdSql.checkRows(0) + + if drop.lower() == "yes": + # check database optins + sql = f"select `vgroups`,`cachemodel` from information_schema.ins_databases where name='{db}';" + tdSql.query(sql) + + if cachemode != None: + + value = frame.eutil.removeQuota(cachemode) + tdLog.info(f" deal both origin={cachemode} after={value}") + tdSql.checkData(0, 1, value) + + if vgroups != None: + tdSql.checkData(0, 0, vgroups) + + return db, stb,child_count, insert_rows \ No newline at end of file diff --git a/tests/army/frame/eutil.py b/tests/army/frame/eutil.py index a32e6e9a38..459a45b0b0 100644 --- a/tests/army/frame/eutil.py +++ b/tests/army/frame/eutil.py @@ -25,4 +25,13 @@ import psutil # cpu frequent as random def cpuRand(max): decimal = int(str(psutil.cpu_freq().current).split(".")[1]) - return decimal % max \ No newline at end of file + return decimal % max + +# remove single and doulbe quotation +def removeQuota(origin): + value = "" + for c in origin: + if c != '\'' and c != '"': + value += c + + return value \ No newline at end of file From c4f6db415be66d89934762320e83d8654399b777 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 27 Nov 2024 11:46:51 +0800 Subject: [PATCH 02/20] test: add case to cases.task --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b7298d359b..d2211a1406 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -52,6 +52,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py ,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py +,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py # # system test From 9f6aac4b73ec752cb0a9f76730511fad5833a93a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 27 Nov 2024 13:45:58 +0800 Subject: [PATCH 03/20] fix: add desc command --- tests/army/cmdline/taosCli.py | 6 +++++- tools/shell/src/shellEngine.c | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index 3248ac85ba..d446a25c86 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -44,8 +44,12 @@ class TDTestCase(TBase): # insert json = "cmdline/json/taosCli.json" db, stb, childCount, insertRows = self.insertBenchJson(json) + # describe sql = f"describe {db}.{stb};" - # check result + tdSql.query(sql) + tdSql.checkRows(2 + 1000) + # desc + sql = f"desc {db}.{stb};" tdSql.query(sql) tdSql.checkRows(2 + 1000) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 6d56aa7fe2..5ddf773dbb 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -730,6 +730,10 @@ bool shellIsShowWhole(const char *sql) { if (taosStrCaseStr(sql, "describe ") != NULL) { return true; } + // desc + if (taosStrCaseStr(sql, "desc ") != NULL) { + return true; + } // show if (taosStrCaseStr(sql, "show ") != NULL) { return true; From cf18ffe4cee873d15c0b31a78e94f58b534c06c9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 10:24:22 +0800 Subject: [PATCH 04/20] feat: add error code to show --- tools/shell/src/shellEngine.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 6d56aa7fe2..713f6757da 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1091,7 +1091,7 @@ void shellCleanupHistory() { void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - fprintf(stderr, "\r\nDB error: %s (%.6fs)\r\n", taos_errstr(tres), (et - st) / 1E6); + fprintf(stderr, "\r\nDB error: %s[%08X] (%.6fs)\r\n", taos_errstr(tres), taos_errno(tres), (et - st) / 1E6); taos_free_result(tres); } @@ -1303,6 +1303,8 @@ int32_t shellExecute() { #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { if (shell_conn_ws_server(1)) { + printf("failed to connect to server, error code:0x08x reason: %s\n", ws_errno(NULL), ws_errstr(NULL)); + fflush(stdout); return -1; } } else { @@ -1314,7 +1316,7 @@ int32_t shellExecute() { } if (shell.conn == NULL) { - printf("failed to connect to server, reason: %s\n", taos_errstr(NULL)); + printf("failed to connect to server, reason: %s[%0x08x]\n",taos_errstr(NULL), taos_errno(NULL)); fflush(stdout); return -1; } From c4a5969d7f263c07982bdd1773dc6185fb120aa1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 10:41:06 +0800 Subject: [PATCH 05/20] uniform show format --- tools/shell/src/shellEngine.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 713f6757da..18d56c23aa 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1091,7 +1091,7 @@ void shellCleanupHistory() { void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - fprintf(stderr, "\r\nDB error: %s[%08X] (%.6fs)\r\n", taos_errstr(tres), taos_errno(tres), (et - st) / 1E6); + fprintf(stderr, "\r\nDB error:[%08X] %s (%.6fs)\r\n", taos_errno(tres), taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } @@ -1303,7 +1303,7 @@ int32_t shellExecute() { #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { if (shell_conn_ws_server(1)) { - printf("failed to connect to server, error code:0x08x reason: %s\n", ws_errno(NULL), ws_errstr(NULL)); + printf("failed to connect to server, error code:0x08X reason: %s\n", ws_errno(NULL), ws_errstr(NULL)); fflush(stdout); return -1; } @@ -1316,7 +1316,7 @@ int32_t shellExecute() { } if (shell.conn == NULL) { - printf("failed to connect to server, reason: %s[%0x08x]\n",taos_errstr(NULL), taos_errno(NULL)); + printf("failed to connect to server, error code:0x%08X reason: %s\n", taos_errno(NULL), taos_errstr(NULL)); fflush(stdout); return -1; } From 8c3a5e246456579223fbb0b213b33e87ab2c7703 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 11:03:07 +0800 Subject: [PATCH 06/20] fix: reduce show text length --- tools/shell/src/shellEngine.c | 6 +++--- tools/shell/src/shellWebsocket.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 18d56c23aa..21048a241c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1091,7 +1091,7 @@ void shellCleanupHistory() { void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - fprintf(stderr, "\r\nDB error:[%08X] %s (%.6fs)\r\n", taos_errno(tres), taos_errstr(tres), (et - st) / 1E6); + fprintf(stderr, "\r\nDB error:0x%08X %s (%.6fs)\r\n", taos_errno(tres), taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } @@ -1303,7 +1303,7 @@ int32_t shellExecute() { #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { if (shell_conn_ws_server(1)) { - printf("failed to connect to server, error code:0x08X reason: %s\n", ws_errno(NULL), ws_errstr(NULL)); + printf("failed to connect to server, code:0x08X %s\n", ws_errno(NULL), ws_errstr(NULL)); fflush(stdout); return -1; } @@ -1316,7 +1316,7 @@ int32_t shellExecute() { } if (shell.conn == NULL) { - printf("failed to connect to server, error code:0x%08X reason: %s\n", taos_errno(NULL), taos_errstr(NULL)); + printf("failed to connect to server, code:0x%08X %s\n", taos_errno(NULL), taos_errstr(NULL)); fflush(stdout); return -1; } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 1ec1697c3a..61074102be 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -286,8 +286,8 @@ void shellRunSingleCommandWebsocketImp(char *command) { // if it's not a ws connection error if (TSDB_CODE_WS_DSN_ERROR != (code&TSDB_CODE_WS_DSN_ERROR)) { et = taosGetTimestampUs(); - fprintf(stderr, "\nDB: error: %s (%.6fs)\n", - ws_errstr(res), (et - st)/1E6); + fprintf(stderr, "\nDB: error:0x%08X %s (%.6fs)\n", + ws_errno(res), ws_errstr(res), (et - st)/1E6); ws_free_result(res); return; } From 28a2dea8fd9be75fa5c4c1d4a15f326ab9b17235 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 14:23:39 +0800 Subject: [PATCH 07/20] enh: add three code show detail info --- tools/shell/inc/shellInt.h | 1 + tools/shell/src/shellArguments.c | 6 +++--- tools/shell/src/shellAuto.c | 1 + tools/shell/src/shellEngine.c | 5 +++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index ba3dadc646..6d372ec559 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -48,6 +48,7 @@ #define SHELL_FLOAT_WIDTH 20 #define SHELL_DOUBLE_WIDTH 25 +#define ERROR_CODE_DETAIL "\r\n\r\nView possible causes and suggested actions for error codes: \r\n https://docs.taosdata.com/reference/error-code/\r\n" typedef struct { char* hist[SHELL_MAX_HISTORY_SIZE]; char file[TSDB_FILENAME_LEN]; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 442329674d..6f3c7fe703 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -92,8 +92,8 @@ void shellPrintHelp() { #else printf("\r\n\r\nReport bugs to %s.\r\n", "support@taosdata.com"); #endif + printf("%s", ERROR_CODE_DETAIL); } - #ifdef LINUX #include #ifdef _ALPINE @@ -104,9 +104,9 @@ void shellPrintHelp() { const char *argp_program_version = td_version; #ifdef CUS_EMAIL -const char *argp_program_bug_address = CUS_EMAIL; +const char *argp_program_bug_address = CUS_EMAIL"ERROR_CODE_DETAIL; #else -const char *argp_program_bug_address = "support@taosdata.com"; +const char *argp_program_bug_address = "support@taosdata.com "ERROR_CODE_DETAIL; #endif static struct argp_option shellOptions[] = { diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 959e2d6d62..9fc929a595 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -662,6 +662,7 @@ void showHelp() { now - current time \n\ Example : \n\ select * from t1 where ts > now - 2w + 3d and ts <= now - 1w -2h ;\n"); + printf(ERROR_CODE_DETAIL); printf("\n"); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 21048a241c..0427868754 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1303,7 +1303,8 @@ int32_t shellExecute() { #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { if (shell_conn_ws_server(1)) { - printf("failed to connect to server, code:0x08X %s\n", ws_errno(NULL), ws_errstr(NULL)); + printf("failed to connect to server, code:0x%08X %s\n%s", ws_errno(NULL), ws_errstr(NULL), ERROR_CODE_DETAIL); + printf(ERROR_CODE_DETAIL); fflush(stdout); return -1; } @@ -1316,7 +1317,7 @@ int32_t shellExecute() { } if (shell.conn == NULL) { - printf("failed to connect to server, code:0x%08X %s\n", taos_errno(NULL), taos_errstr(NULL)); + printf("failed to connect to server, code:0x%08X %s\n%s", taos_errno(NULL), taos_errstr(NULL), ERROR_CODE_DETAIL); fflush(stdout); return -1; } From 42b4bc97deaa51d669886697fc75c96a148ba523 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 14:33:42 +0800 Subject: [PATCH 08/20] enh: add three code show detail info remove quota --- tools/shell/src/shellArguments.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 6f3c7fe703..50520b8f65 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -104,7 +104,7 @@ void shellPrintHelp() { const char *argp_program_version = td_version; #ifdef CUS_EMAIL -const char *argp_program_bug_address = CUS_EMAIL"ERROR_CODE_DETAIL; +const char *argp_program_bug_address = CUS_EMAIL ERROR_CODE_DETAIL; #else const char *argp_program_bug_address = "support@taosdata.com "ERROR_CODE_DETAIL; #endif From 5882d6bbe7ec207d5245214bb290c8cbcd419d2e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 16:48:09 +0800 Subject: [PATCH 09/20] fix: adjust show text and format --- tools/shell/inc/shellInt.h | 2 +- tools/shell/src/shellArguments.c | 5 ++--- tools/shell/src/shellAuto.c | 1 - tools/shell/src/shellEngine.c | 7 +++---- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 6d372ec559..e1efd3cf95 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -48,7 +48,7 @@ #define SHELL_FLOAT_WIDTH 20 #define SHELL_DOUBLE_WIDTH 25 -#define ERROR_CODE_DETAIL "\r\n\r\nView possible causes and suggested actions for error codes: \r\n https://docs.taosdata.com/reference/error-code/\r\n" +#define ERROR_CODE_DETAIL "\r\n\r\nView possible causes and suggested actions for error codes, \r\nplease refer to page of 'Reference'-> 'Error Codes' on TDengine's official website./\r\n" typedef struct { char* hist[SHELL_MAX_HISTORY_SIZE]; char file[TSDB_FILENAME_LEN]; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 50520b8f65..09e7e035c0 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -92,7 +92,6 @@ void shellPrintHelp() { #else printf("\r\n\r\nReport bugs to %s.\r\n", "support@taosdata.com"); #endif - printf("%s", ERROR_CODE_DETAIL); } #ifdef LINUX #include @@ -104,9 +103,9 @@ void shellPrintHelp() { const char *argp_program_version = td_version; #ifdef CUS_EMAIL -const char *argp_program_bug_address = CUS_EMAIL ERROR_CODE_DETAIL; +const char *argp_program_bug_address = CUS_EMAIL; #else -const char *argp_program_bug_address = "support@taosdata.com "ERROR_CODE_DETAIL; +const char *argp_program_bug_address = "support@taosdata.com; #endif static struct argp_option shellOptions[] = { diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 9fc929a595..959e2d6d62 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -662,7 +662,6 @@ void showHelp() { now - current time \n\ Example : \n\ select * from t1 where ts > now - 2w + 3d and ts <= now - 1w -2h ;\n"); - printf(ERROR_CODE_DETAIL); printf("\n"); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 0427868754..7328b67dff 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1091,7 +1091,7 @@ void shellCleanupHistory() { void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - fprintf(stderr, "\r\nDB error:0x%08X %s (%.6fs)\r\n", taos_errno(tres), taos_errstr(tres), (et - st) / 1E6); + fprintf(stderr, "\r\nDB error: %s[0x%08X] (%.6fs)\r\n", taos_errstr(tres), taos_errno(tres), (et - st) / 1E6); taos_free_result(tres); } @@ -1303,8 +1303,7 @@ int32_t shellExecute() { #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { if (shell_conn_ws_server(1)) { - printf("failed to connect to server, code:0x%08X %s\n%s", ws_errno(NULL), ws_errstr(NULL), ERROR_CODE_DETAIL); - printf(ERROR_CODE_DETAIL); + printf("failed to connect to server, reason: %s[0x%08X]\n%s", ws_errstr(NULL), ws_errno(NULL), ERROR_CODE_DETAIL); fflush(stdout); return -1; } @@ -1317,7 +1316,7 @@ int32_t shellExecute() { } if (shell.conn == NULL) { - printf("failed to connect to server, code:0x%08X %s\n%s", taos_errno(NULL), taos_errstr(NULL), ERROR_CODE_DETAIL); + printf("failed to connect to server, reason: %s[0x%08X]\n%s", taos_errstr(NULL), taos_errno(NULL), ERROR_CODE_DETAIL); fflush(stdout); return -1; } From 7cd57f906f0b93a94eceb3543dc97bd6058c2a3f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 16:50:11 +0800 Subject: [PATCH 10/20] fix: lost right quotation --- tools/shell/src/shellArguments.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 09e7e035c0..83f2bc947e 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -105,7 +105,7 @@ const char *argp_program_version = td_version; #ifdef CUS_EMAIL const char *argp_program_bug_address = CUS_EMAIL; #else -const char *argp_program_bug_address = "support@taosdata.com; +const char *argp_program_bug_address = "support@taosdata.com;" #endif static struct argp_option shellOptions[] = { From d83bb1c9c1f29cd0363e79ab6bb057c089e2a7e9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 16:52:08 +0800 Subject: [PATCH 11/20] fix: add semicolon to email address --- tools/shell/src/shellArguments.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 83f2bc947e..f542d71d8b 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -105,7 +105,7 @@ const char *argp_program_version = td_version; #ifdef CUS_EMAIL const char *argp_program_bug_address = CUS_EMAIL; #else -const char *argp_program_bug_address = "support@taosdata.com;" +const char *argp_program_bug_address = "support@taosdata.com"; #endif static struct argp_option shellOptions[] = { From 9f909340297286e8a46cd30b60d841bea53ac1b4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 17:06:49 +0800 Subject: [PATCH 12/20] fix: adjust prompt text review with Daniel --- tools/shell/inc/shellInt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index e1efd3cf95..b1f09d5161 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -48,7 +48,7 @@ #define SHELL_FLOAT_WIDTH 20 #define SHELL_DOUBLE_WIDTH 25 -#define ERROR_CODE_DETAIL "\r\n\r\nView possible causes and suggested actions for error codes, \r\nplease refer to page of 'Reference'-> 'Error Codes' on TDengine's official website./\r\n" +#define ERROR_CODE_DETAIL "\r\n\r\nTo view possible causes and suggested actions for error codes, see \r\n\"Error Code Reference\" in the TDengine online documentation.\r\n" typedef struct { char* hist[SHELL_MAX_HISTORY_SIZE]; char file[TSDB_FILENAME_LEN]; From 47625d6ddcd88a540ff0f8295d3aba5ff129bb2e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 17:12:12 +0800 Subject: [PATCH 13/20] fix: add to help command --- tools/shell/src/shellAuto.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 959e2d6d62..9fc929a595 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -662,6 +662,7 @@ void showHelp() { now - current time \n\ Example : \n\ select * from t1 where ts > now - 2w + 3d and ts <= now - 1w -2h ;\n"); + printf(ERROR_CODE_DETAIL); printf("\n"); } From cdbb473f7df77d35b4e80b8dbbabf87fe39bad18 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 28 Nov 2024 17:20:13 +0800 Subject: [PATCH 14/20] restore shellArguments.c no modify status --- tools/shell/src/shellArguments.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index f542d71d8b..442329674d 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -93,6 +93,7 @@ void shellPrintHelp() { printf("\r\n\r\nReport bugs to %s.\r\n", "support@taosdata.com"); #endif } + #ifdef LINUX #include #ifdef _ALPINE From 1db4556d6a5216bc7439e6400d166748f7762243 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 28 Nov 2024 17:31:11 +0800 Subject: [PATCH 15/20] doc: fix typo in gpt doc --- docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md index dc0a534706..5b49db330e 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md @@ -52,7 +52,7 @@ class _MyAnomalyDetectionService(AbstractAnomalyDetectionService): ```SQL --- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类 -SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad') +SELECT COUNT(*) FROM foo ANOMALY_WINDOW(col, 'algo=myad') ``` 如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 From 8b56c4e27f428c4053a706f8faf9f1dfde493bdd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 28 Nov 2024 17:31:52 +0800 Subject: [PATCH 16/20] doc: fix typo in gpt doc --- docs/zh/06-advanced/06-TDgpt/06-dev/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md index b7f048cefc..8834dba7c6 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md @@ -21,7 +21,7 @@ Anode的主要目录结构如下图所示 . ├── cfg ├── model -│   └── ac_detection +│   └── ad_detection ├── release ├── script └── taosanalytics From 4679f46e054155423d7bc9dd366b2d049c789d46 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 28 Nov 2024 18:32:51 +0800 Subject: [PATCH 17/20] doc: fix some typo. --- .../06-TDgpt/05-anomaly-detection/04-machine-learning.md | 2 +- docs/zh/06-advanced/06-TDgpt/06-dev/index.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md index d72b8e70a9..ec76d6a0a3 100644 --- a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md +++ b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/04-machine-learning.md @@ -9,7 +9,7 @@ Autoencoder[1]: TDgpt 内置使用自编码器（Autoencoder）的异 --- 在 options 中增加 model 的名称，ad_autoencoder_foo， 针对 foo 数据集（表）训练的采用自编码器的异常检测模型进行异常检测 SELECT COUNT(*), _WSTART FROM foo -ANOMALY_DETECTION(col1, 'algo=encoder, model=ad_autoencoder_foo'); +ANOMALY_WINDOW(col1, 'algo=encoder, model=ad_autoencoder_foo'); ``` ### 参考文献 diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md index 8834dba7c6..6db7d13079 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md @@ -63,7 +63,7 @@ Anode采用算法自动加载模式，因此只识别符合命名约定的 Pytho ```SQL --- algo 后面的参数 name 即为类属性 `name` -SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col_name, 'algo=name') +SELECT COUNT(*) FROM foo ANOMALY_WINDOW(col_name, 'algo=name') ``` ## 添加具有模型的分析算法 @@ -78,5 +78,5 @@ SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col_name, 'algo=name') ```SQL --- 在 options 中增加 model 的名称，ad_autoencoder_foo， 针对 foo 数据集（表）训练的采用自编码器的异常检测模型进行异常检测 -SELECT COUNT(*), _WSTART FROM foo ANOMALY_DETECTION(col1, 'algo=encoder, model=ad_autoencoder_foo'); +SELECT COUNT(*), _WSTART FROM foo ANOMALY_WINDOW(col1, 'algo=encoder, model=ad_autoencoder_foo'); ``` From 65ecd64c6eb531bbe53673429653c23211bd4d1a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 28 Nov 2024 18:47:30 +0800 Subject: [PATCH 18/20] doc: add some exec examples. --- docs/zh/06-advanced/06-TDgpt/02-management.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index 9aaa123299..82af1ab350 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -107,12 +107,33 @@ node_url 是提供服务的 Anode 的 IP 和 PORT组成的字符串, 例如：`c 列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`等属性。 ```sql SHOW ANODES; + +taos> show anodes; + id | url | status | create_time | update_time | +================================================================================================================== + 1 | 192.168.0.1:6090 | ready | 2024-11-28 18:44:27.089 | 2024-11-28 18:44:27.089 | +Query OK, 1 row(s) in set (0.037205s) + ``` #### 查看提供的时序数据分析服务 ```SQL SHOW ANODES FULL; + +taos> show anodes full; + id | type | algo | +============================================================================ + 1 | anomaly-detection | shesd | + 1 | anomaly-detection | iqr | + 1 | anomaly-detection | ksigma | + 1 | anomaly-detection | lof | + 1 | anomaly-detection | grubbs | + 1 | anomaly-detection | encoder | + 1 | forecast | holtwinters | + 1 | forecast | arima | +Query OK, 8 row(s) in set (0.008796s) + ``` #### 刷新集群中的分析算法缓存 From 83b6cf15e3f25488e65d10212e58c3a085666a73 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 28 Nov 2024 19:15:57 +0800 Subject: [PATCH 19/20] doc: add more details about add algorithms with training models. --- docs/zh/06-advanced/06-TDgpt/06-dev/index.md | 29 +++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md index 6db7d13079..072a66c7d3 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md @@ -21,7 +21,7 @@ Anode的主要目录结构如下图所示 . ├── cfg ├── model -│   └── ad_detection +│   └── ad_autoencoder ├── release ├── script └── taosanalytics @@ -72,11 +72,32 @@ SELECT COUNT(*) FROM foo ANOMALY_WINDOW(col_name, 'algo=name') 将具有模型的分析算法添加到 Anode 中，首先需要在 `model` 目录中建立该算法对应的目录（目录名称可自拟），将采用该算法针对不同的输入时间序列数据生成的训练模型均需要保存在该目录下，同时目录名称要在分析算法中确定，以便能够固定加载该目录下的分析模型。为了确保模型能够正常读取加载，存储的模型使用`joblib`库进行序列化保存。 下面以自编码器（Autoencoder）为例，说明如何添加要预先训练的模型进行异常检测。 -首先我们在`model`目录中创建一个目录 -- `ad_detection`，该目录将用来保存所有使用自编码器训练的模型。然后，我们使用自编码器对 foo 表的时间序列数据进行训练，得到模型 ad_autoencoder_foo，使用 `joblib`序列化以后保存在`ad_detection` 目录中。 +首先我们在 `model `目录中创建一个目录 -- `ad_autoencoder` (见上图目录结构)，该目录将用来保存所有使用自编码器训练的模型。然后，我们使用自编码器对 foo 表的时间序列数据进行训练，得到模型 针对 foo 表的模型，我们将其命名为 `ad_autoencoder_foo`，使用 `joblib`序列化该模型以后保存在 `ad_autoencoder` 目录中。如下图所示，ad_autoencoder_foo 由两个文件构成，分别是模型文件 (ad_autoencoder_foo.dat) 和模型文件描述文件 (ad_autoencoder_foo.info)。 -使用 SQL 调用已经保存的模型，需要在调用参数中指定模型名称``model=ad_autoencoder_foo`，而 `algo=encoder` 是确定调用的自编码器生成的模型（这里的`encoder`说明调用的是自编码器算法模型，该名称是添加算法的时候在代码中定义）以便能够调用该模型。 +```bash +. +├── cfg +├── model +│   └── ad_autoencoder +│   ├── ad_autoencoder_foo.dat +│   └── ad_autoencoder_foo.info +├── release +├── script +└── taosanalytics + ├── algo + │   ├── ad + │   └── fc + ├── misc + └── test + +``` + +接下来说明如何使用 SQL 调用该模型。 +通过设置参数 `algo=ad_encoder` 告诉分析平台要调用自编码器算法训练的模型（自编码器算法在可用算法列表中），因此直接指定即可。此外还需要指定自编码器针对某数据集训练的确定的模型，此时我们需要使用已经保存的模型 `ad_autoencoder_foo` ，因此需要添加参数 `model=ad_autoencoder_foo` 以便能够调用该模型。 ```SQL --- 在 options 中增加 model 的名称，ad_autoencoder_foo， 针对 foo 数据集（表）训练的采用自编码器的异常检测模型进行异常检测 -SELECT COUNT(*), _WSTART FROM foo ANOMALY_WINDOW(col1, 'algo=encoder, model=ad_autoencoder_foo'); +SELECT COUNT(*), _WSTART +FROM foo +ANOMALY_WINDOW(col1, 'algo=ad_encoder, model=ad_autoencoder_foo'); ``` From 1037b398e0127b5544cc4e75a8b0bf80dfb21828 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 28 Nov 2024 19:16:45 +0800 Subject: [PATCH 20/20] doc: fix some typos. --- docs/zh/06-advanced/06-TDgpt/02-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index 82af1ab350..ef1206fc04 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -129,7 +129,7 @@ taos> show anodes full; 1 | anomaly-detection | ksigma | 1 | anomaly-detection | lof | 1 | anomaly-detection | grubbs | - 1 | anomaly-detection | encoder | + 1 | anomaly-detection | ad_encoder | 1 | forecast | holtwinters | 1 | forecast | arima | Query OK, 8 row(s) in set (0.008796s)